1#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12 collections::{HashMap, HashSet, VecDeque},
13 error::Error,
14 fmt, io,
15 path::PathBuf,
16 process::{ExitStatus, Stdio},
17 sync::{Arc, Mutex, OnceLock},
18 time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29 manifest::{SelfSignalKind, SignalAnchor},
30 session::{
31 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32 MODULE_CONTROL_OP_HEALTH_CHECK,
33 },
34 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37 process::{Child, Command},
38 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39 task::JoinHandle,
40 time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45 child_roster::ChildRoster,
46 daemon_config::{
47 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48 },
49 forwarding::{
50 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51 ModuleDrainTarget, PendingModuleControlRpc,
52 },
53 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54 registry::{ConnectionId, RegistrationEndReason, RegistryError},
55 stderr_tail::{
56 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57 StderrTailSnapshot,
58 },
59 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60 Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84type NonceLaunches = HashMap<String, (String, std::sync::Weak<RegistrationSpawn>)>;
86pub(crate) use registration::NonceAttestation;
87#[cfg(all(test, unix))]
88pub(crate) use registration::RegistrationProof;
89pub(crate) use registration::RegistrationSpawn;
90pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
91#[cfg(all(test, unix))]
92pub(crate) mod unix_channel_placement_tests;
93use program_binding::ProgramBindingPublication;
94mod config_exit_failure;
95#[cfg(all(test, any(target_os = "linux", windows)))]
96mod missing_path_tests;
97#[cfg(all(test, unix))]
98mod program_binding_tests;
99#[cfg(all(test, windows))]
100mod program_binding_windows_tests;
101#[cfg(all(test, windows))]
102mod windows_channel_tests;
103pub use config_exit_failure::ModuleStopReason;
104#[cfg(all(test, unix))]
105mod config_exit_hold_tests;
106pub(crate) mod containment;
107#[cfg(all(test, unix))]
108mod containment_tests;
109mod slow_retry;
110use slow_retry::{FailedRetry, FailureContext};
111mod host_health;
112use host_health::HostHealth;
113
114#[cfg(all(test, unix))]
115mod mark_readiness_tests;
116#[cfg(all(test, unix))]
117mod slow_retry_tests;
118
119pub const SUBC_ARG: &str = "--subc";
125
126const DEFAULT_MAX_RESTARTS: u32 = 3;
127const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
128const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
129const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
133pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
144pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
145const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
146const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
147const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
159const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
182pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
184const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
185pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
190
191#[cfg(target_os = "macos")]
192struct PrivacyExec {
193 content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
194 gated: bool,
195 admission_binding: Option<ProgramBindingPublication>,
196 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
197 deadline: tokio::time::Instant,
198 expected: Option<subc_os::FileIdentity>,
199 trampoline: Option<subc_os::FileIdentity>,
200 signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
201 module_id: String,
202 #[cfg(test)]
203 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
204 #[cfg(test)]
205 image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
206 #[cfg(test)]
207 image_read_delay: Option<Duration>,
208}
209
210#[cfg(all(test, target_os = "macos"))]
211pub(crate) fn test_privacy_trampoline() -> PathBuf {
212 let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
215 assert!(
219 path.exists(),
220 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
221 --bins --features test-support` or `cargo test -p subc-daemon` first",
222 path.display()
223 );
224 path
225}
226
227#[cfg(target_os = "macos")]
228fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
229 use std::io::Read;
230 let mut probe = subc_os::child_spawn::spawn(
231 std::process::Command::new(path)
232 .args(["__disclaim-exec", "--probe-suspended"])
233 .stdin(Stdio::null())
234 .stdout(Stdio::piped())
235 .stderr(Stdio::piped()),
236 )
237 .map_err(|error| {
238 format!(
239 "privacy trampoline probe failed for {}: {error}",
240 path.display()
241 )
242 })?;
243 let deadline = std::time::Instant::now() + Duration::from_secs(5);
244 let status = loop {
245 match probe.try_wait() {
246 Ok(Some(status)) => break status,
247 Ok(None) if std::time::Instant::now() < deadline => {
248 std::thread::sleep(Duration::from_millis(5))
249 }
250 result => {
251 let _ = probe.kill();
252 let _ = probe.wait();
253 return Err(format!(
254 "privacy trampoline probe failed or timed out for {}: {result:?}",
255 path.display()
256 ));
257 }
258 }
259 };
260 let mut answer = String::new();
261 if let Some(stdout) = probe.stdout.take() {
262 let _ = stdout.take(256).read_to_string(&mut answer);
263 }
264 if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
265 return Ok(());
266 }
267 let mut diagnostic = String::new();
268 if let Some(stderr) = probe.stderr.take() {
269 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
270 }
271 let cause = diagnostic
272 .trim()
273 .strip_prefix("ck-subc: own privacy identity refused: ")
274 .unwrap_or("binary does not implement the privacy trampoline protocol");
275 Err(format!(
276 "{cause}: probe of {} exited {status}",
277 path.display()
278 ))
279}
280
281#[cfg(target_os = "macos")]
282fn privacy_command(
283 spec: &ModuleSpec,
284 binding: &SpawnProgramBinding,
285 roster: &ChildRoster,
286 gated: bool,
287) -> Result<
288 (
289 Command,
290 Option<PrivacyExec>,
291 subc_os::privacy_identity::ExecAcknowledgement,
292 ),
293 SuperviseError,
294> {
295 let failure = |cause: String| {
296 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
297 SuperviseError::Spawn {
298 program: spec.program.clone(),
299 source: io::Error::other(cause),
300 cgroup_path: None,
301 }
302 };
303 let trampoline = roster.privacy_trampoline().map_err(failure)?;
304 let expected = Some(binding.identity);
305 let trampoline_image = subc_os::file_identity(&trampoline);
306 let signature = gated
307 .then(|| {
308 subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
309 .map(Arc::new)
310 })
311 .transpose()
312 .map_err(|error| failure(error.to_string()))?;
313 if expected == trampoline_image {
314 return Err(failure(
315 "privacy identity module executable is missing or is the trampoline itself".to_string(),
316 ));
317 }
318 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
319 .map_err(|error| failure(error.to_string()))?;
320 let reader =
321 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
322 let mut command = Command::new(&trampoline);
323 command
324 .arg(if gated {
325 "__disclaim-exec-suspended"
326 } else {
327 "__disclaim-exec"
328 })
329 .arg(ack.fd().to_string())
330 .arg(&binding.path);
331 Ok((
332 command,
333 Some(PrivacyExec {
334 gated,
335 admission_binding: None,
336 content_check: None,
337 reader,
338 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
339 expected,
340 trampoline: trampoline_image,
341 signature,
342 module_id: spec.module_id.clone(),
343 #[cfg(test)]
344 image_probe: None,
345 #[cfg(test)]
346 image_read_delay: spec
347 .env
348 .iter()
349 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
350 .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
351 #[cfg(test)]
352 confirmation_pause: spec
353 .env
354 .iter()
355 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
356 .map(|(_, address)| (address.clone(), None)),
357 }),
358 ack,
359 ))
360}
361
362#[cfg(unix)]
363type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
364#[cfg(windows)]
365type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
366#[cfg(test)]
367type CapturedRegistrationChannels =
368 Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
369#[cfg(test)]
370struct CapturedRegistrationEndpoint {
371 _channel: SpawnRegistrationChannel,
372}
373#[cfg(test)]
374impl fmt::Debug for CapturedRegistrationEndpoint {
375 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
376 f.write_str("CapturedRegistrationEndpoint")
377 }
378}
379
380struct SupervisedChild {
381 registration_spawn: Option<Arc<RegistrationSpawn>>,
382 #[cfg(any(unix, windows))]
384 registration_channel: Option<SpawnRegistrationChannel>,
385 live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
386 registered_config_connection: Option<ConnectionId>,
387 admission_base: Option<subc_config::Digest>,
388 registration_declaration: Option<String>,
389 _program_binding: Option<ProgramBindingPublication>,
390 subtree_target: crate::subtree_resources::Target,
391 child: containment::ProcessChild,
392 exec: crate::activity::ExecOwner,
393 #[cfg(windows)]
394 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
395 #[cfg(target_os = "macos")]
396 privacy_exec: Option<PrivacyExec>,
397 #[cfg(target_os = "macos")]
404 report_ready: Arc<OnceLock<()>>,
405 spawn_failure: Option<String>,
408 admission_kill_sent: bool,
411 protocol: ModuleProtocol,
416 #[cfg(target_os = "linux")]
423 module_id: String,
424 #[cfg(target_os = "linux")]
425 cgroup_placement: Option<subc_cgroup::Placement>,
426 #[cfg(windows)]
449 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
450 stdout_pump: Option<JoinHandle<()>>,
451 stderr_pump: Option<StderrPump>,
452 stderr_ring: Arc<Mutex<StderrRing>>,
453 spawned_at: Instant,
456 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
457 registration_deadline: Option<Instant>,
460 registration_watch: Option<RegistrationWatch>,
461 spawned_at_ms: u64,
462 spawned_from: PathBuf,
463 launch_spec_digest: LaunchSpecDigest,
464 spawned_file_identity: Option<SpawnedFileIdentity>,
465 process_start_time: Option<u64>,
466 process_identity: Option<ProcessIdentity>,
467 pid: u32,
468 roster_guard: Option<crate::child_roster::RosterGuard>,
471}
472
473impl SupervisedChild {
474 fn retire_registration(&self) {
475 if let Some(spawn) = &self.registration_spawn {
480 spawn.retire();
481 } else if let Some(binding) = &self._program_binding {
482 binding.retire();
483 }
484 #[cfg(unix)]
485 if let Some(channel) = &self.registration_channel {
486 let _ = channel.shutdown(std::net::Shutdown::Both);
487 }
488 #[cfg(windows)]
489 if let Some(channel) = &self.registration_channel {
490 if let Ok(shutdown) = channel.shutdown_handle() {
491 shutdown.shutdown();
492 }
493 }
494 }
495 fn registered_since_spawn(&self) -> bool {
496 self.registration_watch
497 .as_ref()
498 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
499 }
500
501 fn awaiting_registration(&self) -> bool {
502 let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
506 !admission_failed && !self.registered_since_spawn()
507 }
508
509 fn id(&self) -> Option<u32> {
510 Some(self.pid)
511 }
512
513 fn process_identity(&self) -> Option<ProcessIdentity> {
514 self.process_identity
515 }
516
517 fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
518 let result = self.child.try_wait();
519 if !matches!(result, Ok(None)) {
520 self.retire_registration();
521 #[cfg(any(unix, windows))]
522 self.registration_channel.take();
523 if let Some(binding) = &self._program_binding {
524 binding.retire();
525 }
526 }
527 result
528 }
529
530 async fn wait(&mut self) -> io::Result<ExitStatus> {
531 #[cfg(target_os = "macos")]
532 self.confirm_privacy_exec().await;
533 let result = self.child.wait().await;
541 self.retire_registration();
542 #[cfg(any(unix, windows))]
543 self.registration_channel.take();
544 if let Some(binding) = &self._program_binding {
547 binding.retire();
548 }
549 #[cfg(all(test, target_os = "linux"))]
550 if let Some(binding) = &self._program_binding {
551 binding.pause_after_wait_for_test().await;
552 }
553 #[cfg(target_os = "linux")]
554 if result.is_ok() && self.child.container().is_none() {
555 if let Some(placement) = self.cgroup_placement.as_ref() {
556 cleanup_reaped_cgroup(placement, &self.module_id).await;
560 self.cgroup_placement = None;
561 }
562 }
563 result
564 }
565
566 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
567 self.retire_registration();
568 if let Some(binding) = self._program_binding.take() {
570 binding.retire();
571 }
572 if self.child.container().is_some() {
573 if !carry {
574 self.child.terminate_container().await;
575 }
576 } else {
577 #[cfg(target_os = "linux")]
578 if let Some(placement) = self.cgroup_placement.as_ref() {
579 cleanup_reaped_cgroup(placement, &self.module_id).await;
580 self.cgroup_placement = None;
581 }
582 }
583 self.child.reap().await?;
584 self.release_roster();
585 Ok(())
586 }
587
588 #[cfg(target_os = "macos")]
589 async fn confirm_privacy_exec(&mut self) {
590 if self
591 .privacy_exec
592 .as_ref()
593 .is_some_and(|pending| pending.content_check.is_some())
594 {
595 self.finish_privacy_content_check().await;
596 return;
597 }
598 let Some(pending) = &mut self.privacy_exec else {
599 return;
600 };
601 let result = tokio::time::timeout_at(pending.deadline, async {
602 let mut record = Vec::new();
603 loop {
604 let mut ready = pending.reader.readable().await?;
605 let read = ready.try_io(|reader| {
606 use std::io::Read;
607 let mut reader = reader.get_ref();
608 let mut buffer = [0u8; 256];
609 reader.read(&mut buffer).map(|count| (count, buffer))
610 });
611 match read {
612 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
613 Ok(Ok((count, buffer))) => {
614 if record.len() + count > 1024 {
615 return Err(io::Error::other(
616 "privacy exec refusal record is too long",
617 ));
618 }
619 record.extend_from_slice(&buffer[..count]);
620 }
621 Ok(Err(error)) => return Err(error),
622 Err(_) => continue,
623 }
624 }
625 })
626 .await;
627 #[cfg(test)]
628 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
629 use tokio::io::{AsyncReadExt, AsyncWriteExt};
630 if stream.is_none() {
631 *stream = Some(
632 tokio::net::TcpStream::connect(address.as_str())
633 .await
634 .unwrap(),
635 );
636 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
637 stream
638 .as_mut()
639 .unwrap()
640 .write_all(&self.pid.to_ne_bytes())
641 .await
642 .unwrap();
643 }
644 let mut reply = [0];
645 stream
646 .as_mut()
647 .unwrap()
648 .read_exact(&mut reply)
649 .await
650 .unwrap();
651 pending.confirmation_pause = None;
652 if &reply == b"F" {
653 Ok(Err(io::Error::other("injected late exec refusal")))
654 } else {
655 assert_eq!(&reply, b"X");
656 result
657 }
658 } else {
659 result
660 };
661 let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
666 Some(self.try_wait())
667 } else {
668 None
669 };
670 let pending = self.privacy_exec.as_ref().expect("pending exec");
671 let cause = match result {
672 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
673 Ok(Err(error)) => Some(format!(
674 "privacy identity exec acknowledgement failed: {error}"
675 )),
676 Ok(Ok(record)) if !record.is_empty() => Some(
677 std::str::from_utf8(&record)
678 .ok()
679 .and_then(|record| {
680 record
681 .trim()
682 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
683 })
684 .filter(|cause| !cause.is_empty())
685 .unwrap_or("invalid privacy exec refusal record")
686 .to_string(),
687 ),
688 Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
689 Ok(Some(_)) => None,
690 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
691 Ok(None) => {
692 #[cfg(test)]
693 if let Some(delay) = pending.image_read_delay {
694 tokio::time::sleep(delay).await;
695 }
696 let image = observe_spawned_image(self.pid);
697 #[cfg(test)]
698 let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
699 if let Some(image) = image.filter(|image| {
703 image.executable.is_some() && image.executable != pending.trampoline
704 }) {
705 match self
706 .roster_guard
707 .as_ref()
708 .map(|guard| guard.confirm_executable(image))
709 .transpose()
710 {
711 Err(error) => Some(format!(
712 "could not durably record confirmed module image: {error}"
713 )),
714 Ok(_) => {
715 let _ = self.report_ready.set(());
716 None
717 }
718 }
719 } else if image.is_none()
720 || image.is_some_and(|image| image.executable.is_none())
721 {
722 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
725 Ok(result) => {
726 self.retire_registration();
727 self.registration_channel.take();
728 if let Some(binding) = &self._program_binding {
729 binding.retire();
730 }
731 result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
732 }
733 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
734 }
735 } else {
736 Some("privacy identity trampoline did not exec the module".to_string())
737 }
738 }
739 },
740 Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
741 Ok(Some(_)) => {
742 Some("privacy identity target exited before suspended admission".to_string())
743 }
744 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
745 Ok(None) => {
746 let stopped = tokio::time::timeout_at(pending.deadline, async {
749 loop {
750 if subc_os::privacy_identity::SuspendedSignature::is_suspended(
751 self.pid,
752 )? {
753 return Ok::<_, io::Error>(());
754 }
755 tokio::time::sleep(Duration::from_millis(1)).await;
756 }
757 })
758 .await;
759 match stopped {
760 Err(_) => {
761 Some("privacy identity target was not suspended within 5s".to_string())
762 }
763 Ok(Err(error)) => Some(format!(
764 "privacy identity suspended target unavailable: {error}"
765 )),
766 Ok(Ok(())) => {
767 let image = observe_spawned_image(self.pid);
768 if image.is_some_and(|image| {
769 image.executable.is_some()
770 && image.executable != pending.trampoline
771 && image.executable == pending.expected
772 }) {
773 None
774 } else {
775 Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
776 }
777 }
778 }
779 }
780 },
781 };
782 #[cfg(test)]
783 if let Some(binding) = &self._program_binding {
784 binding.pause_after_wait_for_test().await;
785 }
786 if cause.is_none() && pending.gated {
787 let publication = pending.admission_binding.clone();
788 let live = self.live_program_binding.clone();
789 let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
790 let pid = self.pid;
791 let gated = pending.gated;
792 self.privacy_exec
793 .as_mut()
794 .expect("pending exec")
795 .content_check = Some(tokio::task::spawn_blocking(move || {
796 signature.verify(pid).map_err(|error| {
797 crate::config_gate::GateError::new("program_refused", error)
798 })?;
799 if !gated {
800 return Ok(());
801 }
802 match (publication, live) {
803 (Some(publication), _) => publication.verify(),
804 (None, Some(live)) => live.check(),
805 _ => Err(crate::config_gate::GateError::new(
806 "config_delivery_deferred",
807 "live program witness is unavailable",
808 )),
809 }
810 }));
811 self.finish_privacy_content_check().await;
812 } else {
813 self.finish_privacy_exec(cause);
814 }
815 }
816
817 #[cfg(target_os = "macos")]
818 async fn finish_privacy_content_check(&mut self) {
819 let result = self
820 .privacy_exec
821 .as_mut()
822 .expect("pending exec")
823 .content_check
824 .as_mut()
825 .expect("pending content check")
826 .await;
827 let cause = match result {
828 Ok(Ok(())) => None,
829 Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
830 Err(error) => Some(format!(
831 "config_delivery_deferred: content check stopped: {error}"
832 )),
833 };
834 self.finish_privacy_exec(cause);
835 }
836
837 #[cfg(target_os = "macos")]
838 fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
839 let mut pending = self.privacy_exec.take().expect("pending exec");
840 if cause.is_none() && pending.gated {
841 let admission = (|| -> io::Result<()> {
842 let image = observe_spawned_image(self.pid)
843 .filter(|image| image.executable == pending.expected)
844 .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
845 if let Some(guard) = &self.roster_guard {
846 guard.confirm_executable(image)?;
847 }
848 pending
849 .signature
850 .as_ref()
851 .expect("gated signature")
852 .resume(self.pid)?;
853 let _ = self.report_ready.set(());
854 info!(module_id = %pending.module_id, pid = self.pid,
855 "verified suspended module resumed with own privacy identity");
856 Ok(())
857 })();
858 if let Err(error) = admission {
859 cause = Some(format!("program_refused: {error}"));
860 }
861 }
862 if cause.is_some() {
863 if let Some(spawn) = &self.registration_spawn {
864 spawn.refuse();
865 }
866 if let Some(binding) = &mut pending.admission_binding {
867 binding.reject();
868 }
869 } else if let Some(spawn) = &self.registration_spawn {
870 spawn.ready();
871 }
872 self.exec.0.finish(if cause.is_some() {
875 crate::activity::ExecState::Refused
876 } else {
877 crate::activity::ExecState::CommittedExec
878 });
879 if let Some(cause) = cause {
880 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
881 self.spawn_failure = Some(
882 if pending.gated
883 && (cause.contains("executable mismatch")
884 || cause.starts_with("program_refused:"))
885 {
886 format!("program_refused: {cause}")
887 } else if pending.gated {
888 format!("config_delivery_deferred: {cause}")
889 } else {
890 cause
891 },
892 );
893 let _ = self.start_admission_kill();
896 }
897 }
898
899 fn release_roster(&mut self) {
903 self.roster_guard = None;
904 }
905
906 fn start_kill(&mut self) -> io::Result<()> {
919 self.retire_registration();
920 if let Some(container) = self.child.container() {
921 container.hard_kill();
922 return self.kill_leader();
923 }
924 #[cfg(windows)]
925 if let Some(job) = &self.job {
926 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
927 debug!(
928 error = %error,
929 "job termination failed; the direct-child kill still owns the outcome"
930 );
931 }
932 }
933 #[cfg(target_os = "linux")]
934 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
935 self.child.start_kill()
936 }
937
938 fn kill_leader(&mut self) -> io::Result<()> {
939 self.retire_registration();
940 self.child.start_kill()
941 }
942
943 fn start_admission_kill(&mut self) -> io::Result<()> {
944 self.retire_registration();
945 #[cfg(windows)]
946 let result = {
947 let result = self
950 .spawned_file_identity
951 .as_ref()
952 .and_then(|identity| identity.windows_image.as_ref())
953 .ok_or_else(|| io::Error::other("admission child handle unavailable"))
954 .and_then(|image| {
955 image.process().terminate_with_exit_code(
956 admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
957 )
958 });
959 if let Some(container) = self.child.container() {
960 container.hard_kill();
961 } else if let Some(job) = &self.job {
962 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
963 debug!(%error, "job termination failed after admission leader termination");
964 }
965 }
966 result
967 };
968 #[cfg(not(windows))]
969 let result = self.start_kill().map(|()| true);
970 #[cfg(unix)]
971 let result = {
972 let group_sent = self.child.container().is_none()
975 && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
976 rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
977 .is_ok()
978 });
979 if group_sent {
980 Ok(true)
981 } else {
982 result
983 }
984 };
985 self.admission_kill_sent |= result?;
986 Ok(())
987 }
988
989 async fn drain_stderr(&mut self, module_id: &str) {
990 if let Some(pump) = self.stdout_pump.take() {
991 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
992 }
993
994 let Some(pump) = self.stderr_pump.take() else {
995 return;
996 };
997 settle_stderr_pump(
998 module_id,
999 &self.stderr_ring,
1000 pump,
1001 STDERR_PUMP_DRAIN_TIMEOUT,
1002 )
1003 .await;
1004 }
1005}
1006
1007async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1011 match timeout(bound, &mut pump).await {
1012 Ok(Ok(())) => {}
1013 Ok(Err(error)) => {
1014 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1015 }
1016 Err(_) => {
1017 drop(pump);
1018 warn!(
1019 module_id,
1020 waited = ?bound,
1021 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1022 );
1023 }
1024 }
1025}
1026
1027struct StderrPump {
1030 task: JoinHandle<()>,
1031 generation: u64,
1032}
1033
1034async fn settle_stderr_pump(
1040 module_id: &str,
1041 ring: &Arc<Mutex<StderrRing>>,
1042 pump: StderrPump,
1043 bound: Duration,
1044) {
1045 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1046 let StderrPump {
1047 mut task,
1048 generation,
1049 } = pump;
1050 lock().retire_pump(generation);
1051 match timeout(bound, &mut task).await {
1052 Ok(Ok(())) => {}
1053 Ok(Err(err)) => {
1054 let mut ring = lock();
1055 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1056 ring.finish_pump(generation);
1057 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1058 }
1059 Err(_) => {
1060 drop(task);
1062 lock().mark_pump_late(
1063 generation,
1064 format!(
1065 "stderr of the exited process had not reached EOF {bound:?} after it was \
1066 retired (a descendant may still hold the pipe open); lines it still \
1067 writes are kept in that process's section"
1068 ),
1069 );
1070 warn!(
1071 module_id,
1072 waited = ?bound,
1073 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1074 );
1075 }
1076 }
1077}
1078
1079fn registration_release_events() -> &'static watch::Sender<u64> {
1080 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1081 EVENTS.get_or_init(|| {
1082 let (sender, _receiver) = watch::channel(0);
1083 sender
1084 })
1085}
1086
1087pub(crate) fn notify_registration_release() {
1088 let events = registration_release_events();
1089 let next_generation = (*events.borrow()).wrapping_add(1);
1090 events.send_replace(next_generation);
1091}
1092
1093#[derive(Debug, Clone, PartialEq, Eq)]
1095pub struct ModuleSpec {
1096 pub module_id: String,
1097 pub program: PathBuf,
1098 pub args: Vec<String>,
1099 pub env: Vec<(String, String)>,
1100 pub reserved: bool,
1105 pub reserved_prefixes: Vec<String>,
1110 pub protocol: ModuleProtocol,
1129 pub overlap: ModuleOverlap,
1134 pub carry_containment: bool,
1138}
1139
1140#[derive(Clone, PartialEq, Eq)]
1143pub(crate) struct LaunchSpecDigest([u8; 32]);
1144
1145impl fmt::Debug for LaunchSpecDigest {
1146 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1147 f.write_str("LaunchSpecDigest([redacted])")
1148 }
1149}
1150
1151impl LaunchSpecDigest {
1152 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1153 fn field(hash: &mut Sha256, bytes: &[u8]) {
1154 hash.update((bytes.len() as u64).to_le_bytes());
1155 hash.update(bytes);
1156 }
1157 let mut hash = Sha256::new();
1158 if let Some(program) = spec.program.to_str() {
1159 field(&mut hash, b"utf8-program");
1160 field(&mut hash, program.as_bytes());
1161 } else {
1162 #[cfg(unix)]
1165 {
1166 use std::os::unix::ffi::OsStrExt;
1167 field(&mut hash, b"unix-program");
1168 field(&mut hash, spec.program.as_os_str().as_bytes());
1169 }
1170 #[cfg(windows)]
1171 {
1172 use std::os::windows::ffi::OsStrExt;
1173 field(&mut hash, b"windows-program");
1174 let bytes = spec
1175 .program
1176 .as_os_str()
1177 .encode_wide()
1178 .flat_map(u16::to_le_bytes)
1179 .collect::<Vec<_>>();
1180 field(&mut hash, &bytes);
1181 }
1182 #[cfg(not(any(unix, windows)))]
1183 field(&mut hash, spec.program.to_string_lossy().as_bytes());
1184 }
1185 hash.update((spec.args.len() as u64).to_le_bytes());
1186 for arg in &spec.args {
1187 field(&mut hash, arg.as_bytes());
1188 }
1189 let mut env = spec.env.iter().collect::<Vec<_>>();
1190 env.sort_unstable();
1191 hash.update((env.len() as u64).to_le_bytes());
1192 for (key, value) in env {
1193 field(&mut hash, key.as_bytes());
1194 field(&mut hash, value.as_bytes());
1195 }
1196 field(
1197 &mut hash,
1198 match spec.protocol {
1199 ModuleProtocol::Subc => b"subc",
1200 ModuleProtocol::None => b"none",
1201 },
1202 );
1203 Self(hash.finalize().into())
1204 }
1205}
1206
1207#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1214pub enum ModuleOverlap {
1215 #[default]
1217 Exclusive,
1218 Safe,
1230}
1231
1232impl ModuleOverlap {
1233 pub fn as_str(self) -> &'static str {
1234 match self {
1235 Self::Exclusive => "exclusive",
1236 Self::Safe => "safe",
1237 }
1238 }
1239}
1240
1241pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1251pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1253pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1258
1259#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1277pub struct RestartPolicy {
1278 pub max_restarts: u32,
1279 pub backoff: Duration,
1282 pub max_backoff: Duration,
1284 pub window: Duration,
1288}
1289
1290impl RestartPolicy {
1291 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1295 Self {
1296 max_restarts,
1297 backoff,
1298 max_backoff: DEFAULT_MAX_BACKOFF,
1299 window: DEFAULT_RESTART_WINDOW,
1300 }
1301 }
1302
1303 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1304 self.max_backoff = max_backoff;
1305 self
1306 }
1307
1308 pub fn with_window(mut self, window: Duration) -> Self {
1309 self.window = window;
1310 self
1311 }
1312
1313 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1318 if self.backoff.is_zero() || self.max_backoff.is_zero() {
1319 return Duration::ZERO;
1320 }
1321
1322 let mut delay = self.backoff;
1323 for _ in 0..restart_in_window {
1324 if delay >= self.max_backoff {
1325 return self.max_backoff;
1326 }
1327 delay = delay
1328 .checked_mul(10)
1329 .unwrap_or(self.max_backoff)
1330 .min(self.max_backoff);
1331 }
1332 delay.min(self.max_backoff)
1333 }
1334
1335 fn budget_exhausted_detail(&self) -> String {
1340 format!(
1341 "crash budget exhausted: max_restarts={} within window_secs={}",
1342 self.max_restarts,
1343 self.window.as_secs()
1344 )
1345 }
1346}
1347
1348impl Default for RestartPolicy {
1349 fn default() -> Self {
1350 Self {
1351 max_restarts: DEFAULT_MAX_RESTARTS,
1352 backoff: DEFAULT_BACKOFF,
1353 max_backoff: DEFAULT_MAX_BACKOFF,
1354 window: DEFAULT_RESTART_WINDOW,
1355 }
1356 }
1357}
1358
1359#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1360struct CrashRestartSchedule {
1361 restart_in_window: u32,
1362 delay: Duration,
1363}
1364
1365fn daemon_will_restart(
1372 state: &mut SupervisorSnapshot,
1373 policy: &RestartPolicy,
1374 now: Instant,
1375) -> bool {
1376 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1377}
1378
1379const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1380const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1381const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1382const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1383
1384#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1385pub enum HealthAction {
1386 Report,
1387 Restart,
1388 Alert,
1389}
1390
1391impl fmt::Display for HealthAction {
1392 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1393 f.write_str(match self {
1394 Self::Report => "report",
1395 Self::Restart => "restart",
1396 Self::Alert => "alert",
1397 })
1398 }
1399}
1400
1401#[derive(Debug, Clone, PartialEq, Eq)]
1402pub struct HealthConfig {
1403 pub http: Option<String>,
1407 pub cadence: Duration,
1408 pub deadline: Duration,
1409 pub failure_threshold: u32,
1410 pub on_degraded: HealthAction,
1411 pub on_failing: HealthAction,
1412 pub critical: bool,
1413}
1414
1415impl Default for HealthConfig {
1416 fn default() -> Self {
1417 Self {
1418 http: None,
1419 cadence: DEFAULT_HEALTH_CADENCE,
1420 deadline: DEFAULT_HEALTH_DEADLINE,
1421 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1422 on_degraded: HealthAction::Report,
1423 on_failing: HealthAction::Report,
1424 critical: false,
1425 }
1426 }
1427}
1428
1429#[derive(Debug, Clone, PartialEq)]
1447pub struct ModuleHealthStatus {
1448 pub status: SupervisorHealthStatus,
1449 pub last_probe_ms: Option<u64>,
1450 pub detail: Option<String>,
1451 pub metrics: Option<Value>,
1452 pub consecutive_failures: u32,
1453 pub late_answer_count: u64,
1456 pub last_late_answer_latency_ms: Option<u64>,
1458 pub last_action: Option<String>,
1459 pub last_action_ms: Option<u64>,
1463}
1464
1465impl Default for ModuleHealthStatus {
1466 fn default() -> Self {
1467 Self {
1468 status: SupervisorHealthStatus::Unknown,
1469 last_probe_ms: None,
1470 detail: None,
1471 metrics: None,
1472 consecutive_failures: 0,
1473 late_answer_count: 0,
1474 last_late_answer_latency_ms: None,
1475 last_action: None,
1476 last_action_ms: None,
1477 }
1478 }
1479}
1480
1481#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1483pub enum ModuleState {
1484 Starting,
1485 Running,
1486 Unresponsive,
1487 Restarting,
1488 Draining,
1489 Stopped,
1490 Failed,
1491 Disabled,
1492}
1493
1494impl fmt::Display for ModuleState {
1495 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1496 f.write_str(match self {
1497 Self::Starting => "starting",
1498 Self::Running => "running",
1499 Self::Unresponsive => "unresponsive",
1500 Self::Restarting => "restarting",
1501 Self::Draining => "draining",
1502 Self::Stopped => "stopped",
1503 Self::Failed => "failed",
1504 Self::Disabled => "disabled",
1505 })
1506 }
1507}
1508
1509#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1511pub enum ExitKind {
1512 Clean,
1513 Crash,
1514 DeliberateSeverance,
1515}
1516
1517impl From<ExitKind> for TerminalExitKind {
1518 fn from(kind: ExitKind) -> Self {
1519 match kind {
1520 ExitKind::Clean => Self::Clean,
1521 ExitKind::Crash => Self::Crash,
1522 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1523 }
1524 }
1525}
1526
1527#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1530pub(crate) struct ProcessIdentity {
1531 pub(crate) pid: u32,
1532 pub(crate) start_time: u64,
1533}
1534
1535#[derive(Debug, Clone, PartialEq, Eq)]
1537pub struct ExitReport {
1538 pub kind: ExitKind,
1539 pub code: Option<i32>,
1540 pub signal: Option<i32>,
1541 pub at_ms: u64,
1542}
1543
1544#[derive(Debug, Clone, PartialEq)]
1547pub struct ModuleStatus {
1548 pub(crate) admission_refused: Option<String>,
1549 pub(crate) legacy_registration_spawn: bool,
1550 pub(crate) registration_declaration: Option<String>,
1551 pub module_id: String,
1552 pub stop_reason: Option<ModuleStopReason>,
1555 pub waiting_for_previous_pid: Option<u32>,
1557 pub waiting_for_previous_detail: Option<String>,
1559 pub launch_nonce_source: Option<String>,
1564 pub state: ModuleState,
1565 pub enabled: bool,
1566 pub process_alive: bool,
1567 pub registration_active: bool,
1568 pub protocol: ModuleProtocol,
1573 pub live: bool,
1584 pub restart_count: u32,
1588 pub lifetime_restarts: u32,
1592 pub spawn_generation: u64,
1593 pub swap_outcome: Option<SwapOutcome>,
1595 pub max_restarts: u32,
1600 pub restart_window: Duration,
1604 pub drain_timeout: Duration,
1608 pub restart_backoff: Duration,
1609 pub restart_max_backoff: Duration,
1610 pub pid: Option<u32>,
1615 pub spawned_at_ms: Option<u64>,
1616 pub spawned_from: Option<PathBuf>,
1617 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1618 pub process_start_time: Option<u64>,
1619 pub last_exit: Option<ExitReport>,
1620 pub health: ModuleHealthStatus,
1621}
1622
1623#[cfg(test)]
1624#[derive(Debug, Clone, Copy, PartialEq)]
1625struct ActorSelectCheckpoint {
1626 generation: u64,
1627 turn: u64,
1628 registered_connection: Option<ConnectionId>,
1629 next_probe_at: Option<Instant>,
1630 wake_after: Option<Duration>,
1631}
1632
1633#[derive(Debug, Clone, PartialEq)]
1634struct SupervisorSnapshot {
1635 admission_refused: Option<String>,
1636 config_hold_parked: bool,
1638 config_hold_daemon_initiated: bool,
1641 config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1643 approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1645 legacy_registration_spawn: bool,
1646 registration_declaration: Option<String>,
1647 stop_reason: Option<ModuleStopReason>,
1648 config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1649 subtree_target: crate::subtree_resources::Target,
1650 failed_retry: FailedRetry,
1651 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1652 #[cfg(windows)]
1653 pipe_consumed: Option<Arc<OnceLock<()>>>,
1654 #[cfg(test)]
1656 actor_turns: u64,
1657 #[cfg(test)]
1660 actor_select: Option<ActorSelectCheckpoint>,
1661 state: ModuleState,
1662 enabled: bool,
1663 process_alive: bool,
1664 spawned_protocol: Option<ModuleProtocol>,
1665 spawn_failure: Option<String>,
1666 waiting_for_previous_pid: Option<u32>,
1667 waiting_for_previous_detail: Option<String>,
1668 crash_restarts: VecDeque<Instant>,
1674 lifetime_restarts: u32,
1675 spawn_generation: u64,
1684 swap_outcome: Option<SwapOutcome>,
1685 pid: Option<u32>,
1686 #[cfg(target_os = "macos")]
1687 report_ready: Option<Arc<OnceLock<()>>>,
1688 reaped_pid: Option<u32>,
1690 respawn_pending: bool,
1692 coalesced_restart_pending: bool,
1694 spawned_at_ms: Option<u64>,
1695 spawned_from: Option<PathBuf>,
1696 launch_spec_digest: Option<LaunchSpecDigest>,
1697 spawned_file_identity: Option<SpawnedFileIdentity>,
1698 process_start_time: Option<u64>,
1699 deliberate_severance: Option<ProcessIdentity>,
1700 last_exit: Option<ExitReport>,
1701 drain_disposition_detail: Option<String>,
1703 health: ModuleHealthStatus,
1704 last_probe_evidence: Option<HealthProbeEvidence>,
1705 in_alternate_slot: bool,
1710 draining_to_replace: bool,
1717 configuration_updated_since_spawn: bool,
1723}
1724
1725impl SupervisorSnapshot {
1726 fn reported_pid(&self) -> Option<u32> {
1731 #[cfg(target_os = "macos")]
1732 if self
1733 .report_ready
1734 .as_ref()
1735 .is_some_and(|ready| ready.get().is_none())
1736 {
1737 return None;
1738 }
1739 self.pid
1740 }
1741
1742 fn starting() -> Self {
1743 Self::new(ModuleState::Starting, true)
1744 }
1745
1746 fn disabled() -> Self {
1747 Self::new(ModuleState::Disabled, false)
1748 }
1749
1750 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1754 while let Some(oldest) = self.crash_restarts.front() {
1755 if now.duration_since(*oldest) > window {
1756 self.crash_restarts.pop_front();
1757 } else {
1758 break;
1759 }
1760 }
1761 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1762 }
1763
1764 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1770 self.crash_restarts.push_back(now);
1771 while self.crash_restarts.len() > policy.max_restarts as usize {
1772 self.crash_restarts.pop_front();
1773 }
1774 self.lifetime_restarts += 1;
1775 }
1776
1777 fn next_crash_restart(
1781 &mut self,
1782 policy: &RestartPolicy,
1783 now: Instant,
1784 ) -> Option<CrashRestartSchedule> {
1785 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1786 if restart_in_window >= policy.max_restarts {
1787 return None;
1788 }
1789 self.record_crash_restart(policy, now);
1790 Some(CrashRestartSchedule {
1791 restart_in_window,
1792 delay: policy.delay_for_restart(restart_in_window),
1793 })
1794 }
1795
1796 fn clear_crash_restarts(&mut self) {
1801 self.crash_restarts.clear();
1802 }
1803
1804 fn new(state: ModuleState, enabled: bool) -> Self {
1805 Self {
1806 admission_refused: None,
1807 config_hold_parked: false,
1808 config_hold_daemon_initiated: false,
1809 config_hold_approval: None,
1810 approved_start_failure: None,
1811 legacy_registration_spawn: false,
1812 registration_declaration: None,
1813 stop_reason: None,
1814 config_exit_failed_approval: None,
1815 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1816 failed_retry: FailedRetry::default(),
1817 exec_outcome: None,
1818 #[cfg(test)]
1819 actor_turns: 0,
1820 #[cfg(test)]
1821 actor_select: None,
1822 state,
1823 enabled,
1824 process_alive: false,
1825 #[cfg(windows)]
1826 pipe_consumed: None,
1827 spawned_protocol: None,
1828 spawn_failure: None,
1829 waiting_for_previous_pid: None,
1830 waiting_for_previous_detail: None,
1831 crash_restarts: VecDeque::new(),
1832 lifetime_restarts: 0,
1833 spawn_generation: 0,
1834 swap_outcome: None,
1835 pid: None,
1836 #[cfg(target_os = "macos")]
1837 report_ready: None,
1838 reaped_pid: None,
1839 respawn_pending: false,
1840 coalesced_restart_pending: false,
1841 spawned_at_ms: None,
1842 spawned_from: None,
1843 launch_spec_digest: None,
1844 spawned_file_identity: None,
1845 process_start_time: None,
1846 deliberate_severance: None,
1847 last_exit: None,
1848 drain_disposition_detail: None,
1849 health: ModuleHealthStatus::default(),
1850 last_probe_evidence: None,
1851 in_alternate_slot: false,
1852 draining_to_replace: false,
1853 configuration_updated_since_spawn: false,
1854 }
1855 }
1856}
1857
1858type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1859
1860type SpawnSubscriberKey = (ConnectionId, u64);
1861
1862#[derive(Debug)]
1863struct SpawnSubscriber {
1864 version: u8,
1865 frames: mpsc::Sender<Frame>,
1866 lagged: Option<oneshot::Sender<SpawnCursor>>,
1870}
1871
1872#[derive(Debug)]
1873struct SpawnEventState {
1874 daemon_incarnation: String,
1875 seq: u64,
1876 capacity: usize,
1877 live: HashMap<String, LiveSpawn>,
1878 generations: HashMap<String, u64>,
1879 events: VecDeque<SpawnEvent>,
1880 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1881 process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1882}
1883
1884impl Default for SpawnEventState {
1885 fn default() -> Self {
1886 Self {
1887 daemon_incarnation: "unconfigured".to_string(),
1888 seq: 0,
1889 capacity: SPAWN_EVENT_RING_CAPACITY,
1890 live: HashMap::new(),
1891 generations: HashMap::new(),
1892 events: VecDeque::new(),
1893 subscribers: HashMap::new(),
1894 process_changes: HashMap::new(),
1895 }
1896 }
1897}
1898
1899#[derive(Debug, Clone, Default)]
1900struct SpawnEventFeed(
1901 Arc<Mutex<SpawnEventState>>,
1902 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1903);
1904
1905#[derive(Debug, Clone, PartialEq, Eq)]
1906pub(crate) enum SpawnSubscribeRefusal {
1907 ForeignIncarnation { current: String },
1908 TooOld { oldest: SpawnCursor },
1909 Frame(String),
1910}
1911
1912impl SpawnEventFeed {
1913 fn configure_incarnation(&self, daemon_incarnation: String) {
1914 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1915 state.daemon_incarnation = daemon_incarnation;
1916 state.seq = 0;
1917 state.live.clear();
1918 state.generations.clear();
1919 state.events.clear();
1920 state.subscribers.clear();
1921 for changed in state.process_changes.values() {
1922 changed.send_replace(0);
1923 }
1924 }
1925
1926 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1927 SpawnCursor {
1928 daemon_incarnation: state.daemon_incarnation.clone(),
1929 seq: state.seq,
1930 }
1931 }
1932
1933 fn snapshot(&self) -> SpawnSnapshot {
1934 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1935 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1936 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1937 SpawnSnapshot {
1938 cursor: Self::cursor(&state),
1939 ring_bound: state.capacity as u64,
1940 live,
1941 }
1942 }
1943
1944 fn allocate_generation_locked(state: &mut SpawnEventState, module_id: &str) -> u64 {
1945 let generation = state
1946 .generations
1947 .get(module_id)
1948 .copied()
1949 .unwrap_or(0)
1950 .checked_add(1)
1951 .expect("spawn generation exhausted");
1952 state.generations.insert(module_id.to_string(), generation);
1953 generation
1954 }
1955
1956 fn allocate_generation(&self, module_id: &str) -> (String, u64) {
1961 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1962 let generation = Self::allocate_generation_locked(&mut state, module_id);
1963 (state.daemon_incarnation.clone(), generation)
1964 }
1965
1966 fn emit_spawned(
1970 &self,
1971 module_id: &str,
1972 allocated: Option<u64>,
1973 pid: u32,
1974 spawned_at_ms: u64,
1975 ) -> u64 {
1976 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1977 let generation = match allocated {
1978 Some(generation) => generation,
1979 None => Self::allocate_generation_locked(&mut state, module_id),
1980 };
1981 let live = LiveSpawn {
1982 module_id: module_id.to_string(),
1983 spawn_generation: generation,
1984 pid,
1985 spawned_at_ms,
1986 };
1987 state.live.insert(module_id.to_string(), live);
1988 Self::emit_locked(
1989 &mut state,
1990 SpawnEventKind::Spawned,
1991 module_id.to_string(),
1992 generation,
1993 pid,
1994 None,
1995 None,
1996 );
1997 generation
1998 }
1999
2000 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
2001 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2002 let Some(live) = state.live.remove(module_id) else {
2003 warn!(
2004 module_id,
2005 "terminal record had no live spawn event identity"
2006 );
2007 return;
2008 };
2009 Self::emit_locked(
2010 &mut state,
2011 SpawnEventKind::Exited,
2012 module_id.to_string(),
2013 live.spawn_generation,
2014 live.pid,
2015 exit_code,
2016 exit_signal,
2017 );
2018 }
2019
2020 fn emit_superseded_exited(
2027 &self,
2028 module_id: &str,
2029 spawn_generation: u64,
2030 pid: u32,
2031 exit_code: Option<i32>,
2032 exit_signal: Option<i32>,
2033 ) {
2034 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2035 if state
2036 .live
2037 .get(module_id)
2038 .is_some_and(|live| live.spawn_generation == spawn_generation)
2039 {
2040 state.live.remove(module_id);
2041 }
2042 Self::emit_locked(
2043 &mut state,
2044 SpawnEventKind::Exited,
2045 module_id.to_string(),
2046 spawn_generation,
2047 pid,
2048 exit_code,
2049 exit_signal,
2050 );
2051 }
2052
2053 #[allow(clippy::too_many_arguments)]
2054 fn emit_locked(
2055 state: &mut SpawnEventState,
2056 kind: SpawnEventKind,
2057 module_id: String,
2058 spawn_generation: u64,
2059 pid: u32,
2060 exit_code: Option<i32>,
2061 exit_signal: Option<i32>,
2062 ) {
2063 state.seq = state
2064 .seq
2065 .checked_add(1)
2066 .expect("spawn event sequence exhausted");
2067 let event = SpawnEvent {
2068 cursor: Self::cursor(state),
2069 kind,
2070 module_id,
2071 spawn_generation,
2072 pid,
2073 exit_code,
2074 exit_signal,
2075 };
2076 state
2077 .process_changes
2078 .retain(|_, changed| changed.receiver_count() > 0);
2079 if let Some(changed) = state.process_changes.get(&event.module_id) {
2080 changed.send_replace(state.seq);
2081 }
2082 state.events.push_back(event.clone());
2083 while state.events.len() > state.capacity {
2084 state.events.pop_front();
2085 }
2086 let body = match serde_json::to_vec(&event) {
2087 Ok(body) => body,
2088 Err(error) => {
2089 error!(%error, "failed to serialize supervisor spawn event");
2090 return;
2091 }
2092 };
2093 state.subscribers.retain(|(connection_id, corr), subscriber| {
2094 let frame = Frame::build_with_version(
2095 subscriber.version,
2096 FrameType::StreamData,
2097 control_flags(),
2098 0,
2099 0,
2100 *corr,
2101 body.clone(),
2102 );
2103 match frame {
2104 Ok(frame) => {
2105 if subscriber.frames.try_send(frame).is_ok() {
2106 true
2107 } else {
2108 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2109 if let Some(lagged) = subscriber.lagged.take() {
2110 let _ = lagged.send(event.cursor.clone());
2111 }
2112 false
2113 }
2114 }
2115 Err(error) => {
2116 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2117 false
2118 }
2119 }
2120 });
2121 }
2122
2123 fn subscribe(
2124 &self,
2125 connection_id: ConnectionId,
2126 corr: u64,
2127 version: u8,
2128 since: Option<SpawnCursor>,
2129 sink: FrameSink,
2130 ) -> Result<(), SpawnSubscribeRefusal> {
2131 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2132 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2133 {
2134 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2135 let replay = if let Some(since) = since {
2136 if since.daemon_incarnation != state.daemon_incarnation {
2137 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2138 current: state.daemon_incarnation.clone(),
2139 });
2140 }
2141 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2142 if since.seq < oldest.seq.saturating_sub(1) {
2143 return Err(SpawnSubscribeRefusal::TooOld { oldest });
2144 }
2145 }
2146 state
2147 .events
2148 .iter()
2149 .filter(|event| event.cursor.seq > since.seq)
2150 .cloned()
2151 .collect::<Vec<_>>()
2152 } else {
2153 Vec::new()
2154 };
2155 for event in replay {
2156 let body = serde_json::to_vec(&event)
2157 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2158 let frame = Frame::build_with_version(
2159 version,
2160 FrameType::StreamData,
2161 control_flags(),
2162 0,
2163 0,
2164 corr,
2165 body,
2166 )
2167 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2168 frames
2169 .try_send(frame)
2170 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2171 }
2172 state.subscribers.insert(
2173 (connection_id, corr),
2174 SpawnSubscriber {
2175 version,
2176 frames,
2177 lagged: Some(lagged),
2178 },
2179 );
2180 }
2181 tokio::spawn(async move {
2192 while let Some(frame) = receiver.recv().await {
2193 if sink.send(frame).await.is_err() {
2194 return;
2195 }
2196 }
2197 let Ok(first_undelivered) = lagged_rx.try_recv() else {
2198 return;
2199 };
2200 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2201 Ok(frame) => {
2202 let _ = sink.send(frame).await;
2203 }
2204 Err(error) => {
2205 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2206 }
2207 }
2208 });
2209 Ok(())
2210 }
2211
2212 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2213 let Some(subscriber) = self
2214 .0
2215 .lock()
2216 .unwrap_or_else(|p| p.into_inner())
2217 .subscribers
2218 .remove(&(connection_id, corr))
2219 else {
2220 return false;
2221 };
2222 if let Ok(frame) = Frame::build_with_version(
2223 subscriber.version,
2224 FrameType::StreamEnd,
2225 control_flags(),
2226 0,
2227 0,
2228 corr,
2229 Vec::new(),
2230 ) {
2231 tokio::spawn(async move {
2232 let _ = subscriber.frames.send(frame).await;
2233 });
2234 }
2235 true
2236 }
2237
2238 fn remove_connection(&self, connection_id: ConnectionId) {
2239 self.0
2240 .lock()
2241 .unwrap_or_else(|p| p.into_inner())
2242 .subscribers
2243 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2244 }
2245
2246 #[cfg(any(test, feature = "test-support"))]
2247 fn set_capacity(&self, capacity: usize) {
2248 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2249 }
2250
2251 #[cfg(any(test, feature = "test-support"))]
2252 fn subscriber_count(&self) -> usize {
2253 self.0
2254 .lock()
2255 .unwrap_or_else(|p| p.into_inner())
2256 .subscribers
2257 .len()
2258 }
2259}
2260
2261fn spawn_subscriber_lagged_frame(
2264 version: u8,
2265 corr: u64,
2266 first_undelivered: SpawnCursor,
2267) -> Result<Frame, String> {
2268 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2270 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2271 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2272 .to_string(),
2273 detail: Some(serde_json::json!({
2274 "first_undelivered_cursor": first_undelivered
2275 })),
2276 })
2277 .map_err(|error| error.to_string())?;
2278 Frame::build_with_version(
2279 version,
2280 FrameType::Error,
2281 crate::router::daemon_error_flags(),
2282 0,
2283 0,
2284 corr,
2285 body,
2286 )
2287 .map_err(|error| error.to_string())
2288}
2289
2290pub trait ModuleProcessLiveness: Send + Sync {
2291 fn process_live(&self, module_id: &str) -> Option<bool>;
2292
2293 fn process_replacing(&self, _module_id: &str) -> bool {
2299 false
2300 }
2301}
2302
2303#[derive(Debug, Clone, Default)]
2305pub struct SupervisorProcessLiveness {
2306 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2307}
2308
2309impl SupervisorProcessLiveness {
2310 pub fn new() -> Self {
2311 Self::default()
2312 }
2313
2314 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2315 let mut snapshots = self
2316 .snapshots
2317 .lock()
2318 .unwrap_or_else(|poisoned| poisoned.into_inner());
2319 snapshots.insert(module_id, snapshot);
2320 }
2321
2322 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2323 let mut snapshots = self
2324 .snapshots
2325 .lock()
2326 .unwrap_or_else(|poisoned| poisoned.into_inner());
2327 let is_current = snapshots
2328 .get(module_id)
2329 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2330 .unwrap_or(false);
2331 if is_current {
2332 snapshots.remove(module_id);
2333 }
2334 }
2335}
2336
2337impl ModuleProcessLiveness for SupervisorProcessLiveness {
2338 fn process_live(&self, module_id: &str) -> Option<bool> {
2339 let snapshot = {
2340 let snapshots = self
2341 .snapshots
2342 .lock()
2343 .unwrap_or_else(|poisoned| poisoned.into_inner());
2344 snapshots.get(module_id).cloned()
2345 }?;
2346 let snapshot = snapshot
2347 .lock()
2348 .unwrap_or_else(|poisoned| poisoned.into_inner());
2349 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2350 }
2351
2352 fn process_replacing(&self, module_id: &str) -> bool {
2353 let Some(snapshot) = self
2354 .snapshots
2355 .lock()
2356 .unwrap_or_else(|poisoned| poisoned.into_inner())
2357 .get(module_id)
2358 .cloned()
2359 else {
2360 return false;
2361 };
2362 let snapshot = snapshot
2363 .lock()
2364 .unwrap_or_else(|poisoned| poisoned.into_inner());
2365 snapshot.enabled
2366 && match snapshot.state {
2367 ModuleState::Restarting => true,
2368 ModuleState::Draining => snapshot.draining_to_replace,
2369 ModuleState::Starting
2370 | ModuleState::Running
2371 | ModuleState::Unresponsive
2372 | ModuleState::Stopped
2373 | ModuleState::Failed
2374 | ModuleState::Disabled => false,
2375 }
2376 }
2377}
2378
2379#[cfg(test)]
2380#[derive(Debug, Default)]
2381struct ReloadExitRecordGate {
2382 reached: tokio::sync::Notify,
2383 resume: tokio::sync::Notify,
2384}
2385
2386#[derive(Debug, Clone, Copy)]
2387enum RespawnKind {
2388 Spawn,
2389 Reload,
2390}
2391
2392#[derive(Debug, Clone)]
2393struct PendingRespawn {
2394 deadline: Instant,
2395 kind: RespawnKind,
2396 daemon_initiated: bool,
2397 approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2398}
2399
2400impl PendingRespawn {
2401 fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2402 FailureContext {
2403 approval: self.approval.clone(),
2404 ..runtime.failure_context(self.daemon_initiated)
2405 }
2406 }
2407}
2408
2409type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2410
2411#[derive(Debug, Clone)]
2412struct SupervisorRuntimeConfig {
2413 registry: Arc<Registry>,
2414 config_delivery: Option<config_delivery::ConfigDelivery>,
2415 recovery_is_daemon: bool,
2417 start_requester: Option<Principal>,
2418 pending_activity_restart: bool,
2419 provenance_probe: ExecutableIdentityProbe,
2420 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2422 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2424 restart_policy: RestartPolicy,
2425 register_timeout: Duration,
2426 drain_timeout: Duration,
2429 effective_drain_timeout: Arc<Mutex<Duration>>,
2432 default_drain_timeout: Duration,
2435 health: HealthConfig,
2436 host_health: HostHealth,
2437 connection_file_path: Option<PathBuf>,
2438 capture_logs_dir: Option<PathBuf>,
2439 forwarding: Option<Arc<ForwardingTable>>,
2440 supervisor_handle: Option<SupervisorHandle>,
2443 stderr_ring: Arc<Mutex<StderrRing>>,
2450 terminal_ring: Arc<Mutex<TerminalRing>>,
2451 spawn_events: SpawnEventFeed,
2452 child_roster: ChildRoster,
2453 #[cfg(target_os = "linux")]
2454 cgroup_placement: Option<subc_cgroup::Placement>,
2455 #[cfg(test)]
2456 test_seed_stale_facts_before_enable_spawn: bool,
2457 #[cfg(test)]
2458 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2459}
2460
2461impl SupervisorRuntimeConfig {
2462 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2463 FailureContext {
2464 daemon_initiated,
2465 approval: None,
2466 shutdown: self.child_roster.shutdown_flag(),
2467 }
2468 }
2469}
2470
2471#[derive(Debug, Clone, PartialEq, Eq)]
2472struct SupervisedConfiguration {
2473 spec: ModuleSpec,
2474 enabled: bool,
2475 health: HealthConfig,
2476 register_timeout: Duration,
2477 restart_policy: RestartPolicy,
2478}
2479
2480#[derive(Debug, Clone, Default)]
2486pub struct SupervisorHandle {
2487 #[cfg(test)]
2488 resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2489 #[cfg(test)]
2490 registration_test_channels: CapturedRegistrationChannels,
2491 registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2492 registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2493 registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2494 gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2495 program_bindings: program_binding::ProgramBindings,
2496 #[cfg(test)]
2497 identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2498 #[cfg(all(test, unix))]
2499 plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2500 host_health: HostHealth,
2501 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2502 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2503 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2504 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2505 configured_ids: Arc<Mutex<HashSet<String>>>,
2517 spawn_events: SpawnEventFeed,
2518 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2529 implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2533 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2539 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2543 nonce_launches: Arc<Mutex<NonceLaunches>>,
2549 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2557 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2563 promotion_observer: PromotionObserverSlot,
2565 operation_lock: Arc<AsyncMutex<()>>,
2569}
2570
2571#[cfg(test)]
2572#[derive(Debug)]
2573struct IdentitySamplePause {
2574 reached: std::sync::mpsc::Sender<()>,
2575 release: std::sync::mpsc::Receiver<()>,
2576}
2577
2578pub(crate) trait SwapPromotionObserver: Send + Sync {
2587 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2588}
2589
2590#[derive(Clone, Default)]
2594struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2595
2596impl fmt::Debug for PromotionObserverSlot {
2597 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2598 f.write_str("PromotionObserverSlot")
2599 }
2600}
2601
2602#[derive(Debug, Clone)]
2604struct OpenSwap {
2605 candidate_nonce: String,
2608 incumbent_nonce: Option<String>,
2611 candidate_admitted: bool,
2615}
2616
2617#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2620pub(crate) enum SwapHelloAdmission {
2621 NotSwapping,
2624 Candidate,
2626 Refused,
2629 ProtocolNone,
2632 ChannelRequired,
2635}
2636
2637#[derive(Debug, Clone, PartialEq, Eq)]
2638pub(crate) enum ReservedHelloRejection {
2639 Exact {
2640 module_id: String,
2641 },
2642 Prefix {
2643 prefix: String,
2644 owner_module_id: String,
2645 },
2646}
2647
2648#[derive(Debug, Default)]
2650pub(crate) struct ReservedHelloVerification {
2651 pub(crate) nonce_verified: bool,
2652 pub(crate) prefix_owner: Option<String>,
2653}
2654
2655#[derive(Debug)]
2656struct DaemonRestartIntent {
2657 expires: Option<Duration>,
2658 window: Duration,
2659}
2660
2661impl Default for DaemonRestartIntent {
2662 fn default() -> Self {
2663 Self {
2664 expires: None,
2665 window: Duration::from_secs(120),
2666 }
2667 }
2668}
2669
2670impl SupervisorHandle {
2671 pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2672 let runtime = {
2673 let Ok(runtime) = self.gate_runtime.lock() else {
2674 return true;
2675 };
2676 runtime.clone()
2677 };
2678 runtime.is_some_and(|runtime| {
2679 runtime
2680 .upgrade()
2681 .is_none_or(|owner| owner.module_is_gated(module))
2682 })
2683 }
2684 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2685 let mut intent = self
2686 .restart_intent
2687 .lock()
2688 .unwrap_or_else(|p| p.into_inner());
2689 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2690 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2691 }
2692
2693 #[cfg(feature = "test-support")]
2694 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2695 self.restart_intent
2696 .lock()
2697 .unwrap_or_else(|p| p.into_inner())
2698 .window = window;
2699 }
2700
2701 #[cfg(unix)]
2702 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2703 let expiry = self
2704 .restart_intent
2705 .lock()
2706 .unwrap_or_else(|p| p.into_inner())
2707 .expires
2708 .take();
2709 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2710 RouteCloseReason::Restart
2711 } else {
2712 RouteCloseReason::Disable
2713 }
2714 }
2715
2716 pub fn new() -> Self {
2717 Self::default()
2718 }
2719
2720 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2721 self.spawn_events.snapshot()
2722 }
2723
2724 pub(crate) fn subscribe_process_changes(
2727 &self,
2728 module: &str,
2729 ) -> tokio::sync::watch::Receiver<u64> {
2730 let mut state = self
2731 .spawn_events
2732 .0
2733 .lock()
2734 .unwrap_or_else(|p| p.into_inner());
2735 state
2736 .process_changes
2737 .retain(|_, changed| changed.receiver_count() > 0);
2738 let seq = state.seq;
2739 state
2740 .process_changes
2741 .entry(module.into())
2742 .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2743 .subscribe()
2744 }
2745
2746 pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2747 self.spawn_events
2748 .0
2749 .lock()
2750 .unwrap_or_else(|p| p.into_inner())
2751 .live
2752 .get(module)
2753 .cloned()
2754 }
2755
2756 pub(crate) fn subscribe_spawns(
2757 &self,
2758 connection_id: ConnectionId,
2759 corr: u64,
2760 version: u8,
2761 since: Option<SpawnCursor>,
2762 sink: FrameSink,
2763 ) -> Result<(), SpawnSubscribeRefusal> {
2764 self.spawn_events
2765 .subscribe(connection_id, corr, version, since, sink)
2766 }
2767
2768 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2769 self.spawn_events.cancel(connection_id, corr)
2770 }
2771
2772 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2773 self.spawn_events.remove_connection(connection_id);
2774 }
2775
2776 #[cfg(any(test, feature = "test-support"))]
2777 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2778 assert!(capacity > 0, "spawn event capacity must be non-zero");
2779 self.spawn_events.set_capacity(capacity);
2780 }
2781
2782 #[cfg(any(test, feature = "test-support"))]
2783 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2784 self.spawn_events.subscriber_count()
2785 }
2786
2787 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2790 self.spawn_nonces
2791 .lock()
2792 .unwrap_or_else(|poisoned| poisoned.into_inner())
2793 .insert(module_id.to_string(), nonce);
2794 self.refresh_program_binding_nonces(module_id);
2795 }
2796
2797 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2800 {
2801 let mut reserved = self
2802 .reserved_nonces
2803 .lock()
2804 .unwrap_or_else(|p| p.into_inner());
2805 self.implicit_protocol_none
2806 .lock()
2807 .unwrap_or_else(|p| p.into_inner())
2808 .remove(module_id);
2809 reserved.insert(module_id.to_string(), Some(nonce));
2810 }
2811 self.refresh_program_binding_nonces(module_id);
2812 }
2813
2814 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2816 let mut owners = self
2817 .reserved_prefix_owners
2818 .lock()
2819 .unwrap_or_else(|poisoned| poisoned.into_inner());
2820 owners.retain(|_, owner| owner != owner_module_id);
2821 for prefix in prefixes {
2822 owners.insert(prefix.clone(), owner_module_id.to_string());
2823 }
2824 }
2825
2826 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2828 if self.module_is_gated(module_id) {
2829 return None;
2830 }
2831 self.spawn_nonces
2832 .lock()
2833 .unwrap_or_else(|poisoned| poisoned.into_inner())
2834 .get(module_id)
2835 .cloned()
2836 }
2837
2838 #[cfg(all(test, unix))]
2839 pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2840 self.spawn_nonces.lock().unwrap().contains_key(module)
2841 }
2842
2843 pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2844 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2847 let protocol = self
2848 .get(&spec.module_id)
2849 .and_then(|module| {
2850 let snapshot = module.inner.snapshot.lock().ok()?;
2851 snapshot
2852 .process_alive
2853 .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2854 })
2855 .unwrap_or(spec.protocol);
2856 self.apply_identity_policy(spec, protocol);
2857 }
2858
2859 #[cfg(all(test, unix))]
2860 pub(crate) fn pause_plain_identity_policy_for_test(
2861 &self,
2862 module: &str,
2863 ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2864 let (reached, waiting) = std::sync::mpsc::channel();
2865 let (release, paused) = std::sync::mpsc::channel();
2866 *self.plain_identity_policy_pause.lock().unwrap() = Some((
2867 module.to_owned(),
2868 IdentitySamplePause {
2869 reached,
2870 release: paused,
2871 },
2872 ));
2873 (waiting, release)
2874 }
2875
2876 fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2877 let channel_only = self.module_is_gated(&spec.module_id);
2878 let mut current_nonces = self
2879 .spawn_nonces
2880 .lock()
2881 .unwrap_or_else(|poisoned| poisoned.into_inner());
2882 if protocol == ModuleProtocol::None || channel_only {
2883 current_nonces.remove(&spec.module_id);
2884 }
2885 let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2886 #[cfg(test)]
2887 if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2888 pause.reached.send(()).unwrap();
2889 pause.release.recv().unwrap();
2890 }
2891 let mut reserved_nonces = self
2892 .reserved_nonces
2893 .lock()
2894 .unwrap_or_else(|poisoned| poisoned.into_inner());
2895 let mut implicit = self
2896 .implicit_protocol_none
2897 .lock()
2898 .unwrap_or_else(|p| p.into_inner());
2899 if spec.reserved {
2900 implicit.remove(&spec.module_id);
2901 } else if protocol == ModuleProtocol::None {
2902 if !reserved_nonces.contains_key(&spec.module_id) {
2903 implicit.insert(spec.module_id.clone());
2904 }
2905 } else if implicit.remove(&spec.module_id) {
2906 reserved_nonces.remove(&spec.module_id);
2907 }
2908 if spec.reserved || protocol == ModuleProtocol::None {
2909 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2914 }
2915 drop(implicit);
2916 drop(reserved_nonces);
2917 drop(current_nonces);
2918 self.refresh_program_binding_nonces(&spec.module_id);
2919 self.removal_tombstones
2923 .lock()
2924 .unwrap_or_else(|poisoned| poisoned.into_inner())
2925 .remove(&spec.module_id);
2926 }
2927
2928 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2933 self.reserved_hello_verification(module_id, presented)
2934 .is_ok()
2935 }
2936
2937 pub(crate) fn reserved_hello_verification(
2940 &self,
2941 module_id: &str,
2942 presented: Option<&str>,
2943 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2944 if self.module_is_gated(module_id) {
2945 return Err(ReservedHelloRejection::Exact {
2946 module_id: module_id.into(),
2947 });
2948 }
2949 self.hello_verification(module_id, presented, false)
2950 }
2951
2952 pub(crate) fn channel_hello_verification(
2953 &self,
2954 module_id: &str,
2955 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2956 if self
2957 .get(module_id)
2958 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
2959 {
2960 return Err(ReservedHelloRejection::Exact {
2961 module_id: module_id.into(),
2962 });
2963 }
2964 let mut verification = self.hello_verification(module_id, None, true)?;
2965 verification.nonce_verified = true;
2966 Ok(verification)
2967 }
2968
2969 pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
2970 self.prefix_claim_publication(owner, module_id).is_some()
2971 }
2972
2973 #[cfg(all(test, unix))]
2974 pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
2975 matches!(
2976 self.reserved_prefix_owners.try_lock(),
2977 Err(std::sync::TryLockError::WouldBlock)
2978 )
2979 }
2980
2981 pub(crate) fn prefix_claim_publication(
2985 &self,
2986 owner: &str,
2987 module_id: &str,
2988 ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
2989 if subc_protocol::manifest::validate_module_id(module_id).is_err()
2990 || self.module_is_gated(module_id)
2991 || self
2992 .reserved_nonces
2993 .lock()
2994 .unwrap_or_else(|p| p.into_inner())
2995 .contains_key(module_id)
2996 {
2997 return None;
2998 }
2999 let owners = self
3000 .reserved_prefix_owners
3001 .lock()
3002 .unwrap_or_else(|p| p.into_inner());
3003 let authorized = owners
3004 .iter()
3005 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3006 .max_by_key(|(prefix, _)| prefix.len())
3007 .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
3008 authorized.then_some(owners)
3009 }
3010
3011 fn hello_verification(
3012 &self,
3013 module_id: &str,
3014 presented: Option<&str>,
3015 channel: bool,
3016 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3017 let nonces = self
3018 .reserved_nonces
3019 .lock()
3020 .unwrap_or_else(|poisoned| poisoned.into_inner());
3021 if let Some(expected) = nonces.get(module_id) {
3022 let authorized = channel
3026 || match expected {
3027 Some(expected) => presented.is_some_and(|p| {
3028 !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
3029 }),
3030 None => false,
3031 };
3032 if authorized {
3033 return Ok(ReservedHelloVerification {
3034 nonce_verified: true,
3035 prefix_owner: None,
3036 });
3037 }
3038 return Err(ReservedHelloRejection::Exact {
3039 module_id: module_id.to_string(),
3040 });
3041 }
3042 drop(nonces);
3043
3044 let matched_prefix = self
3045 .reserved_prefix_owners
3046 .lock()
3047 .unwrap_or_else(|poisoned| poisoned.into_inner())
3048 .iter()
3049 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3050 .max_by_key(|(prefix, _)| prefix.len())
3051 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3052 let Some((prefix, owner_module_id)) = matched_prefix else {
3053 return Ok(ReservedHelloVerification::default());
3054 };
3055
3056 if module_id.len() == prefix.len()
3057 || (!channel
3058 && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3059 {
3060 return Err(ReservedHelloRejection::Prefix {
3061 prefix,
3062 owner_module_id,
3063 });
3064 }
3065
3066 let authorized = if channel {
3067 owner_module_id == module_id
3068 } else {
3069 presented.is_some_and(|presented| {
3070 self.spawn_nonces
3071 .lock()
3072 .unwrap_or_else(|poisoned| poisoned.into_inner())
3073 .get(&owner_module_id)
3074 .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3075 || self.swap_nonce_matches(&owner_module_id, presented)
3078 })
3079 };
3080 if authorized {
3081 Ok(ReservedHelloVerification {
3082 nonce_verified: true,
3083 prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3084 })
3085 } else {
3086 Err(ReservedHelloRejection::Prefix {
3087 prefix,
3088 owner_module_id,
3089 })
3090 }
3091 }
3092
3093 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3098 if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3099 return false;
3100 }
3101 let nonces = self
3102 .spawn_nonces
3103 .lock()
3104 .unwrap_or_else(|poisoned| poisoned.into_inner());
3105 let current = nonces.get(module_id).is_some_and(|expected| {
3106 !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3107 });
3108 drop(nonces);
3109 current || self.swap_nonce_matches(module_id, presented)
3112 }
3113
3114 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3116 if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3117 {
3118 return false;
3119 }
3120 let swaps = self
3121 .swaps
3122 .lock()
3123 .unwrap_or_else(|poisoned| poisoned.into_inner());
3124 swaps.get(module_id).is_some_and(|swap| {
3125 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3126 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3127 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3128 })
3129 })
3130 }
3131
3132 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3135 let incumbent_nonce = self
3136 .spawn_nonces
3137 .lock()
3138 .unwrap_or_else(|poisoned| poisoned.into_inner())
3139 .get(module_id)
3140 .cloned();
3141 self.swaps
3142 .lock()
3143 .unwrap_or_else(|poisoned| poisoned.into_inner())
3144 .insert(
3145 module_id.to_string(),
3146 OpenSwap {
3147 candidate_nonce,
3148 incumbent_nonce,
3149 candidate_admitted: false,
3150 },
3151 );
3152 self.refresh_program_binding_nonces(module_id);
3153 }
3154
3155 pub(crate) fn close_swap(&self, module_id: &str) {
3158 self.swaps
3159 .lock()
3160 .unwrap_or_else(|poisoned| poisoned.into_inner())
3161 .remove(module_id);
3162 self.refresh_program_binding_nonces(module_id);
3163 }
3164
3165 pub(crate) fn set_swap_promotion_observer(
3168 &self,
3169 observer: std::sync::Weak<dyn SwapPromotionObserver>,
3170 ) {
3171 *self
3172 .promotion_observer
3173 .0
3174 .lock()
3175 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3176 }
3177
3178 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3181 let observer = self
3182 .promotion_observer
3183 .0
3184 .lock()
3185 .unwrap_or_else(|poisoned| poisoned.into_inner())
3186 .as_ref()
3187 .and_then(std::sync::Weak::upgrade);
3188 if let Some(observer) = observer {
3189 observer.swap_promoted(registration);
3190 }
3191 }
3192
3193 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3195 self.swaps
3196 .lock()
3197 .unwrap_or_else(|poisoned| poisoned.into_inner())
3198 .contains_key(module_id)
3199 }
3200
3201 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3206 let candidate_nonce = self
3207 .swaps
3208 .lock()
3209 .unwrap_or_else(|poisoned| poisoned.into_inner())
3210 .get_mut(module_id)
3211 .map(|swap| {
3212 swap.incumbent_nonce = None;
3213 swap.candidate_nonce.clone()
3214 });
3215 self.spawn_nonces
3216 .lock()
3217 .unwrap_or_else(|p| p.into_inner())
3218 .remove(module_id);
3219 {
3220 let mut nonces = self
3221 .reserved_nonces
3222 .lock()
3223 .unwrap_or_else(|p| p.into_inner());
3224 if reserved || nonces.contains_key(module_id) {
3225 nonces.insert(module_id.into(), None);
3226 }
3227 }
3228 if self.module_is_gated(module_id) {
3229 return;
3230 }
3231 let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3232 return;
3233 };
3234 self.set_spawn_nonce(module_id, nonce.clone());
3235 if reserved {
3236 self.set_reserved_nonce(module_id, nonce);
3237 }
3238 }
3239
3240 pub(crate) fn swap_hello_admission(
3255 &self,
3256 module_id: &str,
3257 presented: Option<&str>,
3258 ) -> SwapHelloAdmission {
3259 if self.protocol_none(module_id) {
3260 return SwapHelloAdmission::ProtocolNone;
3261 }
3262 if self.module_is_gated(module_id) {
3263 return SwapHelloAdmission::ChannelRequired;
3264 }
3265 let swaps = self
3266 .swaps
3267 .lock()
3268 .unwrap_or_else(|poisoned| poisoned.into_inner());
3269 let Some(swap) = swaps.get(module_id) else {
3270 return SwapHelloAdmission::NotSwapping;
3271 };
3272 let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3273 return SwapHelloAdmission::Refused;
3274 };
3275 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3276 return if swap.candidate_admitted {
3277 SwapHelloAdmission::Refused
3278 } else {
3279 SwapHelloAdmission::Candidate
3280 };
3281 }
3282 if swap
3283 .incumbent_nonce
3284 .as_deref()
3285 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3286 {
3287 return SwapHelloAdmission::NotSwapping;
3288 }
3289 SwapHelloAdmission::Refused
3290 }
3291
3292 fn protocol_none(&self, module: &str) -> bool {
3293 self.get(module)
3294 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3295 }
3296
3297 pub(crate) fn with_swap_candidate_admission<T>(
3299 &self,
3300 module_id: &str,
3301 publish: impl FnOnce() -> (T, bool),
3302 ) -> Option<T> {
3303 let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3304 let swap = swaps
3305 .get_mut(module_id)
3306 .filter(|swap| !swap.candidate_admitted)?;
3307 let (result, admitted) = publish();
3308 swap.candidate_admitted = admitted;
3309 Some(result)
3310 }
3311
3312 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3314 self.spawn_nonce(module_id)
3315 }
3316
3317 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3319 if self.module_is_gated(module_id) {
3320 return None;
3321 }
3322 self.reserved_nonces
3323 .lock()
3324 .unwrap_or_else(|poisoned| poisoned.into_inner())
3325 .get(module_id)
3326 .cloned()
3327 .flatten()
3328 }
3329
3330 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3331 self.mark_configured(module.module_id());
3335 let mut modules = self
3336 .modules
3337 .lock()
3338 .unwrap_or_else(|poisoned| poisoned.into_inner());
3339 modules.insert(module.module_id().to_string(), module)
3340 }
3341
3342 fn mark_configured(&self, module_id: &str) {
3346 self.update_configured_ids(|ids| {
3347 ids.insert(module_id.to_string());
3348 });
3349 }
3350
3351 fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3352 let owner = self
3353 .gate_runtime
3354 .lock()
3355 .unwrap_or_else(|p| p.into_inner())
3356 .clone()
3357 .and_then(|owner| owner.upgrade());
3358 if let Some(owner) = owner {
3359 owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3360 }
3361 }
3362
3363 fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3364 match self.configured_ids.lock() {
3365 Ok(mut ids) => {
3366 update(&mut ids);
3367 self.publish_configured_ids(Some(&ids));
3370 }
3371 Err(poisoned) => {
3372 update(&mut poisoned.into_inner());
3373 self.publish_configured_ids(None);
3374 }
3375 }
3376 }
3377
3378 fn unmark_configured_unless_rostered(&self, module_id: &str) {
3382 let modules = self
3383 .modules
3384 .lock()
3385 .unwrap_or_else(|poisoned| poisoned.into_inner());
3386 if !modules.contains_key(module_id) {
3387 self.update_configured_ids(|ids| {
3388 ids.remove(module_id);
3389 });
3390 }
3391 }
3392
3393 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3401 self.configured_ids
3402 .lock()
3403 .unwrap_or_else(|poisoned| poisoned.into_inner())
3404 .contains(module_id)
3405 }
3406
3407 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3408 let modules = self
3409 .modules
3410 .lock()
3411 .unwrap_or_else(|poisoned| poisoned.into_inner());
3412 modules.get(module_id).cloned()
3413 }
3414
3415 pub(crate) fn record_late_health_answer(
3416 &self,
3417 module_id: &str,
3418 latency_ms: u64,
3419 ) -> Result<bool, SuperviseError> {
3420 let Some(module) = self.get(module_id) else {
3421 return Ok(false);
3422 };
3423 self.host_health.record_late_module(module_id);
3424 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3425 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3426 state.health.last_late_answer_latency_ms = Some(latency_ms);
3427 state.health.consecutive_failures = 0;
3435 })?;
3436 Ok(true)
3437 }
3438
3439 pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3440 self.host_health.spawn_monitor()
3441 }
3442
3443 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3449 let Some(module) = self.get(module_id) else {
3450 return Ok(false);
3451 };
3452 let snapshot = lock_snapshot(&module.inner.snapshot)?;
3453 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3454 return Ok(false);
3455 };
3456 drop(snapshot);
3457 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3458 }
3459
3460 pub fn list(&self) -> Vec<SupervisedModule> {
3461 let modules = self
3462 .modules
3463 .lock()
3464 .unwrap_or_else(|poisoned| poisoned.into_inner());
3465 let mut modules = modules.values().cloned().collect::<Vec<_>>();
3466 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3467 modules
3468 }
3469
3470 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3471 self.activity_images
3472 .lock()
3473 .unwrap_or_else(|p| p.into_inner())
3474 .remove(module_id);
3475 self.spawn_nonces
3476 .lock()
3477 .unwrap_or_else(|poisoned| poisoned.into_inner())
3478 .remove(module_id);
3479 self.close_swap(module_id);
3480 let mut reserved_nonces = self
3481 .reserved_nonces
3482 .lock()
3483 .unwrap_or_else(|poisoned| poisoned.into_inner());
3484 if reserved_nonces.contains_key(module_id) {
3485 reserved_nonces.insert(module_id.to_string(), None);
3488 }
3489 drop(reserved_nonces);
3490 self.reserved_prefix_owners
3491 .lock()
3492 .unwrap_or_else(|poisoned| poisoned.into_inner())
3493 .retain(|_, owner| owner != module_id);
3494 let removed = self
3495 .modules
3496 .lock()
3497 .unwrap_or_else(|poisoned| poisoned.into_inner())
3498 .remove(module_id);
3499 self.update_configured_ids(|ids| {
3500 ids.remove(module_id);
3501 });
3502 self.remove_program_binding_records(module_id);
3503 removed
3504 }
3505
3506 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3509 self.removal_tombstones
3510 .lock()
3511 .unwrap_or_else(|poisoned| poisoned.into_inner())
3512 .insert(module_id.to_string(), unix_ms_now());
3513 }
3514
3515 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3517 self.removal_tombstones
3518 .lock()
3519 .unwrap_or_else(|poisoned| poisoned.into_inner())
3520 .get(module_id)
3521 .copied()
3522 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3523 }
3524
3525 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3530 if self.get(module_id).is_some() {
3531 return false;
3532 }
3533 let mut reserved_nonces = self
3534 .reserved_nonces
3535 .lock()
3536 .unwrap_or_else(|poisoned| poisoned.into_inner());
3537 if !matches!(reserved_nonces.get(module_id), Some(None)) {
3538 return false;
3539 }
3540 reserved_nonces.remove(module_id);
3541 self.implicit_protocol_none
3542 .lock()
3543 .unwrap_or_else(|p| p.into_inner())
3544 .remove(module_id);
3545 true
3546 }
3547
3548 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3549 Arc::clone(&self.operation_lock)
3550 }
3551}
3552
3553#[derive(Debug, Clone)]
3555pub struct Supervisor {
3556 registry: Arc<Registry>,
3557 config_delivery: Option<config_delivery::ConfigDelivery>,
3558 restart_policy: RestartPolicy,
3559 drain_timeout: Duration,
3560 connection_file_path: Option<PathBuf>,
3561 capture_logs_dir: Option<PathBuf>,
3562 forwarding: Option<Arc<ForwardingTable>>,
3563 process_liveness: Arc<SupervisorProcessLiveness>,
3564 supervisor_handle: Option<SupervisorHandle>,
3565 health: HealthConfig,
3566 host_health: HostHealth,
3567 daemon_start_clock: crate::clock::StartClock,
3568 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3569 spawn_events: SpawnEventFeed,
3570 provenance_probe: ExecutableIdentityProbe,
3571 child_roster: ChildRoster,
3574 #[cfg(target_os = "linux")]
3575 cgroup_placement: Option<subc_cgroup::Placement>,
3576 #[cfg(test)]
3577 test_after_first_spawn: AfterFirstSpawnHook,
3578}
3579
3580pub(crate) struct RegisteredConfiguredModule {
3584 spec: ModuleSpec,
3585 runtime: SupervisorRuntimeConfig,
3586 module: SupervisedModule,
3587 commands: mpsc::Receiver<SupervisorCommand>,
3588}
3589
3590#[cfg(test)]
3596#[derive(Clone, Default)]
3597struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3598
3599#[cfg(test)]
3600type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3601
3602#[cfg(test)]
3603impl fmt::Debug for AfterFirstSpawnHook {
3604 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3605 f.write_str("AfterFirstSpawnHook")
3606 }
3607}
3608
3609#[cfg(test)]
3610impl AfterFirstSpawnHook {
3611 fn run(&self, module_id: &str) {
3612 if let Some(hook) = &self.0 {
3613 hook(module_id);
3614 }
3615 }
3616}
3617
3618impl Supervisor {
3619 #[cfg(all(test, target_os = "linux"))]
3620 pub(crate) fn set_admission_hook_for_test(
3621 &self,
3622 hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3623 ) {
3624 self.child_roster.set_admission_hook_for_test(hook);
3625 }
3626
3627 #[cfg(test)]
3628 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3629 let supervisor = Self::new(registry, policy);
3630 #[cfg(target_os = "macos")]
3631 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3632 supervisor
3633 }
3634 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3639 let path = path.into();
3640 #[cfg(target_os = "macos")]
3641 {
3642 let result = probe_privacy_trampoline(&path).map(|()| path);
3643 if let Err(cause) = &result {
3644 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3645 }
3646 self.child_roster.set_privacy_trampoline(result);
3647 }
3648 #[cfg(not(target_os = "macos"))]
3649 let _ = path;
3650 self
3651 }
3652 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3655 self.child_roster.shutdown_flag()
3656 }
3657
3658 #[cfg(unix)]
3669 pub(crate) fn begin_daemon_shutdown(&self) {
3670 self.child_roster.close();
3671 if let Some(journal) = &self.terminal_journal {
3672 journal.stamp_shutdown();
3673 }
3674 }
3675
3676 #[cfg(unix)]
3681 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3682 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3683 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3684 let Some(forwarding) = &self.forwarding else {
3685 return Ok(());
3686 };
3687 let notice_started = Instant::now();
3688 let modules = forwarding
3689 .begin_daemon_drain()
3690 .map_err(SuperviseError::Forwarding)?;
3691 let deadline_ms =
3692 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3693 let shutdown_reason = self
3694 .supervisor_handle
3695 .as_ref()
3696 .map(SupervisorHandle::take_daemon_shutdown_reason)
3697 .unwrap_or(RouteCloseReason::Disable);
3698 let mut modules_told = 0;
3699 let mut notices = tokio::task::JoinSet::new();
3700 let mut drains = Vec::new();
3701 for connection in forwarding
3702 .module_connections()
3703 .map_err(SuperviseError::Forwarding)?
3704 {
3705 let target = ModuleDrainTarget {
3706 endpoint: connection.endpoint,
3707 sink: connection.sink,
3708 negotiated_ver: connection.negotiated_ver,
3709 abandoned_bindings: Vec::new(),
3710 excluded_subscriptions: 0,
3711 };
3712 notices.spawn(async move {
3713 (
3714 true,
3715 send_module_draining(
3716 &connection.module_id,
3717 shutdown_reason,
3718 deadline_ms,
3719 &target,
3720 true,
3721 )
3722 .await,
3723 )
3724 });
3725 }
3726 for (module_id, endpoint) in modules {
3727 let Some(target) = forwarding
3728 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3729 .map_err(SuperviseError::Forwarding)?
3730 else {
3731 continue;
3732 };
3733 let routes = forwarding
3734 .endpoint_routes(target.endpoint)
3735 .map_err(SuperviseError::Forwarding)?;
3736 let mut recipients = Vec::new();
3739 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3740 for route in routes {
3741 let client = route.goodbye_target;
3742 if let Some((_, channels)) = clients
3743 .iter_mut()
3744 .find(|(existing, _)| existing.connection_id == client.connection_id)
3745 {
3746 channels.push(client.channel);
3747 } else {
3748 let channel = client.channel;
3749 clients.push((client, vec![channel]));
3750 }
3751 }
3752 for (client, mut channels) in clients {
3753 channels.sort_unstable();
3754 channels.dedup();
3755 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3756 module_id: module_id.clone(),
3757 channels,
3758 reason: RouteCloseReason::Restart,
3759 })
3760 .expect("route closing serializes");
3761 recipients.push((client.sink, client.negotiated_ver, closing));
3762 }
3763 for (sink, version, body) in recipients {
3764 notices.spawn(async move {
3765 let frame = Frame::build_with_version(
3766 version,
3767 FrameType::Push,
3768 control_flags(),
3769 0,
3770 0,
3771 0,
3772 body,
3773 )
3774 .expect("bounded lifecycle notice frame builds");
3775 (false, sink.send_flushed(frame).await.is_ok())
3776 });
3777 }
3778 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3779 drains.push((module_id, target.endpoint, gauges));
3780 }
3781 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3784 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3785 match result {
3786 Ok((true, true)) => modules_told += 1,
3787 Ok((false, true)) => {}
3788 result => warn!(?result, "daemon shutdown notice delivery failed"),
3789 }
3790 }
3791 notices.abort_all();
3792 info!(
3793 ?shutdown_reason,
3794 modules_told, "daemon shutdown module notices delivered"
3795 );
3796 let notice_elapsed = notice_started.elapsed();
3797 let drain_started = Instant::now();
3798 let deadline = drain_started + DRAIN_BUDGET;
3799 let mut waits = tokio::task::JoinSet::new();
3800 for (module_id, endpoint, gauges) in drains {
3801 let forwarding = Arc::clone(forwarding);
3802 let mut runtime = self.runtime_config();
3803 runtime.health.cadence = Duration::from_millis(100);
3804 waits.spawn(async move {
3805 let wait_result = wait_for_forwarding_quiescence(
3806 &forwarding,
3807 &module_id,
3808 &runtime,
3809 endpoint,
3810 deadline,
3811 &gauges,
3812 DrainScope::Endpoint(endpoint),
3813 &mut DrainGaugeTrace::new(&gauges),
3814 )
3815 .await;
3816 forwarding
3820 .record_daemon_drain_result(
3821 endpoint,
3822 drained_after_quiescence_wait(&wait_result),
3823 )
3824 .map_err(SuperviseError::Forwarding)?;
3825 wait_result
3826 });
3827 }
3828 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3829 if !matches!(result, Ok(Ok(true))) {
3830 warn!(?result, "daemon shutdown drain did not reach quiescence");
3831 }
3832 }
3833 info!(
3836 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3837 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3838 "daemon shutdown forwarding drain complete"
3839 );
3840 Ok(())
3841 }
3842
3843 #[cfg(unix)]
3855 pub(crate) async fn end_children_for_daemon_shutdown(
3856 &self,
3857 already_escalated: bool,
3858 escalate: impl std::future::Future<Output = ()>,
3859 ) {
3860 tokio::pin!(escalate);
3861 let mut escalated = already_escalated;
3862 if let Some(forwarding) = &self.forwarding {
3863 let reason = CloseReason::new(
3864 "daemon_shutdown",
3865 "the daemon is exiting after its shutdown notice and drain",
3866 );
3867 if escalated {
3868 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3871 } else {
3872 tokio::select! {
3873 biased;
3874 _ = escalate.as_mut() => {
3875 info!("second SIGTERM: abandoning module GOODBYE delivery");
3876 escalated = true;
3877 }
3878 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3879 }
3880 }
3881 let closed = forwarding.close_all_connections(&reason);
3882 debug!(closed, "closed established connections for daemon shutdown");
3883 }
3884 let escalated_here = escalated && !already_escalated;
3888 let remaining_escalate = async move {
3889 if escalated_here {
3890 std::future::pending::<()>().await;
3891 } else {
3892 escalate.await;
3893 }
3894 };
3895 crate::child_roster::end_children_for_daemon_shutdown(
3896 &self.child_roster,
3897 escalated,
3898 remaining_escalate,
3899 )
3900 .await;
3901 }
3902
3903 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3904 Self {
3905 registry,
3906 config_delivery: None,
3907 restart_policy,
3908 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3909 connection_file_path: None,
3910 capture_logs_dir: None,
3911 forwarding: None,
3912 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3913 supervisor_handle: None,
3914 health: HealthConfig::default(),
3915 host_health: HostHealth::default(),
3916 daemon_start_clock: crate::clock::StartClock::capture(),
3917 terminal_journal: None,
3918 spawn_events: SpawnEventFeed::default(),
3919 provenance_probe: ExecutableIdentityProbe::default(),
3920 child_roster: ChildRoster::default(),
3921 #[cfg(target_os = "linux")]
3922 cgroup_placement: None,
3923 #[cfg(test)]
3924 test_after_first_spawn: AfterFirstSpawnHook::default(),
3925 }
3926 }
3927
3928 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3929 self.drain_timeout = drain_timeout;
3930 self
3931 }
3932
3933 pub(crate) fn default_drain_timeout(&self) -> Duration {
3935 self.drain_timeout
3936 }
3937
3938 pub fn with_process_liveness(
3939 mut self,
3940 process_liveness: Arc<SupervisorProcessLiveness>,
3941 ) -> Self {
3942 self.process_liveness = process_liveness;
3943 self
3944 }
3945
3946 pub fn with_config_runtime(
3948 mut self,
3949 owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3950 ) -> Self {
3951 if let Some(handle) = &self.supervisor_handle {
3952 handle.install_binding_retirement(&owner);
3953 }
3954 self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3955 self
3956 }
3957
3958 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3959 self.connection_file_path = Some(connection_file_path.into());
3960 self
3961 }
3962
3963 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3965 self.capture_logs_dir = Some(logs_dir.into());
3966 self
3967 }
3968
3969 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3972 self.spawn_events.configure_incarnation(daemon_incarnation);
3976 self
3977 }
3978
3979 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3982 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3983 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3984 path,
3985 daemon_incarnation,
3986 )));
3987 this
3988 }
3989
3990 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3991 self.child_roster.counters = forwarding.counters();
3992 self.forwarding = Some(forwarding);
3993 self
3994 }
3995
3996 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3997 if let Some(delivery) = &self.config_delivery {
3998 supervisor_handle.install_binding_retirement(&delivery.0);
3999 }
4000 self.host_health = supervisor_handle.host_health.clone();
4001 self.spawn_events = supervisor_handle.spawn_events.clone();
4002 self.supervisor_handle = Some(supervisor_handle);
4003 self
4004 }
4005
4006 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
4007 self.health = health;
4008 self
4009 }
4010
4011 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
4015 self.child_roster.record_to(path.into());
4016 self
4017 }
4018
4019 #[cfg(target_os = "linux")]
4020 pub fn with_cgroup_placement(
4021 mut self,
4022 cgroup_placement: Option<subc_cgroup::Placement>,
4023 ) -> Self {
4024 self.child_roster
4025 .set_cgroup_placement(cgroup_placement.clone());
4026 self.cgroup_placement = cgroup_placement;
4027 self
4028 }
4029
4030 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4039 validate_spec(&spec)?;
4040 self.establish_identity(&spec);
4041
4042 let runtime = self.runtime_config();
4043 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4044 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4045 let registration_watch = watch_registration(&self.registry, &spec);
4046 let spawned = spawn_child(
4047 &spec,
4048 runtime.connection_file_path.as_deref(),
4049 self.supervisor_handle.as_ref(),
4050 &runtime.stderr_ring,
4051 runtime.capture_logs_dir.as_deref(),
4052 &runtime.child_roster,
4053 runtime.spawn_delivery(),
4054 #[cfg(target_os = "linux")]
4055 runtime.cgroup_placement.as_ref(),
4056 runtime.register_timeout,
4057 );
4058 #[cfg(test)]
4059 self.test_after_first_spawn.run(&spec.module_id);
4060 let mut child = match spawned {
4061 Ok(child) => child,
4062 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4063 defer_config_spawn(
4064 &spec,
4065 &runtime,
4066 &snapshot,
4067 RespawnKind::Spawn,
4068 &runtime.failure_context(runtime.recovery_is_daemon),
4069 )?;
4070 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4071 }
4072 Err(SuperviseError::ConfigHeld { .. }) => {
4073 hold_config_spawn(
4074 &spec,
4075 &runtime,
4076 &snapshot,
4077 &runtime.failure_context(runtime.recovery_is_daemon),
4078 )?;
4079 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4080 }
4081 Err(err) => {
4082 self.abandon_unrostered(&spec.module_id);
4085 return Err(err);
4086 }
4087 };
4088 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4089 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4090 if let Some(spawn) = activity_spawn {
4091 spawn.complete(&snapshot);
4092 }
4093
4094 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4095 }
4096
4097 pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4100 validate_spec(&spec)?;
4101 self.establish_identity(&spec);
4102 let mut identity = AsyncInitialIdentity {
4103 supervisor: self,
4104 module: spec.module_id.clone(),
4105 committed: false,
4106 };
4107 let runtime = self.runtime_config();
4108 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4109 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4110 let registration_watch = watch_registration(&self.registry, &spec);
4111 let spawned = spawn_child_async(&spec, &runtime).await;
4112 #[cfg(test)]
4113 self.test_after_first_spawn.run(&spec.module_id);
4114 let mut child = match spawned {
4115 Ok(child) => child,
4116 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4117 defer_config_spawn(
4118 &spec,
4119 &runtime,
4120 &snapshot,
4121 RespawnKind::Spawn,
4122 &runtime.failure_context(runtime.recovery_is_daemon),
4123 )?;
4124 identity.committed = true;
4125 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4126 }
4127 Err(SuperviseError::ConfigHeld { .. }) => {
4128 hold_config_spawn(
4129 &spec,
4130 &runtime,
4131 &snapshot,
4132 &runtime.failure_context(runtime.recovery_is_daemon),
4133 )?;
4134 identity.committed = true;
4135 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4136 }
4137 Err(err) => {
4138 self.abandon_unrostered(&spec.module_id);
4139 return Err(err);
4140 }
4141 };
4142 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4143 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4144 if let Some(spawn) = activity_spawn {
4145 spawn.complete(&snapshot);
4146 }
4147 identity.committed = true;
4148 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4149 }
4150
4151 fn establish_identity(&self, spec: &ModuleSpec) {
4164 if let Some(supervisor_handle) = &self.supervisor_handle {
4165 supervisor_handle.apply_identity_configuration(spec);
4166 supervisor_handle.mark_configured(&spec.module_id);
4167 }
4168 }
4169
4170 fn abandon_unrostered(&self, module_id: &str) {
4173 if let Some(supervisor_handle) = &self.supervisor_handle {
4174 supervisor_handle.unmark_configured_unless_rostered(module_id);
4175 }
4176 }
4177
4178 fn mark_first_process_running(
4181 &self,
4182 spec: &ModuleSpec,
4183 runtime: &SupervisorRuntimeConfig,
4184 snapshot: &SharedSnapshot,
4185 child: &mut SupervisedChild,
4186 ) -> Result<(), SuperviseError> {
4187 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4188 containment::terminate_blocking(&runtime.child_roster.containment);
4189 self.abandon_unrostered(&spec.module_id);
4190 return Err(err);
4191 }
4192 self.process_liveness
4193 .track(spec.module_id.clone(), Arc::clone(snapshot));
4194 Ok(())
4195 }
4196
4197 pub fn supervise_configured(
4203 &self,
4204 spec: ModuleSpec,
4205 enabled: bool,
4206 ) -> Result<SupervisedModule, SuperviseError> {
4207 self.supervise_configured_requested(
4208 spec,
4209 enabled,
4210 self.health.clone(),
4211 None,
4212 self.restart_policy,
4213 DEFAULT_REGISTER_TIMEOUT,
4214 None,
4215 )
4216 }
4217
4218 pub fn supervise_configured_with_health(
4224 &self,
4225 spec: ModuleSpec,
4226 enabled: bool,
4227 health: HealthConfig,
4228 drain_timeout_ms: Option<u64>,
4229 restart_policy: RestartPolicy,
4230 ) -> Result<SupervisedModule, SuperviseError> {
4231 self.supervise_configured_requested(
4232 spec,
4233 enabled,
4234 health,
4235 drain_timeout_ms,
4236 restart_policy,
4237 DEFAULT_REGISTER_TIMEOUT,
4238 None,
4239 )
4240 }
4241
4242 #[allow(clippy::too_many_arguments)]
4243 pub(crate) fn supervise_configured_requested(
4244 &self,
4245 spec: ModuleSpec,
4246 enabled: bool,
4247 health: HealthConfig,
4248 drain_timeout_ms: Option<u64>,
4249 restart_policy: RestartPolicy,
4250 register_timeout: Duration,
4251 requester: Option<Principal>,
4252 ) -> Result<SupervisedModule, SuperviseError> {
4253 let registered = self.register_configured_with_health(
4254 spec,
4255 enabled,
4256 health,
4257 drain_timeout_ms,
4258 restart_policy,
4259 register_timeout,
4260 )?;
4261 let daemon_initiated = requester.is_none();
4262 self.start_registered_configured(registered, requester, daemon_initiated)
4263 }
4264
4265 #[allow(clippy::too_many_arguments)]
4266 pub(crate) async fn supervise_configured_requested_async(
4267 &self,
4268 spec: ModuleSpec,
4269 enabled: bool,
4270 health: HealthConfig,
4271 drain_timeout_ms: Option<u64>,
4272 restart_policy: RestartPolicy,
4273 register_timeout: Duration,
4274 requester: Option<Principal>,
4275 ) -> Result<SupervisedModule, SuperviseError> {
4276 let registered = self.register_configured_with_health(
4277 spec,
4278 enabled,
4279 health,
4280 drain_timeout_ms,
4281 restart_policy,
4282 register_timeout,
4283 )?;
4284 let daemon_initiated = requester.is_none();
4285 self.start_registered_configured_async(registered, requester, daemon_initiated)
4286 .await
4287 }
4288
4289 pub(crate) fn register_configured_with_health(
4294 &self,
4295 spec: ModuleSpec,
4296 enabled: bool,
4297 health: HealthConfig,
4298 drain_timeout_ms: Option<u64>,
4299 restart_policy: RestartPolicy,
4300 register_timeout: Duration,
4301 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4302 validate_spec(&spec)?;
4303 self.establish_identity(&spec);
4304
4305 let mut runtime = self.runtime_config();
4306 runtime.health = health;
4307 runtime.restart_policy = restart_policy;
4308 runtime.register_timeout = register_timeout;
4309 if let Some(ms) = drain_timeout_ms {
4310 runtime.drain_timeout = Duration::from_millis(ms);
4311 *runtime
4312 .effective_drain_timeout
4313 .lock()
4314 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4315 }
4316 let snapshot = if enabled {
4317 SupervisorSnapshot::starting()
4318 } else {
4319 SupervisorSnapshot::disabled()
4320 };
4321 let snapshot = {
4322 let mut snapshot = snapshot;
4323 if enabled {
4324 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4325 {
4326 snapshot.waiting_for_previous_pid = Some(pid);
4327 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4328 }
4329 }
4330 snapshot
4331 };
4332 let (module, commands) =
4333 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4334 Ok(RegisteredConfiguredModule {
4335 spec,
4336 runtime,
4337 module,
4338 commands,
4339 })
4340 }
4341
4342 pub(crate) fn start_registered_configured(
4345 &self,
4346 registered: RegisteredConfiguredModule,
4347 requester: Option<Principal>,
4348 daemon_initiated: bool,
4349 ) -> Result<SupervisedModule, SuperviseError> {
4350 let RegisteredConfiguredModule {
4351 spec,
4352 mut runtime,
4353 module,
4354 commands,
4355 } = registered;
4356 runtime.start_requester = requester;
4357 runtime.recovery_is_daemon = daemon_initiated;
4358 let snapshot = &module.inner.snapshot;
4359 let should_spawn = {
4360 let state = lock_snapshot(snapshot)?;
4361 state.enabled && state.waiting_for_previous_pid.is_none()
4362 };
4363 let child = if should_spawn {
4364 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4365 let registration_watch = watch_registration(&self.registry, &spec);
4366 let spawned = spawn_child(
4367 &spec,
4368 runtime.connection_file_path.as_deref(),
4369 self.supervisor_handle.as_ref(),
4370 &runtime.stderr_ring,
4371 runtime.capture_logs_dir.as_deref(),
4372 &runtime.child_roster,
4373 runtime.spawn_delivery(),
4374 #[cfg(target_os = "linux")]
4375 runtime.cgroup_placement.as_ref(),
4376 runtime.register_timeout,
4377 );
4378 #[cfg(test)]
4379 self.test_after_first_spawn.run(&spec.module_id);
4380 match spawned {
4381 Ok(mut child) => {
4382 arm_registration_deadline(
4383 &mut child,
4384 runtime.register_timeout,
4385 registration_watch,
4386 );
4387 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4388 if let Some(spawn) = activity_spawn {
4389 spawn.complete(snapshot);
4390 }
4391 Some(child)
4392 }
4393 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4394 defer_config_spawn(
4395 &spec,
4396 &runtime,
4397 snapshot,
4398 RespawnKind::Spawn,
4399 &runtime.failure_context(daemon_initiated),
4400 )?;
4401 None
4402 }
4403 Err(mut err) => {
4404 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4405 match hold_config_spawn(
4406 &spec,
4407 &runtime,
4408 snapshot,
4409 &runtime.failure_context(daemon_initiated),
4410 ) {
4411 Ok(()) => {
4412 self.start_monitor(spec, runtime, &module, None, commands);
4413 return Ok(module);
4414 }
4415 Err(parking_error) => err = parking_error,
4416 }
4417 }
4418 if runtime.health.critical {
4419 error!(
4420 module_id = %spec.module_id,
4421 program = %spec.program.display(),
4422 error = %err,
4423 "critical configured module failed to spawn; marking failed and alerting"
4424 );
4425 } else {
4426 error!(
4427 module_id = %spec.module_id,
4428 program = %spec.program.display(),
4429 error = %err,
4430 "configured module failed to spawn; marking failed and continuing"
4431 );
4432 }
4433 fail_snapshot(
4434 snapshot,
4435 Some(&spec.module_id),
4436 None,
4437 &runtime.failure_context(daemon_initiated),
4438 );
4439 None
4440 }
4441 }
4442 } else {
4443 None
4444 };
4445 self.start_monitor(spec, runtime, &module, child, commands);
4446 Ok(module)
4447 }
4448
4449 pub(crate) async fn start_registered_configured_async(
4450 &self,
4451 registered: RegisteredConfiguredModule,
4452 requester: Option<Principal>,
4453 daemon_initiated: bool,
4454 ) -> Result<SupervisedModule, SuperviseError> {
4455 self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4456 .await
4457 }
4458
4459 pub(crate) async fn start_registered_approved_async(
4460 &self,
4461 registered: RegisteredConfiguredModule,
4462 requester: Principal,
4463 generation: crate::config_gate::runtime::ApprovalIdentity,
4464 ) -> Result<SupervisedModule, SuperviseError> {
4465 self.start_registered_with_failure_policy(
4466 registered,
4467 Some(requester),
4468 false,
4469 Some(generation),
4470 )
4471 .await
4472 }
4473
4474 async fn start_registered_with_failure_policy(
4475 &self,
4476 registered: RegisteredConfiguredModule,
4477 requester: Option<Principal>,
4478 daemon_initiated: bool,
4479 approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4480 ) -> Result<SupervisedModule, SuperviseError> {
4481 let RegisteredConfiguredModule {
4482 spec,
4483 mut runtime,
4484 module,
4485 commands,
4486 } = registered;
4487 runtime.start_requester = requester;
4488 runtime.recovery_is_daemon = daemon_initiated;
4489 let snapshot = &module.inner.snapshot;
4490 let mut startup_error = None;
4491 let failure = FailureContext {
4492 approval: approved_generation.clone(),
4493 ..runtime.failure_context(daemon_initiated)
4494 };
4495 let should_spawn = {
4496 let state = lock_snapshot(snapshot)?;
4497 state.enabled && state.waiting_for_previous_pid.is_none()
4498 };
4499 let child = if should_spawn {
4500 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4501 let registration_watch = watch_registration(&self.registry, &spec);
4502 let spawned = spawn_child_async(&spec, &runtime).await;
4503 #[cfg(test)]
4504 self.test_after_first_spawn.run(&spec.module_id);
4505 match spawned {
4506 Ok(mut child) => {
4507 arm_registration_deadline(
4508 &mut child,
4509 runtime.register_timeout,
4510 registration_watch,
4511 );
4512 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4513 if let Some(spawn) = activity_spawn {
4514 spawn.complete(snapshot);
4515 }
4516 Some(child)
4517 }
4518 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4519 defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4520 None
4521 }
4522 Err(mut err) => {
4523 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4524 match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4525 Ok(()) => {
4526 self.start_monitor(spec, runtime, &module, None, commands);
4527 return Ok(module);
4528 }
4529 Err(parking_error) => err = parking_error,
4530 }
4531 }
4532 if runtime.health.critical {
4533 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4534 "critical configured module failed to spawn; marking failed and alerting");
4535 } else {
4536 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4537 "configured module failed to spawn; marking failed and continuing");
4538 }
4539 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4540 if let Some(generation) = approved_generation {
4541 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4542 state.approved_start_failure = Some((generation, err.to_string()));
4543 });
4544 startup_error = Some(err);
4545 }
4546 None
4547 }
4548 }
4549 } else {
4550 None
4551 };
4552 self.start_monitor(spec, runtime, &module, child, commands);
4553 startup_error.map_or(Ok(module), Err)
4557 }
4558
4559 fn runtime_config(&self) -> SupervisorRuntimeConfig {
4560 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4561 SupervisorRuntimeConfig {
4562 registry: self.registry.clone(),
4563 config_delivery: self.config_delivery.clone(),
4564 recovery_is_daemon: true,
4565 start_requester: None,
4566 pending_activity_restart: false,
4567 provenance_probe: self.provenance_probe.clone(),
4568 scheduled_respawn: Arc::default(),
4569 deferred_reload_reply: Arc::default(),
4570 restart_policy: self.restart_policy,
4571 register_timeout: DEFAULT_REGISTER_TIMEOUT,
4572 drain_timeout: self.drain_timeout,
4573 child_roster: self
4576 .child_roster
4577 .for_module(Arc::clone(&effective_drain_timeout)),
4578 effective_drain_timeout,
4579 default_drain_timeout: self.drain_timeout,
4580 health: self.health.clone(),
4581 host_health: self.host_health.clone(),
4582 connection_file_path: self.connection_file_path.clone(),
4583 capture_logs_dir: self.capture_logs_dir.clone(),
4584 forwarding: self.forwarding.clone(),
4585 supervisor_handle: self.supervisor_handle.clone(),
4586 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4587 terminal_ring: Arc::new(Mutex::new(
4588 TerminalRing::new(
4589 TerminalRingConfig::default(),
4590 self.daemon_start_clock.started_at_ms(),
4591 )
4592 .with_start_clock(self.daemon_start_clock)
4593 .with_journal(self.terminal_journal.clone())
4594 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4595 )),
4596 spawn_events: self.spawn_events.clone(),
4597 #[cfg(target_os = "linux")]
4598 cgroup_placement: self.cgroup_placement.clone(),
4599 #[cfg(test)]
4600 test_seed_stale_facts_before_enable_spawn: false,
4601 #[cfg(test)]
4602 test_reload_exit_record_gate: None,
4603 }
4604 }
4605
4606 fn supervised_module(
4607 &self,
4608 spec: ModuleSpec,
4609 runtime: SupervisorRuntimeConfig,
4610 snapshot: SharedSnapshot,
4611 child: Option<SupervisedChild>,
4612 ) -> SupervisedModule {
4613 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4614 self.start_monitor(spec, runtime, &module, child, commands);
4615 module
4616 }
4617
4618 fn register_module(
4619 &self,
4620 spec: &ModuleSpec,
4621 runtime: &SupervisorRuntimeConfig,
4622 snapshot: SharedSnapshot,
4623 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4624 runtime.host_health.track(spec, &snapshot);
4625 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4626 spec: spec.clone(),
4627 enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4628 health: runtime.health.clone(),
4629 register_timeout: runtime.register_timeout,
4630 restart_policy: runtime.restart_policy,
4631 }));
4632 let stderr_ring = Arc::clone(&runtime.stderr_ring);
4633 let terminal_ring = Arc::clone(&runtime.terminal_ring);
4634 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4635 let (tx, rx) = mpsc::channel(4);
4636 let module_id = spec.module_id.clone();
4637 let module = SupervisedModule {
4638 inner: Arc::new(SupervisedModuleInner {
4639 module_id: module_id.clone(),
4640 registry: Arc::clone(&self.registry),
4641 snapshot,
4642 configuration,
4643 stderr_ring,
4644 terminal_ring,
4645 commands: tx,
4646 monitor: Mutex::new(None),
4647 effective_drain_timeout,
4648 provenance_probe: self.provenance_probe.clone(),
4649 retained: runtime.child_roster.retained_view(),
4650 }),
4651 };
4652 if let Some(supervisor_handle) = &self.supervisor_handle {
4655 supervisor_handle.insert(module.clone());
4656 }
4657 (module, rx)
4658 }
4659
4660 fn start_monitor(
4661 &self,
4662 spec: ModuleSpec,
4663 runtime: SupervisorRuntimeConfig,
4664 module: &SupervisedModule,
4665 child: Option<SupervisedChild>,
4666 commands: mpsc::Receiver<SupervisorCommand>,
4667 ) {
4668 let monitor = tokio::spawn(supervise_loop(
4669 spec,
4670 runtime,
4671 Arc::clone(&self.registry),
4672 Arc::clone(&self.process_liveness),
4673 Arc::clone(&module.inner.snapshot),
4674 child,
4675 commands,
4676 ));
4677 *module
4678 .inner
4679 .monitor
4680 .lock()
4681 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4682 }
4683}
4684
4685impl Default for Supervisor {
4686 fn default() -> Self {
4687 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4688 }
4689}
4690
4691#[derive(Clone)]
4693pub struct SupervisedModule {
4694 inner: Arc<SupervisedModuleInner>,
4695}
4696
4697struct SupervisedModuleInner {
4698 retained: crate::child_roster::RetainedView,
4699 module_id: String,
4700 registry: Arc<Registry>,
4701 snapshot: SharedSnapshot,
4702 configuration: Arc<Mutex<SupervisedConfiguration>>,
4703 stderr_ring: Arc<Mutex<StderrRing>>,
4704 terminal_ring: Arc<Mutex<TerminalRing>>,
4705 commands: mpsc::Sender<SupervisorCommand>,
4706 monitor: Mutex<Option<JoinHandle<()>>>,
4707 effective_drain_timeout: Arc<Mutex<Duration>>,
4708 provenance_probe: ExecutableIdentityProbe,
4709}
4710
4711impl fmt::Debug for SupervisedModule {
4712 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4713 f.debug_struct("SupervisedModule")
4714 .field("module_id", &self.inner.module_id)
4715 .field("status", &self.status())
4716 .finish_non_exhaustive()
4717 }
4718}
4719
4720impl SupervisedModule {
4721 pub(crate) fn check_approved_start_failure(
4722 &self,
4723 generation: &crate::config_gate::runtime::ApprovalIdentity,
4724 ) -> Result<bool, SuperviseError> {
4725 let state = lock_snapshot(&self.inner.snapshot)?;
4726 if let Some((failed_generation, detail)) = &state.approved_start_failure {
4727 if failed_generation == generation {
4728 return Err(SuperviseError::ReloadFailed {
4729 module_id: self.inner.module_id.clone(),
4730 reason: detail.clone(),
4731 });
4732 }
4733 }
4734 Ok(state.approved_start_failure.is_some())
4735 }
4736
4737 pub(crate) fn has_approved_start_failure(
4738 &self,
4739 identity: &crate::config_gate::runtime::ApprovalIdentity,
4740 ) -> bool {
4741 lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4742 state
4743 .approved_start_failure
4744 .as_ref()
4745 .is_some_and(|(failed, _)| failed == identity)
4746 })
4747 }
4748
4749 pub(crate) fn needs_approved_gate_restart(
4750 &self,
4751 approval: &crate::config_gate::runtime::ApprovalIdentity,
4752 ) -> bool {
4753 lock_snapshot(&self.inner.snapshot)
4754 .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4755 }
4756
4757 #[cfg(all(test, unix))]
4758 pub(crate) fn config_hold_origin_for_test(
4759 &self,
4760 ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4761 let state = lock_snapshot(&self.inner.snapshot).unwrap();
4762 (
4763 state.config_hold_daemon_initiated,
4764 state.config_hold_approval.clone(),
4765 )
4766 }
4767
4768 pub(crate) async fn config_policy_probe(
4769 &self,
4770 connection: crate::ConnectionId,
4771 generation: u64,
4772 attempt: u64,
4773 ) -> Result<(), SuperviseError> {
4774 self.inner
4775 .commands
4776 .send(SupervisorCommand::ConfigPolicyProbe {
4777 connection,
4778 generation,
4779 attempt,
4780 })
4781 .await
4782 .map_err(|_| SuperviseError::CommandClosed {
4783 module_id: self.inner.module_id.clone(),
4784 })
4785 }
4786 pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4787 let (reply, result) = oneshot::channel();
4788 self.inner
4789 .commands
4790 .send(SupervisorCommand::ConfigExitHold { reply })
4791 .await
4792 .map_err(|_| SuperviseError::CommandClosed {
4793 module_id: self.module_id().to_string(),
4794 })?;
4795 result.await.map_err(|_| SuperviseError::CommandClosed {
4796 module_id: self.module_id().to_string(),
4797 })?
4798 }
4799 pub(crate) async fn config_delivery_failed(
4800 &self,
4801 connection: crate::ConnectionId,
4802 hold: bool,
4803 ) -> Result<(), SuperviseError> {
4804 self.inner
4805 .commands
4806 .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4807 .await
4808 .map_err(|_| SuperviseError::CommandClosed {
4809 module_id: self.module_id().to_string(),
4810 })
4811 }
4812
4813 pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4814 self.inner
4815 .commands
4816 .send(SupervisorCommand::ConfigApplyRestart {
4817 generation,
4818 supersedes_swap: false,
4819 })
4820 .await
4821 .map_err(|_| SuperviseError::CommandClosed {
4822 module_id: self.inner.module_id.clone(),
4823 })
4824 }
4825 pub fn module_id(&self) -> &str {
4826 &self.inner.module_id
4827 }
4828
4829 #[cfg(test)]
4833 pub(crate) fn record_health_probe_failure_for_test(
4834 &self,
4835 detail: &str,
4836 ) -> Result<(), SuperviseError> {
4837 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4838 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4839 state.health.detail = Some(detail.to_string());
4840 })
4841 }
4842
4843 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4844 Ok(lock_snapshot(&self.inner.snapshot)?.state)
4845 }
4846
4847 pub(crate) fn registration_end_reason(
4848 &self,
4849 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4850 let snapshot = lock_snapshot(&self.inner.snapshot)?;
4851 Ok(match snapshot.state {
4852 ModuleState::Draining if snapshot.draining_to_replace => {
4853 Some(RegistrationEndReason::SupervisorRestart)
4854 }
4855 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4856 ModuleState::Restarting if snapshot.draining_to_replace => {
4857 Some(RegistrationEndReason::SupervisorRestart)
4858 }
4859 _ => None,
4860 })
4861 }
4862
4863 pub fn stderr_tail(
4870 &self,
4871 max_lines: Option<usize>,
4872 max_bytes: Option<usize>,
4873 ) -> StderrTailSnapshot {
4874 self.inner
4875 .stderr_ring
4876 .lock()
4877 .unwrap_or_else(|poisoned| poisoned.into_inner())
4878 .snapshot(max_lines, max_bytes)
4879 }
4880
4881 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4886 self.inner
4887 .terminal_ring
4888 .lock()
4889 .unwrap_or_else(|poisoned| poisoned.into_inner())
4890 .snapshot()
4891 }
4892
4893 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4898 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4899 }
4900
4901 pub(crate) async fn read_durable_terminal_history(
4906 &self,
4907 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4908 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4909 let module_id = self.inner.module_id.clone();
4910 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4911 .await
4912 }
4913
4914 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4915 self.status_with_snapshot_lock(&self.inner.snapshot, None)
4916 .map(|(status, _)| status)
4917 }
4918
4919 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4920 let state = lock_snapshot(&self.inner.snapshot)?;
4921 Ok(state
4922 .failed_retry
4923 .pending
4924 .filter(|_| state.state == ModuleState::Failed && state.enabled)
4925 .map(|retry| {
4926 unix_ms_now().saturating_add(
4927 u64::try_from(
4928 retry
4929 .deadline
4930 .saturating_duration_since(Instant::now())
4931 .as_millis(),
4932 )
4933 .unwrap_or(u64::MAX),
4934 )
4935 }))
4936 }
4937
4938 #[cfg(all(test, unix))]
4939 pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4940 lock_snapshot(&self.inner.snapshot)
4941 .unwrap()
4942 .failed_retry
4943 .pending
4944 .map(|retry| retry.deadline)
4945 }
4946
4947 pub(crate) fn record_deliberate_severance(
4948 &self,
4949 identity: ProcessIdentity,
4950 ) -> Result<bool, SuperviseError> {
4951 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4952 if snapshot.pid != Some(identity.pid)
4953 || snapshot.process_start_time != Some(identity.start_time)
4954 {
4955 return Ok(false);
4956 }
4957 snapshot.deliberate_severance = Some(identity);
4958 Ok(true)
4959 }
4960
4961 pub(crate) fn status_for_control(
4966 &self,
4967 caller: &'static str,
4968 ) -> Result<ModuleStatus, SuperviseError> {
4969 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
4970 .map(|(status, _)| status)
4971 }
4972
4973 fn status_with_snapshot_lock(
4974 &self,
4975 snapshot: &SharedSnapshot,
4976 caller: Option<&'static str>,
4977 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
4978 let restart_policy = self
4979 .inner
4980 .configuration
4981 .lock()
4982 .map_err(|_| SuperviseError::StatePoisoned {
4983 module_id: Some(self.inner.module_id.clone()),
4984 })?
4985 .restart_policy;
4986 let mut guard = match caller {
4987 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
4988 None => lock_snapshot(snapshot)?,
4989 };
4990 let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
4993 let snapshot = guard.clone();
4994 drop(guard);
4995 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
4996 SuperviseError::StatePoisoned {
4997 module_id: Some(self.inner.module_id.clone()),
4998 }
4999 })?;
5000 let registration_active = self
5001 .inner
5002 .registry
5003 .get_module(&self.inner.module_id)
5004 .map_err(SuperviseError::Registry)?
5005 .is_some();
5006 let protocol = snapshot
5007 .spawned_protocol
5008 .unwrap_or(self.declared_protocol()?);
5009 let running_process =
5010 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
5011 let live = match protocol {
5017 ModuleProtocol::Subc => running_process && registration_active,
5018 ModuleProtocol::None => running_process,
5019 };
5020 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
5023 let carry_configured = self
5024 .inner
5025 .configuration
5026 .lock()
5027 .map_err(|_| SuperviseError::StatePoisoned {
5028 module_id: Some(self.inner.module_id.clone()),
5029 })?
5030 .spec
5031 .carry_containment;
5032 let recovery_block = carries_containment(
5033 carry_configured,
5034 self.inner
5035 .retained
5036 .has_carried_container(&self.inner.module_id),
5037 )
5038 .then(|| self.inner.retained.recovery_block())
5039 .flatten();
5040 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5041 let pid = snapshot.reported_pid();
5042 let mut health = snapshot.health;
5043 if let Some(detail) = recovery_block
5044 .as_ref()
5045 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5046 {
5047 health.detail = Some(match health.detail {
5048 Some(previous) => format!("{previous}; {detail}"),
5049 None => detail.clone(),
5050 });
5051 }
5052
5053 Ok((
5054 ModuleStatus {
5055 admission_refused: snapshot.admission_refused,
5056 legacy_registration_spawn: snapshot.legacy_registration_spawn,
5057 registration_declaration: snapshot.registration_declaration.clone(),
5058 stop_reason: snapshot.stop_reason,
5059 waiting_for_previous_pid: retained
5060 .as_ref()
5061 .map(|(pid, _)| *pid)
5062 .filter(|pid| *pid > 0)
5063 .or(snapshot.waiting_for_previous_pid),
5064 waiting_for_previous_detail: recovery_block
5065 .or_else(|| retained.map(|(_, detail)| detail))
5066 .or(snapshot.waiting_for_previous_detail),
5067 module_id: self.inner.module_id.clone(),
5068 launch_nonce_source,
5069 state: snapshot.state,
5070 enabled: snapshot.enabled,
5071 process_alive: snapshot.process_alive,
5072 registration_active,
5073 protocol,
5074 live,
5075 restart_count,
5076 lifetime_restarts: snapshot.lifetime_restarts,
5077 spawn_generation: snapshot.spawn_generation,
5078 swap_outcome: snapshot.swap_outcome,
5079 max_restarts: restart_policy.max_restarts,
5080 restart_window: restart_policy.window,
5081 drain_timeout,
5082 restart_backoff: restart_policy.backoff,
5083 restart_max_backoff: restart_policy.max_backoff,
5084 pid,
5085 spawned_at_ms: snapshot.spawned_at_ms,
5086 spawned_from: snapshot.spawned_from,
5087 launch_spec_digest: snapshot.launch_spec_digest,
5088 process_start_time: snapshot.process_start_time,
5089 last_exit: snapshot.last_exit,
5090 health,
5091 },
5092 snapshot.spawned_file_identity.clone(),
5093 ))
5094 }
5095
5096 #[cfg(test)]
5097 pub(crate) fn hold_snapshot_for_test(
5098 &self,
5099 acquired: std::sync::mpsc::Sender<()>,
5100 hold: Duration,
5101 ) -> std::thread::JoinHandle<()> {
5102 let snapshot = Arc::clone(&self.inner.snapshot);
5103 std::thread::spawn(move || {
5104 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5105 acquired
5106 .send(())
5107 .expect("test receiver waits for snapshot lock");
5108 std::thread::sleep(hold);
5109 })
5110 }
5111
5112 pub(crate) async fn status_and_running_image_agreement(
5118 &self,
5119 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5120 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5121 let image = self
5122 .inner
5123 .provenance_probe
5124 .observe(
5125 status.pid,
5126 status.spawned_from.as_deref(),
5127 identity,
5128 status.process_start_time,
5129 )
5130 .await;
5131 Ok((status, image))
5132 }
5133
5134 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5135 let snapshot = match lock_snapshot(&self.inner.snapshot) {
5136 Ok(snapshot) => snapshot.clone(),
5137 Err(_) => {
5138 return subc_control::RunningImageAgreement::Unavailable {
5139 reason: subc_control::RunningImageUnavailableReason::NotRunning,
5140 };
5141 }
5142 };
5143 self.inner
5144 .provenance_probe
5145 .observe(
5146 snapshot.reported_pid(),
5147 snapshot.spawned_from.as_deref(),
5148 snapshot.spawned_file_identity.clone(),
5149 snapshot.process_start_time,
5150 )
5151 .await
5152 }
5153
5154 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5157 #[cfg(windows)]
5158 {
5159 let image = lock_snapshot(&self.inner.snapshot)
5160 .ok()
5161 .and_then(|snapshot| {
5162 snapshot
5163 .spawned_file_identity
5164 .as_ref()
5165 .and_then(|identity| identity.windows_image.clone())
5166 });
5167 match image {
5168 Some(image) => {
5169 crate::child_resources::read_windows_handle(image.process(), image.start_time())
5170 }
5171 None => subc_control::ChildResourceUsage::Unavailable {
5172 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5173 },
5174 }
5175 }
5176 #[cfg(not(windows))]
5177 {
5178 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5179 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5180 Err(_) => {
5181 return subc_control::ChildResourceUsage::Unavailable {
5182 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5183 }
5184 }
5185 };
5186 crate::child_resources::read(pid, start_time)
5187 }
5188 }
5189
5190 #[cfg(all(test, any(target_os = "linux", windows)))]
5191 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5192 self.inner
5193 .snapshot
5194 .try_lock()
5195 .map(|snapshot| snapshot.subtree_target.clone())
5196 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5197 }
5198
5199 #[cfg(test)]
5200 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5201 self.inner.snapshot.lock().unwrap().subtree_target = target;
5202 }
5203
5204 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5205 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5206 Ok(match snapshot.state {
5207 ModuleState::Restarting => true,
5208 ModuleState::Failed | ModuleState::Disabled => false,
5209 _ if snapshot.enabled && snapshot.respawn_pending => true,
5210 _ => daemon_will_restart(
5211 &mut snapshot,
5212 &self
5213 .inner
5214 .configuration
5215 .lock()
5216 .map_err(|_| SuperviseError::StatePoisoned {
5217 module_id: Some(self.inner.module_id.clone()),
5218 })?
5219 .restart_policy,
5220 Instant::now(),
5221 ),
5222 })
5223 }
5224
5225 #[cfg(test)]
5226 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5227 self.is_warming_with_snapshot_lock(None)
5228 }
5229
5230 pub(crate) fn is_warming_for_control(
5231 &self,
5232 caller: &'static str,
5233 ) -> Result<bool, SuperviseError> {
5234 self.is_warming_with_snapshot_lock(Some(caller))
5235 }
5236
5237 fn is_warming_with_snapshot_lock(
5238 &self,
5239 caller: Option<&'static str>,
5240 ) -> Result<bool, SuperviseError> {
5241 let snapshot = match caller {
5242 Some(caller) => {
5243 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5244 }
5245 None => lock_snapshot(&self.inner.snapshot)?,
5246 }
5247 .clone();
5248 Ok(matches!(
5249 snapshot.state,
5250 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5251 ))
5252 }
5253
5254 pub async fn drain(&self) -> Result<(), SuperviseError> {
5256 self.stop().await
5257 }
5258
5259 pub(crate) async fn retire_requested(
5260 &self,
5261 requester: Option<Principal>,
5262 ) -> Result<(), SuperviseError> {
5263 match self.state()? {
5264 ModuleState::Stopped => return Ok(()),
5265 ModuleState::Failed
5266 if lock_snapshot(&self.inner.snapshot)?
5267 .failed_retry
5268 .pending
5269 .is_none() =>
5270 {
5271 return Ok(())
5272 }
5273 ModuleState::Failed
5274 | ModuleState::Starting
5275 | ModuleState::Running
5276 | ModuleState::Unresponsive
5277 | ModuleState::Restarting
5278 | ModuleState::Draining
5279 | ModuleState::Disabled => {}
5280 }
5281
5282 let (reply_tx, reply_rx) = oneshot::channel();
5283 self.inner
5284 .commands
5285 .send(SupervisorCommand::Retire {
5286 requester,
5287 reply: reply_tx,
5288 })
5289 .await
5290 .map_err(|_| SuperviseError::CommandClosed {
5291 module_id: self.inner.module_id.clone(),
5292 })?;
5293 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5294 module_id: self.inner.module_id.clone(),
5295 })?
5296 }
5297
5298 pub async fn stop(&self) -> Result<(), SuperviseError> {
5299 self.stop_requested(None).await
5300 }
5301
5302 pub(crate) async fn stop_requested(
5303 &self,
5304 requester: Option<Principal>,
5305 ) -> Result<(), SuperviseError> {
5306 match self.state()? {
5307 ModuleState::Stopped => return Ok(()),
5308 ModuleState::Failed
5309 if lock_snapshot(&self.inner.snapshot)?
5310 .failed_retry
5311 .pending
5312 .is_none() =>
5313 {
5314 return Ok(())
5315 }
5316 ModuleState::Failed
5317 | ModuleState::Starting
5318 | ModuleState::Running
5319 | ModuleState::Unresponsive
5320 | ModuleState::Restarting
5321 | ModuleState::Draining
5322 | ModuleState::Disabled => {}
5323 }
5324
5325 let (reply_tx, reply_rx) = oneshot::channel();
5326 self.inner
5327 .commands
5328 .send(SupervisorCommand::Drain {
5329 requester,
5330 reply: reply_tx,
5331 })
5332 .await
5333 .map_err(|_| SuperviseError::CommandClosed {
5334 module_id: self.inner.module_id.clone(),
5335 })?;
5336 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5337 module_id: self.inner.module_id.clone(),
5338 })?
5339 }
5340
5341 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5342 self.restart_requested(drain_timeout_ms, None).await
5343 }
5344
5345 pub(crate) async fn restart_requested(
5346 &self,
5347 drain_timeout_ms: Option<u64>,
5348 requester: Option<Principal>,
5349 ) -> Result<(), SuperviseError> {
5350 let queued_at = Instant::now();
5351 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5352 let (reply_tx, reply_rx) = oneshot::channel();
5353 self.inner
5354 .commands
5355 .send(SupervisorCommand::Restart {
5356 requester,
5357 drain_timeout_ms,
5358 received_at_generation,
5359 queued_at,
5360 reply: reply_tx,
5361 })
5362 .await
5363 .map_err(|_| SuperviseError::CommandClosed {
5364 module_id: self.inner.module_id.clone(),
5365 })?;
5366 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5367 module_id: self.inner.module_id.clone(),
5368 })?
5369 }
5370
5371 pub async fn swap(
5377 &self,
5378 ready_timeout: Option<Duration>,
5379 ) -> Result<SwapOutcome, SuperviseError> {
5380 self.swap_requested(ready_timeout, None).await
5381 }
5382
5383 pub(crate) async fn swap_requested(
5384 &self,
5385 ready_timeout: Option<Duration>,
5386 requester: Option<Principal>,
5387 ) -> Result<SwapOutcome, SuperviseError> {
5388 let (reply_tx, reply_rx) = oneshot::channel();
5389 self.inner
5390 .commands
5391 .send(SupervisorCommand::Swap {
5392 requester,
5393 ready_timeout,
5394 reply: reply_tx,
5395 })
5396 .await
5397 .map_err(|_| SuperviseError::CommandClosed {
5398 module_id: self.inner.module_id.clone(),
5399 })?;
5400 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5401 module_id: self.inner.module_id.clone(),
5402 })?
5403 }
5404
5405 pub async fn reload(&self) -> Result<(), SuperviseError> {
5406 self.reload_requested(None).await
5407 }
5408
5409 pub(crate) async fn reload_requested(
5410 &self,
5411 requester: Option<Principal>,
5412 ) -> Result<(), SuperviseError> {
5413 let (reply_tx, reply_rx) = oneshot::channel();
5414 self.inner
5415 .commands
5416 .send(SupervisorCommand::Reload {
5417 requester,
5418 reply: reply_tx,
5419 })
5420 .await
5421 .map_err(|_| SuperviseError::CommandClosed {
5422 module_id: self.inner.module_id.clone(),
5423 })?;
5424 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5425 module_id: self.inner.module_id.clone(),
5426 })?
5427 }
5428
5429 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5430 self.set_enabled_requested(enabled, None).await
5431 }
5432
5433 pub(crate) async fn set_enabled_requested(
5434 &self,
5435 enabled: bool,
5436 requester: Option<Principal>,
5437 ) -> Result<bool, SuperviseError> {
5438 self.set_enabled_with_budget_policy(enabled, requester, true)
5439 .await
5440 }
5441
5442 pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5443 self.set_enabled_with_budget_policy(true, None, false).await
5444 }
5445
5446 #[cfg(all(test, unix))]
5447 pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5448 let policy = self.inner.configuration.lock().unwrap().restart_policy;
5449 let mut state = self.inner.snapshot.lock().unwrap();
5450 for _ in 0..count {
5451 state.record_crash_restart(&policy, Instant::now());
5452 }
5453 }
5454
5455 async fn set_enabled_with_budget_policy(
5456 &self,
5457 enabled: bool,
5458 requester: Option<Principal>,
5459 reset_budget: bool,
5460 ) -> Result<bool, SuperviseError> {
5461 let (reply_tx, reply_rx) = oneshot::channel();
5462 self.inner
5463 .commands
5464 .send(SupervisorCommand::SetEnabled {
5465 requester,
5466 enabled,
5467 reset_budget,
5468 reply: reply_tx,
5469 })
5470 .await
5471 .map_err(|_| SuperviseError::CommandClosed {
5472 module_id: self.inner.module_id.clone(),
5473 })?;
5474 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5475 module_id: self.inner.module_id.clone(),
5476 })?
5477 }
5478
5479 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5483 let configured = self
5484 .inner
5485 .configuration
5486 .lock()
5487 .map_err(|_| SuperviseError::StatePoisoned {
5488 module_id: Some(self.inner.module_id.clone()),
5489 })?
5490 .spec
5491 .protocol;
5492 let state = lock_snapshot(&self.inner.snapshot)?;
5493 Ok(state.spawned_protocol.unwrap_or(configured))
5494 }
5495
5496 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5497 let configuration =
5498 self.inner
5499 .configuration
5500 .lock()
5501 .map_err(|_| SuperviseError::StatePoisoned {
5502 module_id: Some(self.inner.module_id.clone()),
5503 })?;
5504 Ok((configuration.spec.clone(), configuration.health.clone()))
5505 }
5506
5507 pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5508 self.inner
5509 .configuration
5510 .lock()
5511 .map(|configuration| configuration.register_timeout)
5512 .map_err(|_| SuperviseError::StatePoisoned {
5513 module_id: Some(self.inner.module_id.clone()),
5514 })
5515 }
5516
5517 pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5518 self.inner
5519 .configuration
5520 .lock()
5521 .map(|configuration| configuration.enabled)
5522 .map_err(|_| SuperviseError::StatePoisoned {
5523 module_id: Some(self.inner.module_id.clone()),
5524 })
5525 }
5526
5527 #[cfg(any(test, feature = "test-support"))]
5531 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5532 let (_, health) = self.configuration()?;
5533 let drain_timeout_ms = u64::try_from(
5534 self.inner
5535 .effective_drain_timeout
5536 .lock()
5537 .unwrap_or_else(|poisoned| poisoned.into_inner())
5538 .as_millis(),
5539 )
5540 .ok();
5541 self.update_configuration(spec, health, drain_timeout_ms)
5542 .await
5543 }
5544
5545 pub(crate) async fn reconcile_approved_configuration(
5546 &self,
5547 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5548 ) -> Result<(), SuperviseError> {
5549 let entry = approved
5550 .module_entry(&self.inner.module_id)
5551 .map_err(|error| SuperviseError::InvalidSpec {
5552 reason: error.to_string(),
5553 })?;
5554 validate_spec(&entry.module_spec())?;
5555 let (reply, received) = oneshot::channel();
5556 self.inner
5557 .commands
5558 .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5559 .await
5560 .map_err(|_| SuperviseError::CommandClosed {
5561 module_id: self.inner.module_id.clone(),
5562 })?;
5563 received.await.map_err(|_| SuperviseError::CommandClosed {
5564 module_id: self.inner.module_id.clone(),
5565 })??;
5566 let mut configuration =
5567 self.inner
5568 .configuration
5569 .lock()
5570 .map_err(|_| SuperviseError::StatePoisoned {
5571 module_id: Some(self.inner.module_id.clone()),
5572 })?;
5573 configuration.spec = entry.module_spec();
5574 configuration.enabled = entry.enabled;
5575 configuration.health = entry.health;
5576 configuration.register_timeout = entry.register_timeout;
5577 configuration.restart_policy = entry.restart;
5578 Ok(())
5579 }
5580
5581 pub(crate) async fn update_configuration(
5582 &self,
5583 spec: ModuleSpec,
5584 health: HealthConfig,
5585 drain_timeout_ms: Option<u64>,
5586 ) -> Result<(), SuperviseError> {
5587 if spec.module_id != self.inner.module_id {
5588 return Err(SuperviseError::InvalidSpec {
5589 reason: "a supervised module's module_id cannot be changed".to_string(),
5590 });
5591 }
5592 validate_spec(&spec)?;
5593 let (reply_tx, reply_rx) = oneshot::channel();
5594 self.inner
5595 .commands
5596 .send(SupervisorCommand::UpdateConfiguration {
5597 spec: spec.clone(),
5598 health: health.clone(),
5599 drain_timeout_ms,
5600 reply: reply_tx,
5601 })
5602 .await
5603 .map_err(|_| SuperviseError::CommandClosed {
5604 module_id: self.inner.module_id.clone(),
5605 })?;
5606 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5607 module_id: self.inner.module_id.clone(),
5608 })??;
5609 let mut configuration =
5610 self.inner
5611 .configuration
5612 .lock()
5613 .map_err(|_| SuperviseError::StatePoisoned {
5614 module_id: Some(self.inner.module_id.clone()),
5615 })?;
5616 configuration.spec = spec;
5617 configuration.health = health;
5618 Ok(())
5619 }
5620}
5621
5622impl Drop for SupervisedModuleInner {
5623 fn drop(&mut self) {
5624 let Ok(mut monitor) = self.monitor.lock() else {
5625 return;
5626 };
5627 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5628 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5629 state.state = ModuleState::Stopped;
5630 clear_current_process_facts(state);
5631 });
5632 monitor.abort();
5633 }
5634 let _ = monitor.take();
5635 }
5636}
5637
5638#[derive(Debug)]
5639enum SupervisorCommand {
5640 ApprovedConfiguration {
5641 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5642 reply: oneshot::Sender<Result<(), SuperviseError>>,
5643 },
5644 ConfigPolicyProbe {
5645 connection: crate::ConnectionId,
5646 generation: u64,
5647 attempt: u64,
5648 },
5649 ConfigApplyRestart {
5650 generation: u64,
5651 supersedes_swap: bool,
5652 },
5653 ConfigExitHold {
5654 reply: oneshot::Sender<Result<(), SuperviseError>>,
5655 },
5656 ConfigDeliveryFailed {
5657 connection: crate::ConnectionId,
5658 hold: bool,
5659 },
5660 Drain {
5661 requester: Option<Principal>,
5662 reply: oneshot::Sender<Result<(), SuperviseError>>,
5663 },
5664 Retire {
5665 requester: Option<Principal>,
5666 reply: oneshot::Sender<Result<(), SuperviseError>>,
5667 },
5668 Restart {
5669 requester: Option<Principal>,
5670 drain_timeout_ms: Option<u64>,
5675 received_at_generation: u64,
5679 queued_at: Instant,
5682 reply: oneshot::Sender<Result<(), SuperviseError>>,
5683 },
5684 Reload {
5685 requester: Option<Principal>,
5686 reply: oneshot::Sender<Result<(), SuperviseError>>,
5687 },
5688 SetEnabled {
5689 reset_budget: bool,
5690 requester: Option<Principal>,
5691 enabled: bool,
5692 reply: oneshot::Sender<Result<bool, SuperviseError>>,
5693 },
5694 UpdateConfiguration {
5695 spec: ModuleSpec,
5696 health: HealthConfig,
5697 drain_timeout_ms: Option<u64>,
5700 reply: oneshot::Sender<Result<(), SuperviseError>>,
5701 },
5702 Swap {
5703 requester: Option<Principal>,
5704 ready_timeout: Option<Duration>,
5707 reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5709 },
5710}
5711
5712#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5716pub enum SwapOutcome {
5717 Cutover,
5718 SupersededByConfigRestart,
5719}
5720
5721impl SwapOutcome {
5722 pub fn as_str(self) -> &'static str {
5723 match self {
5724 Self::Cutover => "cutover",
5725 Self::SupersededByConfigRestart => "superseded_by_config_restart",
5726 }
5727 }
5728}
5729
5730#[derive(Debug)]
5731pub enum SuperviseError {
5732 ConfigHeld {
5733 module_id: String,
5734 code: String,
5735 detail: String,
5736 },
5737 ConfigDeliveryDeferred {
5738 module_id: String,
5739 },
5740 InvalidSpec {
5741 reason: String,
5742 },
5743 Spawn {
5744 program: PathBuf,
5745 source: io::Error,
5746 cgroup_path: Option<PathBuf>,
5747 },
5748 Cgroup {
5749 module_id: String,
5750 source: io::Error,
5751 },
5752 LaunchNonce {
5755 reason: String,
5756 },
5757 Wait {
5758 module_id: String,
5759 source: io::Error,
5760 },
5761 Kill {
5762 module_id: String,
5763 source: io::Error,
5764 },
5765 Forwarding(ForwardingError),
5766 Registry(RegistryError),
5767 ReloadUnavailable {
5768 module_id: String,
5769 reason: String,
5770 },
5771 Disabled {
5776 module_id: String,
5777 },
5778 ReloadFailed {
5779 module_id: String,
5780 reason: String,
5781 },
5782 RegistrationStillActive {
5783 module_id: String,
5784 waited: Duration,
5785 },
5786 StatePoisoned {
5787 module_id: Option<String>,
5788 },
5789 CommandClosed {
5790 module_id: String,
5791 },
5792 SwapInProgress {
5796 module_id: String,
5797 },
5798 SwapRefused {
5800 module_id: String,
5801 reason: SwapRefusal,
5802 },
5803 SwapFailed {
5807 module_id: String,
5808 arm: SwapFailureArm,
5809 detail: String,
5810 candidate_exit: Option<ExitReport>,
5813 },
5814}
5815
5816#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5818pub enum SwapRefusal {
5819 OverlapExclusive,
5821 NotRegistered,
5824 ProtocolNone,
5827 NotConfigured,
5830 AlreadySwapping,
5832}
5833
5834impl SwapRefusal {
5835 pub fn as_str(self) -> &'static str {
5836 match self {
5837 Self::OverlapExclusive => "overlap_exclusive",
5838 Self::NotRegistered => "not_registered",
5839 Self::ProtocolNone => "protocol_none",
5840 Self::NotConfigured => "not_configured",
5841 Self::AlreadySwapping => "already_swapping",
5842 }
5843 }
5844}
5845
5846#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5849pub enum SwapFailureArm {
5850 SpawnFailed,
5852 NeverRegistered,
5854 NeverReady,
5856 CandidateExited,
5858 CandidateUnhealthy,
5860 Interrupted,
5864 CutoverLost,
5869}
5870
5871impl SwapFailureArm {
5872 pub fn as_str(self) -> &'static str {
5873 match self {
5874 Self::SpawnFailed => "spawn_failed",
5875 Self::NeverRegistered => "never_registered",
5876 Self::NeverReady => "never_ready",
5877 Self::CandidateExited => "candidate_exited",
5878 Self::CandidateUnhealthy => "candidate_unhealthy",
5879 Self::Interrupted => "interrupted",
5880 Self::CutoverLost => "cutover_lost",
5881 }
5882 }
5883}
5884
5885impl SuperviseError {
5886 pub(crate) fn is_config_gate_error(&self) -> bool {
5887 match self {
5888 Self::ConfigHeld { .. } => true,
5889 Self::Spawn { source, .. } => source
5890 .get_ref()
5891 .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5892 _ => false,
5893 }
5894 }
5895}
5896
5897impl fmt::Display for SuperviseError {
5898 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5899 match self {
5900 Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5901 Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5902 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5903 Self::Spawn {
5904 program,
5905 source,
5906 cgroup_path: Some(cgroup_path),
5907 } => write!(
5908 f,
5909 "failed to place module in cgroup '{}' while spawning '{}': {source}",
5910 cgroup_path.display(),
5911 program.display()
5912 ),
5913 Self::Spawn {
5914 program,
5915 source,
5916 cgroup_path: None,
5917 } => write!(
5918 f,
5919 "failed to spawn module '{}': {source}",
5920 program.display()
5921 ),
5922 Self::Cgroup { module_id, source } => {
5923 write!(
5924 f,
5925 "failed to prepare cgroup for module '{module_id}': {source}"
5926 )
5927 }
5928 Self::LaunchNonce { reason } => {
5929 write!(
5930 f,
5931 "failed to generate reserved-module launch nonce: {reason}"
5932 )
5933 }
5934 Self::Wait { module_id, source } => {
5935 write!(f, "failed to wait for module '{module_id}': {source}")
5936 }
5937 Self::Kill { module_id, source } => {
5938 write!(f, "failed to kill module '{module_id}': {source}")
5939 }
5940 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5941 Self::Registry(err) => write!(f, "registry error: {err}"),
5942 Self::ReloadUnavailable { module_id, reason } => {
5943 write!(f, "reload unavailable for module '{module_id}': {reason}")
5944 }
5945 Self::Disabled { module_id } => {
5946 write!(
5947 f,
5948 "module '{module_id}' is disabled; enable it before restart or reload"
5949 )
5950 }
5951 Self::ReloadFailed { module_id, reason } => {
5952 write!(f, "reload failed for module '{module_id}': {reason}")
5953 }
5954 Self::RegistrationStillActive { module_id, waited } => write!(
5955 f,
5956 "module '{module_id}' registration remained active after waiting {waited:?}"
5957 ),
5958 Self::StatePoisoned { module_id } => match module_id {
5959 Some(module_id) => {
5960 write!(f, "supervisor state for module '{module_id}' was poisoned")
5961 }
5962 None => write!(f, "supervisor state was poisoned"),
5963 },
5964 Self::CommandClosed { module_id } => {
5965 write!(
5966 f,
5967 "supervisor command channel for module '{module_id}' is closed"
5968 )
5969 }
5970 Self::SwapInProgress { module_id } => write!(
5971 f,
5972 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
5973 ),
5974 Self::SwapRefused { module_id, reason } => match reason {
5975 SwapRefusal::OverlapExclusive => write!(
5976 f,
5977 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
5978 ),
5979 SwapRefusal::NotRegistered => write!(
5980 f,
5981 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
5982 ),
5983 SwapRefusal::ProtocolNone => write!(
5984 f,
5985 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
5986 ),
5987 SwapRefusal::NotConfigured => write!(
5988 f,
5989 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
5990 ),
5991 SwapRefusal::AlreadySwapping => {
5992 write!(f, "module '{module_id}' is already being swapped")
5993 }
5994 },
5995 Self::SwapFailed {
5996 module_id,
5997 arm,
5998 detail,
5999 ..
6000 } => write!(
6001 f,
6002 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
6003 arm.as_str()
6004 ),
6005 }
6006 }
6007}
6008
6009impl Error for SuperviseError {
6010 fn source(&self) -> Option<&(dyn Error + 'static)> {
6011 match self {
6012 Self::Spawn { source, .. }
6013 | Self::Cgroup { source, .. }
6014 | Self::Wait { source, .. }
6015 | Self::Kill { source, .. } => Some(source),
6016 Self::Forwarding(err) => Some(err),
6017 Self::Registry(err) => Some(err),
6018 Self::LaunchNonce { .. }
6019 | Self::ConfigDeliveryDeferred { .. }
6020 | Self::ConfigHeld { .. }
6021 | Self::InvalidSpec { .. }
6022 | Self::ReloadUnavailable { .. }
6023 | Self::Disabled { .. }
6024 | Self::ReloadFailed { .. }
6025 | Self::RegistrationStillActive { .. }
6026 | Self::StatePoisoned { .. }
6027 | Self::CommandClosed { .. }
6028 | Self::SwapInProgress { .. }
6029 | Self::SwapRefused { .. }
6030 | Self::SwapFailed { .. } => None,
6031 }
6032 }
6033}
6034
6035pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
6036 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
6037 return Err(SuperviseError::InvalidSpec {
6038 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6039 });
6040 }
6041 if spec.module_id.trim().is_empty() {
6042 return Err(SuperviseError::InvalidSpec {
6043 reason: "module_id must not be empty".to_string(),
6044 });
6045 }
6046
6047 Ok(())
6048}
6049
6050#[derive(Debug, Default)]
6051struct HealthProbeRuntime {
6052 spawn_generation: Option<u64>,
6053 configured_health: Option<HealthConfig>,
6054 registered_connection: Option<crate::ConnectionId>,
6055 advertised: bool,
6056 next_probe_at: Option<Instant>,
6057 probe_index: u64,
6058}
6059
6060fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6061 lock_snapshot(snapshot)
6062 .ok()
6063 .and_then(|state| state.spawned_protocol)
6064 .unwrap_or(spec.protocol)
6065}
6066
6067impl HealthProbeRuntime {
6068 fn refresh_registration(
6069 &mut self,
6070 spec: &ModuleSpec,
6071 runtime: &SupervisorRuntimeConfig,
6072 registry: &Registry,
6073 snapshot: &SharedSnapshot,
6074 ) {
6075 let generation = lock_snapshot(snapshot)
6076 .ok()
6077 .map(|state| state.spawn_generation);
6078 if self.spawn_generation != generation {
6079 *self = Self {
6083 spawn_generation: generation,
6084 ..Self::default()
6085 };
6086 }
6087 if self.configured_health.as_ref() != Some(&runtime.health) {
6088 self.configured_health = Some(runtime.health.clone());
6089 self.next_probe_at = None;
6090 self.registered_connection = None;
6091 self.probe_index = 0;
6092 }
6093 if running_protocol(spec, snapshot) == ModuleProtocol::None {
6097 self.registered_connection = None;
6098 self.advertised = runtime.health.http.is_some();
6099 if !self.advertised {
6100 self.next_probe_at = None;
6101 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6102 let unknown = ModuleHealthStatus::default();
6103 if state.health != unknown {
6104 state.health = unknown;
6105 state.last_probe_evidence = None;
6106 }
6107 });
6108 } else if self.next_probe_at.is_none() {
6109 self.next_probe_at = Some(
6110 Instant::now()
6111 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6112 );
6113 }
6114 return;
6115 }
6116
6117 let registration = match registry.get_module(&spec.module_id) {
6118 Ok(registration) => registration,
6119 Err(err) => {
6120 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6121 self.advertised = false;
6122 self.next_probe_at = None;
6123 return;
6124 }
6125 };
6126
6127 let Some(registration) = registration else {
6128 self.registered_connection = None;
6129 self.advertised = false;
6130 self.next_probe_at = None;
6131 return;
6132 };
6133
6134 let advertised = registration
6135 .control_ops
6136 .iter()
6137 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6138 if !advertised {
6139 self.registered_connection = Some(registration.connection_id);
6140 self.advertised = false;
6141 self.next_probe_at = None;
6142 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6143 if state.health.status != SupervisorHealthStatus::Unknown
6144 || state.health.consecutive_failures != 0
6145 || state.health.last_probe_ms.is_some()
6146 || state.health.detail.is_some()
6147 || state.health.metrics.is_some()
6148 {
6149 state.health.status = SupervisorHealthStatus::Unknown;
6150 state.last_probe_evidence = None;
6151 state.health.consecutive_failures = 0;
6152 state.health.last_probe_ms = None;
6153 state.health.detail = None;
6154 state.health.metrics = None;
6155 }
6156 });
6157 return;
6158 }
6159
6160 let reregistered = self.registered_connection != Some(registration.connection_id);
6161 self.registered_connection = Some(registration.connection_id);
6162 self.advertised = true;
6163 if reregistered || self.next_probe_at.is_none() {
6164 self.probe_index = 0;
6165 self.next_probe_at = Some(
6166 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6167 );
6168 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6169 state.health.status = SupervisorHealthStatus::Unknown;
6170 state.last_probe_evidence = None;
6171 state.health.consecutive_failures = 0;
6172 state.health.detail = None;
6173 state.health.metrics = None;
6174 });
6175 }
6176 }
6177
6178 fn wake_at(&self) -> Option<Instant> {
6179 if !self.advertised {
6180 return None;
6181 }
6182 self.next_probe_at
6183 }
6184
6185 fn due(&self) -> bool {
6186 self.advertised
6187 && self
6188 .next_probe_at
6189 .is_some_and(|next| Instant::now() >= next)
6190 }
6191
6192 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6193 self.probe_index = self.probe_index.wrapping_add(1);
6194 self.next_probe_at = Some(
6195 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6196 );
6197 }
6198}
6199
6200#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6204enum HealthProbeEvidence {
6205 LaneDead,
6207 NoAnswer,
6209 BadAnswer,
6211 Misconfigured,
6213}
6214
6215#[derive(Debug)]
6216struct HealthProbeError {
6217 evidence: HealthProbeEvidence,
6218 message: String,
6219}
6220
6221impl HealthProbeError {
6222 fn lane_dead(message: impl Into<String>) -> Self {
6223 Self::with(HealthProbeEvidence::LaneDead, message)
6224 }
6225
6226 fn no_answer(message: impl Into<String>) -> Self {
6227 Self::with(HealthProbeEvidence::NoAnswer, message)
6228 }
6229
6230 fn bad_answer(message: impl Into<String>) -> Self {
6231 Self::with(HealthProbeEvidence::BadAnswer, message)
6232 }
6233
6234 fn misconfigured(message: impl Into<String>) -> Self {
6235 Self::with(HealthProbeEvidence::Misconfigured, message)
6236 }
6237
6238 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6239 Self {
6240 evidence,
6241 message: message.into(),
6242 }
6243 }
6244
6245 #[allow(dead_code)]
6254 fn is_proof_of_death(&self) -> bool {
6255 matches!(self.evidence, HealthProbeEvidence::LaneDead)
6256 }
6257
6258 fn label(&self) -> &'static str {
6266 match self.evidence {
6267 HealthProbeEvidence::LaneDead => "lane-dead",
6268 HealthProbeEvidence::NoAnswer => "no-answer",
6269 HealthProbeEvidence::BadAnswer => "bad-answer",
6270 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6271 }
6272 }
6273}
6274
6275impl fmt::Display for HealthProbeError {
6276 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6277 f.write_str(&self.message)
6278 }
6279}
6280
6281async fn run_health_probe_cycle(
6282 spec: &ModuleSpec,
6283 runtime: &SupervisorRuntimeConfig,
6284 registry: &Registry,
6285 process_liveness: &SupervisorProcessLiveness,
6286 snapshot: &SharedSnapshot,
6287 child: &mut Option<SupervisedChild>,
6288) {
6289 let now_ms = unix_ms_now();
6290 let delivery_probe = match &runtime.config_delivery {
6291 Some(delivery) => {
6292 delivery
6293 .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6294 .await
6295 }
6296 None => None,
6297 };
6298 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6299 .then_some(runtime.health.http.as_deref())
6300 .flatten();
6301 let result = match http {
6302 Some(url) => probe_http_health(url, runtime.health.deadline)
6303 .await
6304 .map(|report| (report, None)),
6305 None => probe_module_health_from(&spec.module_id, runtime, None)
6306 .await
6307 .map(|(report, probed)| (report, Some(probed))),
6308 };
6309 match result {
6310 Ok((report, probed)) => {
6311 handle_health_report(
6312 spec,
6313 runtime,
6314 registry,
6315 process_liveness,
6316 snapshot,
6317 child,
6318 report,
6319 now_ms,
6320 delivery_probe,
6321 probed,
6322 )
6323 .await;
6324 }
6325 Err(err) => {
6326 if http.is_some() {
6327 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6328 state.health.status = SupervisorHealthStatus::Failing;
6329 });
6330 }
6331 handle_health_probe_failure(
6332 spec,
6333 runtime,
6334 registry,
6335 process_liveness,
6336 snapshot,
6337 child,
6338 err,
6339 now_ms,
6340 delivery_probe,
6341 )
6342 .await;
6343 }
6344 }
6345}
6346
6347pub(crate) struct HttpProbeTarget<'a> {
6348 address: std::net::SocketAddr,
6349 localhost: bool,
6350 authority: &'a str,
6351 path: String,
6352}
6353
6354pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6357 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6358 return Err("must not contain whitespace, controls, or a fragment".into());
6359 }
6360 let rest = url
6361 .strip_prefix("http://")
6362 .ok_or("must use plain http://")?;
6363 let split = rest.find(['/', '?']).unwrap_or(rest.len());
6364 let (authority, suffix) = rest.split_at(split);
6365 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6366 ("::1", rest)
6367 } else {
6368 let split = authority.find(':').unwrap_or(authority.len());
6369 authority.split_at(split)
6370 };
6371 let ip: std::net::IpAddr = match host {
6372 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6373 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6374 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6375 };
6376 let port = if port.is_empty() {
6377 80
6378 } else {
6379 port.strip_prefix(':')
6380 .and_then(|p| p.parse::<u16>().ok())
6381 .filter(|p| *p > 0)
6382 .ok_or("must have a valid nonzero TCP port")?
6383 };
6384 let path = if suffix.is_empty() {
6385 "/".into()
6386 } else if suffix.starts_with('?') {
6387 format!("/{suffix}")
6388 } else {
6389 suffix.into()
6390 };
6391 Ok(HttpProbeTarget {
6392 address: std::net::SocketAddr::new(ip, port),
6393 localhost: host == "localhost",
6394 authority,
6395 path,
6396 })
6397}
6398
6399async fn probe_http_health(
6400 url: &str,
6401 deadline: Duration,
6402) -> Result<HealthReport, HealthProbeError> {
6403 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6404 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6405 let mut response_status = String::new();
6408 let mut body = Vec::new();
6409 let probe = async {
6410 let connection = match tokio::net::TcpStream::connect(target.address).await {
6413 Err(_) if target.localhost => {
6414 tokio::net::TcpStream::connect((
6415 std::net::Ipv6Addr::LOCALHOST,
6416 target.address.port(),
6417 ))
6418 .await
6419 }
6420 result => result,
6421 };
6422 let mut stream = connection.map_err(|error| {
6423 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6424 })?;
6425 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6426 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6427 let mut reader = BufReader::new(stream);
6428 let mut budget = 16 * 1024;
6429 let status = http_line(&mut reader, &mut budget).await?;
6430 let mut words = status.split_ascii_whitespace();
6431 let version = words.next();
6432 let code = words
6433 .next()
6434 .filter(|word| word.len() == 3)
6435 .and_then(|word| word.parse::<u16>().ok());
6436 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6437 || !code.is_some_and(|code| (100..600).contains(&code))
6438 {
6439 return Err(HealthProbeError::bad_answer(format!(
6440 "invalid HTTP status: {status}"
6441 )));
6442 }
6443 let code = code.expect("validated status code");
6444 response_status = status.clone();
6445 let mut length = None;
6446 let mut chunked = false;
6447 loop {
6448 let line = http_line(&mut reader, &mut budget).await?;
6449 if line.is_empty() {
6450 break;
6451 }
6452 if let Some((name, value)) = line.split_once(':') {
6453 if name.eq_ignore_ascii_case("content-length") {
6454 length = Some(value.trim().parse::<u64>().map_err(|_| {
6455 HealthProbeError::bad_answer("invalid HTTP Content-Length")
6456 })?);
6457 } else if name.eq_ignore_ascii_case("transfer-encoding") {
6458 chunked = value.trim().eq_ignore_ascii_case("chunked");
6459 }
6460 }
6461 }
6462 if chunked {
6463 while body.len() < 200 {
6464 let line = http_line(&mut reader, &mut budget).await?;
6465 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6466 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6467 if size == 0 {
6468 break;
6469 }
6470 let count = size.min((200 - body.len()) as u64) as usize;
6471 let start = body.len();
6472 (&mut reader)
6473 .take(count as u64)
6474 .read_to_end(&mut body)
6475 .await
6476 .map_err(|error| {
6477 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6478 })?;
6479 if body.len() - start != count {
6480 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6481 }
6482 if size > count as u64 || body.len() == 200 {
6483 break;
6484 }
6485 if !http_line(&mut reader, &mut budget).await?.is_empty() {
6486 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6487 }
6488 }
6489 } else {
6490 reader
6491 .take(length.unwrap_or(200).min(200))
6492 .read_to_end(&mut body)
6493 .await
6494 .map_err(|error| {
6495 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6496 })?;
6497 }
6498 if (200..300).contains(&code) {
6499 Ok(HealthReport::ok())
6500 } else {
6501 Err(HealthProbeError::bad_answer(
6502 "HTTP health endpoint returned non-2xx",
6503 ))
6504 }
6505 };
6506 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6507 Err(HealthProbeError::no_answer(format!(
6508 "HTTP probe timed out after {deadline:?}"
6509 )))
6510 });
6511 if let Err(error) = &mut result {
6512 if !response_status.is_empty() {
6513 error.message = format!(
6514 "{}; {response_status}: {}",
6515 error.message,
6516 String::from_utf8_lossy(&body)
6517 );
6518 }
6519 }
6520 result
6521}
6522
6523async fn http_line(
6524 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6525 remaining: &mut usize,
6526) -> Result<String, HealthProbeError> {
6527 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6528 let mut line = Vec::new();
6529 (&mut *reader)
6530 .take(*remaining as u64)
6531 .read_until(b'\n', &mut line)
6532 .await
6533 .map_err(|error| {
6534 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6535 })?;
6536 *remaining -= line.len();
6537 if !line.ends_with(b"\r\n") {
6538 return Err(HealthProbeError::bad_answer(
6539 "HTTP headers are incomplete or exceed 16 KiB",
6540 ));
6541 }
6542 line.truncate(line.len() - 2);
6543 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6544}
6545
6546async fn probe_module_health(
6547 module_id: &str,
6548 runtime: &SupervisorRuntimeConfig,
6549 drain_deadline: Option<Instant>,
6550) -> Result<HealthReport, HealthProbeError> {
6551 probe_module_health_from(module_id, runtime, drain_deadline)
6552 .await
6553 .map(|(report, _)| report)
6554}
6555
6556async fn probe_module_health_from(
6559 module_id: &str,
6560 runtime: &SupervisorRuntimeConfig,
6561 drain_deadline: Option<Instant>,
6562) -> Result<(HealthReport, ConnectionId), HealthProbeError> {
6563 let Some(forwarding) = runtime.forwarding.as_ref() else {
6564 return Err(HealthProbeError::misconfigured(
6565 "supervisor was not configured with a forwarding table",
6566 ));
6567 };
6568 let probe_started_at = Instant::now();
6569 let mut deadline = probe_started_at + runtime.health.deadline;
6570 if let Some(drain_deadline) = drain_deadline {
6571 deadline = deadline.min(drain_deadline);
6572 }
6573 let pending = if drain_deadline.is_some() {
6574 forwarding.begin_drain_health_probe_rpc_for(
6575 module_id,
6576 MODULE_CONTROL_OP_HEALTH_CHECK,
6577 probe_started_at,
6578 deadline,
6579 )
6580 } else {
6581 forwarding.begin_health_probe_rpc_for(
6582 module_id,
6583 MODULE_CONTROL_OP_HEALTH_CHECK,
6584 probe_started_at,
6585 deadline,
6586 )
6587 }
6588 .map_err(|err| {
6589 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6592 })?;
6593 let probed = pending.endpoint.connection_id;
6594 await_health_probe(forwarding, pending, deadline, runtime.health.deadline)
6595 .await
6596 .map(|report| (report, probed))
6597}
6598
6599async fn probe_endpoint_health(
6606 endpoint: crate::ModuleEndpointId,
6607 runtime: &SupervisorRuntimeConfig,
6608 deadline_cap: Option<Instant>,
6609) -> Result<HealthReport, HealthProbeError> {
6610 let Some(forwarding) = runtime.forwarding.as_ref() else {
6611 return Err(HealthProbeError::misconfigured(
6612 "supervisor was not configured with a forwarding table",
6613 ));
6614 };
6615 let probe_started_at = Instant::now();
6616 let mut deadline = probe_started_at + runtime.health.deadline;
6617 if let Some(cap) = deadline_cap {
6618 deadline = deadline.min(cap);
6619 }
6620 let pending = forwarding
6621 .begin_endpoint_health_probe_rpc_for(
6622 endpoint,
6623 MODULE_CONTROL_OP_HEALTH_CHECK,
6624 probe_started_at,
6625 deadline,
6626 )
6627 .map_err(|err| {
6628 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6629 })?;
6630 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6631}
6632
6633async fn await_health_probe(
6635 forwarding: &ForwardingTable,
6636 pending: PendingModuleControlRpc,
6637 deadline: Instant,
6638 probe_budget: Duration,
6639) -> Result<HealthReport, HealthProbeError> {
6640 let PendingModuleControlRpc {
6641 endpoint,
6642 module_sink,
6643 negotiated_ver,
6644 corr,
6645 receiver,
6646 } = pending;
6647 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6648 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6649 })?;
6650 let frame = Frame::build_with_version(
6651 negotiated_ver,
6652 FrameType::Request,
6653 control_flags(),
6654 0,
6655 0,
6656 corr,
6657 body,
6658 )
6659 .map_err(|err| {
6660 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6661 })?;
6662
6663 match timeout_at(deadline, module_sink.send(frame)).await {
6669 Ok(Ok(())) => {}
6670 Ok(Err(err)) => {
6671 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6672 return Err(HealthProbeError::lane_dead(format!(
6675 "failed to send health.check: {err}"
6676 )));
6677 }
6678 Err(_elapsed) => {
6679 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6680 return Err(HealthProbeError::no_answer(
6684 "health.check send timed out before enqueue (module egress full)",
6685 ));
6686 }
6687 }
6688
6689 match timeout_at(deadline, receiver).await {
6690 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6691 "health.check returned an unexpected config response",
6692 )),
6693 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6697 response.health_report().ok_or_else(|| {
6698 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6699 })
6700 }
6701 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6702 format!("health.check rejected: {}", body.message),
6703 )),
6704 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6705 Err(HealthProbeError::lane_dead(message))
6706 }
6707 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6708 Err(HealthProbeError::bad_answer(message))
6709 }
6710 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6711 Err(HealthProbeError::bad_answer(format!(
6712 "expected module-control op '{expected}', got '{actual}'"
6713 )))
6714 }
6715 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6719 "module answered health.check after its daemon deadline",
6720 )),
6721 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6722 "health.check waiter was canceled before the module responded",
6723 )),
6724 Err(_) => {
6725 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6726 Err(HealthProbeError::no_answer(format!(
6727 "module did not answer health.check within {probe_budget:?}"
6728 )))
6729 }
6730 }
6731}
6732
6733#[allow(clippy::too_many_arguments)]
6737async fn handle_health_report(
6738 spec: &ModuleSpec,
6739 runtime: &SupervisorRuntimeConfig,
6740 registry: &Registry,
6741 process_liveness: &SupervisorProcessLiveness,
6742 snapshot: &SharedSnapshot,
6743 child: &mut Option<SupervisedChild>,
6744 report: HealthReport,
6745 now_ms: u64,
6746 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6747 probed: Option<ConnectionId>,
6748) {
6749 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6750 (Some(delivery), Some(probe)) => {
6751 delivery
6752 .health_result(
6753 &spec.module_id,
6754 runtime.forwarding.as_deref(),
6755 registry,
6756 probe,
6757 report.status == HealthStatus::Ok,
6758 )
6759 .await
6760 }
6761 _ => false,
6762 };
6763 if let (Some(delivery), Some(spawn)) = (
6764 &runtime.config_delivery,
6765 child
6766 .as_ref()
6767 .and_then(|child| child.config_delivery.as_ref()),
6768 ) {
6769 delivery
6770 .mark_readiness(
6771 &spec.module_id,
6772 runtime.forwarding.as_deref(),
6773 probed,
6774 spawn,
6775 report.status == HealthStatus::Ok,
6776 )
6777 .await;
6778 }
6779 let status = supervisor_health_status(report.status);
6780 let detail = report.detail.clone();
6781 let metrics = truncate_health_metrics(report.metrics);
6782 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6783 state.health.status = status;
6784 state.last_probe_evidence = None;
6785 state.health.last_probe_ms = Some(now_ms);
6786 state.health.detail = detail.clone();
6787 state.health.metrics = metrics.clone();
6788 state.health.consecutive_failures = 0;
6789 state.failed_retry.observe_health(
6790 report.status == HealthStatus::Ok,
6791 runtime.restart_policy.window,
6792 );
6793 });
6794
6795 if delivery_failed {
6796 return;
6797 }
6798 let action = match report.status {
6799 HealthStatus::Ok => return,
6800 HealthStatus::Degraded => runtime.health.on_degraded,
6801 HealthStatus::Failing => runtime.health.on_failing,
6802 };
6803 apply_l3_health_action(
6804 spec,
6805 runtime,
6806 registry,
6807 process_liveness,
6808 snapshot,
6809 child,
6810 status,
6811 detail.as_deref(),
6812 action,
6813 now_ms,
6814 )
6815 .await;
6816}
6817
6818#[allow(clippy::too_many_arguments)]
6819async fn handle_health_probe_failure(
6820 spec: &ModuleSpec,
6821 runtime: &SupervisorRuntimeConfig,
6822 registry: &Registry,
6823 process_liveness: &SupervisorProcessLiveness,
6824 snapshot: &SharedSnapshot,
6825 child: &mut Option<SupervisedChild>,
6826 err: HealthProbeError,
6827 now_ms: u64,
6828 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6829) {
6830 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6831 (Some(delivery), Some(probe)) => {
6832 delivery
6833 .health_result(
6834 &spec.module_id,
6835 runtime.forwarding.as_deref(),
6836 registry,
6837 probe,
6838 false,
6839 )
6840 .await
6841 }
6842 _ => false,
6843 };
6844 let threshold = runtime.health.failure_threshold.max(1);
6845 let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6846 runtime.host_health.record_late_module(&spec.module_id)
6847 } else {
6848 false
6849 };
6850 let mut failures = 0;
6851 let detail = if held {
6856 format!("[{}] {err}; host overloaded", err.label())
6857 } else {
6858 format!("[{}] {err}", err.label())
6859 };
6860 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6861 state.last_probe_evidence = Some(err.evidence);
6862 state
6863 .failed_retry
6864 .observe_health(false, runtime.restart_policy.window);
6865 if held {
6868 state.health.status = SupervisorHealthStatus::Degraded;
6869 } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6870 state.health.status = SupervisorHealthStatus::Unknown;
6871 }
6872 state.health.last_probe_ms = Some(now_ms);
6873 state.health.consecutive_failures = if held {
6876 0
6877 } else {
6878 state.health.consecutive_failures.saturating_add(1)
6879 };
6880 state.health.detail = Some(detail.clone());
6881 state.health.metrics = None;
6882 failures = state.health.consecutive_failures;
6883 });
6884
6885 if delivery_failed {
6886 return;
6887 }
6888 if held || failures < threshold {
6889 warn!(
6890 module_id = %spec.module_id,
6891 consecutive_failures = failures,
6892 threshold,
6893 evidence = err.label(),
6894 detail = %detail,
6895 "health.check probe failed"
6896 );
6897 return;
6898 }
6899
6900 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6901 state.state = ModuleState::Unresponsive;
6902 state.health.status = SupervisorHealthStatus::Unresponsive;
6903 });
6904 if runtime.health.critical {
6908 error!(
6909 module_id = %spec.module_id,
6910 status = "unresponsive",
6911 evidence = err.label(),
6912 detail = %detail,
6913 "critical module health alert"
6914 );
6915 } else {
6916 warn!(
6917 module_id = %spec.module_id,
6918 status = "unresponsive",
6919 evidence = err.label(),
6920 detail = %detail,
6921 "module health threshold breached"
6922 );
6923 }
6924 if let Err(err) = health_restart_child(
6925 spec,
6926 runtime,
6927 registry,
6928 process_liveness,
6929 snapshot,
6930 child,
6931 SupervisorHealthStatus::Unresponsive,
6932 Some(&detail),
6933 now_ms,
6934 )
6935 .await
6936 {
6937 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6938 }
6939}
6940
6941#[allow(clippy::too_many_arguments)]
6942async fn apply_l3_health_action(
6943 spec: &ModuleSpec,
6944 runtime: &SupervisorRuntimeConfig,
6945 registry: &Registry,
6946 process_liveness: &SupervisorProcessLiveness,
6947 snapshot: &SharedSnapshot,
6948 child: &mut Option<SupervisedChild>,
6949 status: SupervisorHealthStatus,
6950 detail: Option<&str>,
6951 action: HealthAction,
6952 now_ms: u64,
6953) {
6954 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6955 match action {
6956 HealthAction::Report => {
6957 info!(
6958 module_id = %spec.module_id,
6959 status = ?status,
6960 detail,
6961 "module reported non-ok health"
6962 );
6963 }
6964 HealthAction::Alert => {
6965 error!(
6966 module_id = %spec.module_id,
6967 status = ?status,
6968 detail,
6969 "module health alert"
6970 );
6971 }
6972 HealthAction::Restart => {
6973 if let Err(err) = health_restart_child(
6974 spec,
6975 runtime,
6976 registry,
6977 process_liveness,
6978 snapshot,
6979 child,
6980 status,
6981 detail,
6982 now_ms,
6983 )
6984 .await
6985 {
6986 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6987 }
6988 }
6989 }
6990}
6991
6992#[allow(clippy::too_many_arguments)]
6993async fn health_restart_child(
6994 spec: &ModuleSpec,
6995 runtime: &SupervisorRuntimeConfig,
6996 registry: &Registry,
6997 process_liveness: &SupervisorProcessLiveness,
6998 snapshot: &SharedSnapshot,
6999 child: &mut Option<SupervisedChild>,
7000 status: SupervisorHealthStatus,
7001 detail: Option<&str>,
7002 now_ms: u64,
7003) -> Result<(), SuperviseError> {
7004 budgeted_restart_child(
7005 spec,
7006 runtime,
7007 registry,
7008 process_liveness,
7009 snapshot,
7010 child,
7011 status,
7012 detail,
7013 now_ms,
7014 true,
7015 )
7016 .await
7017}
7018
7019#[allow(clippy::too_many_arguments)]
7020async fn budgeted_restart_child(
7021 spec: &ModuleSpec,
7022 runtime: &SupervisorRuntimeConfig,
7023 registry: &Registry,
7024 process_liveness: &SupervisorProcessLiveness,
7025 snapshot: &SharedSnapshot,
7026 child: &mut Option<SupervisedChild>,
7027 status: SupervisorHealthStatus,
7028 detail: Option<&str>,
7029 now_ms: u64,
7030 health_failure: bool,
7031) -> Result<(), SuperviseError> {
7032 if let Some(error) = runtime
7033 .config_delivery
7034 .as_ref()
7035 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
7036 .filter(|error| error.code != "config_delivery_deferred")
7037 {
7038 return Err(config_delivery::gate_hold_error(
7039 spec,
7040 error,
7041 runtime.config_delivery.as_ref(),
7042 ));
7043 }
7044 let recovery_reason = if health_failure {
7045 "health_check"
7046 } else {
7047 "configuration_delivery"
7048 };
7049 let (enabled, schedule) = {
7050 let mut state = lock_snapshot(snapshot)?;
7051 let enabled = state.enabled;
7052 let schedule = if enabled {
7053 state.next_crash_restart(&runtime.restart_policy, Instant::now())
7054 } else {
7055 None
7056 };
7057 (enabled, schedule)
7058 };
7059
7060 if !enabled {
7061 return Err(SuperviseError::Disabled {
7062 module_id: spec.module_id.clone(),
7063 });
7064 }
7065
7066 if schedule.is_none() {
7067 if health_failure {
7068 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
7069 }
7070 error!(
7071 module_id = %spec.module_id,
7072 status = ?status,
7073 detail,
7074 max_restarts = runtime.restart_policy.max_restarts,
7075 window_secs = runtime.restart_policy.window.as_secs(),
7076 reason = %runtime.restart_policy.budget_exhausted_detail(),
7077 recovery_reason,
7078 "automatic restart budget exhausted; marking module failed"
7079 );
7080 let stop_notice = begin_forwarding_drain_if_configured(
7081 spec,
7082 runtime,
7083 registry,
7084 snapshot,
7085 Some(true),
7086 RouteCloseReason::Disable,
7087 )
7088 .await?;
7089 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7090 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7091 })?;
7092 let result = drain_optional_child(
7093 &spec.module_id,
7094 spec.protocol,
7095 stop_notice,
7096 registry,
7097 runtime.forwarding.as_deref(),
7098 snapshot,
7099 &runtime.terminal_ring,
7100 &runtime.spawn_events,
7101 child,
7102 runtime.drain_timeout,
7103 ModuleState::Failed,
7104 Some(true),
7105 &runtime.failure_context(true),
7106 )
7107 .await;
7108 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7109 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7110 state.failed_retry.schedule(&spec.module_id);
7111 }
7112 })?;
7113 result?;
7114 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7115 return Ok(());
7116 }
7117
7118 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7119 let mut restart_count = 0;
7120 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7121 restart_count = state.crash_restarts.len();
7122 state.state = ModuleState::Unresponsive;
7123 if health_failure {
7124 state.health.status = status;
7125 state.health.last_action = Some(HealthAction::Restart.to_string());
7126 state.health.last_action_ms = Some(now_ms);
7127 }
7128 })?;
7129 warn!(
7130 module_id = %spec.module_id,
7131 status = ?status,
7132 detail,
7133 restart_count,
7134 restart_in_window = schedule.restart_in_window,
7135 delay_ms = schedule.delay.as_millis() as u64,
7136 recovery_reason,
7137 "automatic module restart"
7138 );
7139
7140 let stop_notice = begin_forwarding_drain_if_configured(
7141 spec,
7142 runtime,
7143 registry,
7144 snapshot,
7145 Some(true),
7146 RouteCloseReason::Restart,
7147 )
7148 .await?;
7149 drain_optional_child(
7150 &spec.module_id,
7151 spec.protocol,
7152 stop_notice,
7153 registry,
7154 runtime.forwarding.as_deref(),
7155 snapshot,
7156 &runtime.terminal_ring,
7157 &runtime.spawn_events,
7158 child,
7159 runtime.drain_timeout,
7160 ModuleState::Restarting,
7161 Some(true),
7162 &runtime.failure_context(true),
7163 )
7164 .await?;
7165 schedule_respawn(
7166 runtime,
7167 snapshot,
7168 &spec.module_id,
7169 schedule.delay,
7170 RespawnKind::Spawn,
7171 true,
7172 )
7173}
7174
7175fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7176 if let Some(reply) = runtime
7177 .deferred_reload_reply
7178 .lock()
7179 .unwrap_or_else(|p| p.into_inner())
7180 .take()
7181 {
7182 let _ = reply.send(Err(SuperviseError::ReloadFailed {
7183 module_id: module_id.to_string(),
7184 reason: reason.to_string(),
7185 }));
7186 }
7187}
7188
7189fn park_config_spawn(
7190 spec: &ModuleSpec,
7191 runtime: &SupervisorRuntimeConfig,
7192 snapshot: &SharedSnapshot,
7193 failure: &FailureContext,
7194 admission_refused: Option<String>,
7195) -> Result<(), SuperviseError> {
7196 *runtime
7197 .scheduled_respawn
7198 .lock()
7199 .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7200 if let Some(reply) = runtime
7201 .deferred_reload_reply
7202 .lock()
7203 .unwrap_or_else(|poisoned| poisoned.into_inner())
7204 .take()
7205 {
7206 let _ = reply.send(Err(SuperviseError::ConfigHeld {
7207 module_id: spec.module_id.clone(),
7208 code: "config_gated".into(),
7209 detail: "module is held by configuration admission".into(),
7210 }));
7211 }
7212 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7213 state.admission_refused = admission_refused;
7214 state.config_hold_daemon_initiated = failure.daemon_initiated;
7215 state.config_hold_approval = failure.approval.clone();
7216 state.config_hold_parked = true;
7217 state.state = ModuleState::Stopped;
7218 state.respawn_pending = false;
7219 state.failed_retry.pending = None;
7220 state.failed_retry.failed_at = None;
7223 clear_current_process_facts(state);
7224 })
7225}
7226
7227fn try_park_config_attempt(
7228 spec: &ModuleSpec,
7229 runtime: &SupervisorRuntimeConfig,
7230 snapshot: &SharedSnapshot,
7231 failure: &FailureContext,
7232 program_base: Option<&subc_config::Digest>,
7233) -> Result<bool, SuperviseError> {
7234 let result = (|| {
7235 if !lock_snapshot(snapshot)?.enabled {
7236 return Ok(false);
7237 }
7238 #[cfg(test)]
7239 if let Some(delivery) = &runtime.config_delivery {
7240 delivery.0.pause_before_config_park_for_test();
7241 }
7242 let Some(delivery) = &runtime.config_delivery else {
7243 return Ok(false);
7244 };
7245 let parked = delivery
7246 .0
7247 .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7248 park_config_spawn(spec, runtime, snapshot, failure, refusal)
7249 })
7250 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7251 if let Some(result) = parked {
7252 result?;
7253 Ok(true)
7254 } else {
7255 Ok(false)
7256 }
7257 })();
7258 if let Err(error) = &result {
7259 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7263 record_approved_attempt_failure(spec, snapshot, failure, error);
7264 }
7265 result
7266}
7267
7268fn record_approved_attempt_failure(
7269 spec: &ModuleSpec,
7270 snapshot: &SharedSnapshot,
7271 failure: &FailureContext,
7272 error: &SuperviseError,
7273) {
7274 let Some(approval) = &failure.approval else {
7275 return;
7276 };
7277 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7278 if state.enabled
7279 && state.state == ModuleState::Failed
7280 && state
7281 .approved_start_failure
7282 .as_ref()
7283 .is_none_or(|(identity, _)| identity != approval)
7284 {
7285 state.approved_start_failure = Some((approval.clone(), error.to_string()));
7286 }
7287 });
7288}
7289
7290fn hold_config_spawn(
7291 spec: &ModuleSpec,
7292 runtime: &SupervisorRuntimeConfig,
7293 snapshot: &SharedSnapshot,
7294 failure: &FailureContext,
7295) -> Result<(), SuperviseError> {
7296 if !lock_snapshot(snapshot)?.enabled {
7297 return Ok(());
7298 }
7299 if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7300 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7303 }
7304 Ok(())
7305}
7306
7307fn schedule_respawn(
7308 runtime: &SupervisorRuntimeConfig,
7309 snapshot: &SharedSnapshot,
7310 module_id: &str,
7311 delay: Duration,
7312 kind: RespawnKind,
7313 daemon_initiated: bool,
7314) -> Result<(), SuperviseError> {
7315 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7316 update_snapshot(snapshot, Some(module_id), |state| {
7317 state.respawn_pending = true
7318 })?;
7319 *runtime
7320 .scheduled_respawn
7321 .lock()
7322 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7323 deadline: Instant::now() + delay,
7324 kind,
7325 daemon_initiated,
7326 approval: None,
7327 });
7328 Ok(())
7329}
7330
7331const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7332
7333fn defer_config_spawn(
7334 spec: &ModuleSpec,
7335 runtime: &SupervisorRuntimeConfig,
7336 snapshot: &SharedSnapshot,
7337 kind: RespawnKind,
7338 failure: &FailureContext,
7339) -> Result<(), SuperviseError> {
7340 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7341 state.state = ModuleState::Restarting;
7342 state.respawn_pending = true;
7343 clear_current_process_facts(state);
7344 })?;
7345 *runtime
7346 .scheduled_respawn
7347 .lock()
7348 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7349 deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7350 kind,
7351 daemon_initiated: failure.daemon_initiated,
7352 approval: failure.approval.clone(),
7353 });
7354 Ok(())
7355}
7356
7357fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7358 let _ = update_snapshot(snapshot, Some(module_id), |state| {
7359 state.health.last_action = Some(action);
7360 state.health.last_action_ms = Some(now_ms);
7361 });
7362}
7363
7364fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7365 match status {
7366 HealthStatus::Ok => SupervisorHealthStatus::Ok,
7367 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7368 HealthStatus::Failing => SupervisorHealthStatus::Failing,
7369 }
7370}
7371
7372fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7384 let metrics = metrics?;
7385 match serde_json::to_vec(&metrics) {
7386 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7387 "truncated": true,
7388 "original_bytes": encoded.len(),
7389 })),
7390 Ok(_) | Err(_) => Some(metrics),
7391 }
7392}
7393
7394fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7400 if cadence.is_zero() {
7401 return Duration::ZERO;
7402 }
7403 let cadence_ms = cadence.as_millis() as u64;
7404 if cadence_ms == 0 {
7420 return cadence;
7421 }
7422 let jitter_span = (cadence_ms / 10).max(1);
7437 let hash = module_id.as_bytes().iter().fold(
7438 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7439 |acc, byte| {
7440 acc.wrapping_mul(1099511628211)
7441 .wrapping_add(u64::from(*byte))
7442 },
7443 );
7444 cadence + Duration::from_millis(hash % jitter_span)
7445}
7446
7447#[cfg(test)]
7448mod tests {
7449 use super::*;
7450
7451 #[test]
7452 fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7453 let handle = SupervisorHandle::new();
7454 handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7455 handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7456 let owner = handle
7457 .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7458 .unwrap();
7459 assert!(owner.nonce_verified);
7460 assert!(
7461 owner.prefix_owner.is_none(),
7462 "an unreserved programmatic owner is not its own prefix child"
7463 );
7464 let child = handle
7465 .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7466 .unwrap();
7467 assert!(child.nonce_verified);
7468 assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7469 }
7470
7471 #[test]
7472 fn readding_a_module_clears_its_rescan_removal_tombstone() {
7473 let handle = SupervisorHandle::new();
7474 let module_id = "readded-tombstone";
7475 handle.record_rescan_removal(module_id);
7476 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7477
7478 handle.apply_identity_configuration(&ModuleSpec {
7479 module_id: module_id.to_string(),
7480 program: PathBuf::from("/test/module"),
7481 args: Vec::new(),
7482 env: Vec::new(),
7483 reserved: false,
7484 reserved_prefixes: Vec::new(),
7485 protocol: ModuleProtocol::Subc,
7486 overlap: Default::default(),
7487 carry_containment: false,
7488 });
7489
7490 assert!(
7491 handle.removal_tombstone_age_ms(module_id).is_none(),
7492 "a re-added module must not retain a stale removal tombstone"
7493 );
7494 }
7495
7496 #[derive(Debug, PartialEq, Eq)]
7499 struct OwnerInSpawnWindow {
7500 module_id: String,
7501 configured: bool,
7502 on_roster: bool,
7503 admission_refusal: Option<&'static str>,
7504 }
7505
7506 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7517 async fn a_new_module_is_configured_before_its_first_process_can_register() {
7518 use crate::scopes::ScopeTable;
7519 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7520
7521 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7522 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7523 module_id: module_id.to_string(),
7524 program,
7525 args: Vec::new(),
7526 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7527 .into_iter()
7528 .map(|key| (key.to_string(), dir.path().display().to_string()))
7529 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7530 .collect(),
7531 reserved: false,
7532 reserved_prefixes: Vec::new(),
7533 protocol: ModuleProtocol::Subc,
7534 overlap: Default::default(),
7535 carry_containment: false,
7536 };
7537 let live = super::terminal_history_tests::fake_aft_stub_path();
7538 let missing = dir.path().join("definitely-missing-module");
7539
7540 let handle = SupervisorHandle::new();
7541 let mut supervisor =
7542 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7543 .with_handle(handle.clone());
7544 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7545 let hook_handle = handle.clone();
7546 let hook_observed = Arc::clone(&observed);
7547 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7548 let configured = hook_handle.is_configured(module_id);
7551 let selector = ScopeSelector::new(
7552 Principal::Reserved {
7553 module_id: module_id.to_string(),
7554 },
7555 "s".to_string(),
7556 Some(1),
7557 );
7558 let carrier = Principal::Reserved {
7559 module_id: "carrier".to_string(),
7560 };
7561 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7562 .admit(&carrier, module_id, &selector, configured)
7563 {
7564 Ok(_) => None,
7565 Err(refusal) => Some(refusal.code),
7566 };
7567 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7568 module_id: module_id.to_string(),
7569 configured,
7570 on_roster: hook_handle.get(module_id).is_some(),
7571 admission_refusal,
7572 });
7573 })));
7574
7575 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7576 let configured = supervisor
7577 .supervise_configured(stub("configured", live.clone()), true)
7578 .unwrap();
7579 let with_health = supervisor
7580 .supervise_configured_with_health(
7581 stub("with-health", live.clone()),
7582 true,
7583 HealthConfig::default(),
7584 None,
7585 RestartPolicy::default(),
7586 )
7587 .unwrap();
7588 let failed = supervisor
7591 .supervise_configured_with_health(
7592 stub("failed-spawn", missing.clone()),
7593 true,
7594 HealthConfig::default(),
7595 None,
7596 RestartPolicy::default(),
7597 )
7598 .unwrap();
7599 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7602
7603 let expected = [
7604 "plain",
7605 "configured",
7606 "with-health",
7607 "failed-spawn",
7608 "spawn-error",
7609 ]
7610 .into_iter()
7611 .map(|module_id| OwnerInSpawnWindow {
7612 module_id: module_id.to_string(),
7613 configured: true,
7614 on_roster: !matches!(module_id, "plain" | "spawn-error"),
7615 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7616 })
7617 .collect::<Vec<_>>();
7618 assert_eq!(*observed.lock().unwrap(), expected);
7619
7620 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7621 assert!(
7622 handle.get(module_id).is_some(),
7623 "{module_id} is on the roster"
7624 );
7625 assert!(
7626 handle.is_configured(module_id),
7627 "{module_id} stays configured"
7628 );
7629 }
7630 assert!(handle.get("spawn-error").is_none());
7631 assert!(
7632 !handle.is_configured("spawn-error"),
7633 "a plain spawn that failed must not leave its module marked configured"
7634 );
7635
7636 handle.retire("failed-spawn");
7638 assert!(!handle.is_configured("failed-spawn"));
7639
7640 for module in [plain, configured, with_health] {
7641 module.stop().await.unwrap();
7642 }
7643 drop(failed);
7644 }
7645
7646 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7653 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7654 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7655 let spec = ModuleSpec {
7656 module_id: "spawned-not-admitted".to_string(),
7657 program: super::terminal_history_tests::fake_aft_stub_path(),
7658 args: Vec::new(),
7659 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7660 .into_iter()
7661 .map(|key| (key.to_string(), dir.path().display().to_string()))
7662 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7663 .collect(),
7664 reserved: false,
7665 reserved_prefixes: Vec::new(),
7666 protocol: ModuleProtocol::Subc,
7667 overlap: Default::default(),
7668 carry_containment: false,
7669 };
7670 let handle = SupervisorHandle::new();
7671 let mut supervisor =
7672 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7673 .with_handle(handle.clone());
7674 let observed = Arc::new(Mutex::new(None));
7675 let hook_handle = handle.clone();
7676 let hook_observed = Arc::clone(&observed);
7677 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7678 let state = hook_handle
7679 .get(module_id)
7680 .expect("a configured module is on the roster before it is spawned")
7681 .state()
7682 .unwrap();
7683 *hook_observed.lock().unwrap() = Some(state);
7684 })));
7685
7686 let module = supervisor.supervise_configured(spec, true).unwrap();
7687 assert_eq!(
7688 *observed.lock().unwrap(),
7689 Some(ModuleState::Starting),
7690 "between the spawn and its admission the supervisor still reports starting"
7691 );
7692 assert_eq!(module.state().unwrap(), ModuleState::Running);
7693 module.stop().await.unwrap();
7694 }
7695
7696 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7697 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7698 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7699 snapshot.process_alive = true;
7700 snapshot.pid = Some(41);
7701 snapshot.spawned_at_ms = Some(42);
7702 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7703 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7704 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7705 device: 43,
7706 inode: 44,
7707 #[cfg(windows)]
7708 windows_image: None,
7709 });
7710 })
7711 .unwrap();
7712 snapshot
7713 }
7714
7715 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7716 let snapshot = lock_snapshot(snapshot).unwrap();
7717 assert!(!snapshot.process_alive);
7718 assert_eq!(snapshot.pid, None);
7719 assert_eq!(snapshot.spawned_at_ms, None);
7720 assert_eq!(snapshot.spawned_from, None);
7721 assert_eq!(snapshot.launch_spec_digest, None);
7722 assert_eq!(snapshot.spawned_file_identity, None);
7723 }
7724
7725 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7726 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7727 let supervisor =
7728 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7729 let mut runtime = supervisor.runtime_config();
7730 runtime.test_seed_stale_facts_before_enable_spawn = true;
7731 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7732 let mut child = None;
7733 let spec = ModuleSpec {
7734 module_id: "failed-enable-clears-facts".to_string(),
7735 program: PathBuf::from("/definitely/missing/failed-enable-module"),
7736 args: Vec::new(),
7737 env: Vec::new(),
7738 reserved: false,
7739 reserved_prefixes: Vec::new(),
7740 protocol: ModuleProtocol::Subc,
7741 overlap: Default::default(),
7742 carry_containment: false,
7743 };
7744
7745 let result = set_child_enabled(
7746 &spec,
7747 &runtime,
7748 &supervisor.registry,
7749 &supervisor.process_liveness,
7750 &snapshot,
7751 &mut child,
7752 true,
7753 &runtime.failure_context(false),
7754 )
7755 .await;
7756
7757 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7758 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7759 assert_snapshot_process_facts_cleared(&snapshot);
7760 }
7761
7762 #[tokio::test]
7763 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7764 let supervisor =
7765 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7766 let runtime = supervisor.runtime_config();
7767 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7768 ModuleState::Restarting,
7769 true,
7770 )));
7771 let spec = ModuleSpec {
7772 module_id: "start-stranded-restarting".to_string(),
7773 program: super::terminal_history_tests::fake_aft_stub_path(),
7774 args: Vec::new(),
7775 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7776 reserved: false,
7777 reserved_prefixes: Vec::new(),
7778 protocol: ModuleProtocol::None,
7779 overlap: Default::default(),
7780 carry_containment: false,
7781 };
7782 let mut child = None;
7783 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7784 assert!(!super::set_child_enabled(
7785 &spec,
7786 &runtime,
7787 &Registry::default(),
7788 &supervisor.process_liveness,
7789 &snapshot,
7790 &mut child,
7791 true,
7792 &runtime.failure_context(false),
7793 )
7794 .await
7795 .unwrap());
7796 assert!(child.is_none());
7797 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7798 assert!(super::set_child_enabled(
7799 &spec,
7800 &runtime,
7801 &Registry::default(),
7802 &supervisor.process_liveness,
7803 &snapshot,
7804 &mut child,
7805 true,
7806 &runtime.failure_context(false),
7807 )
7808 .await
7809 .unwrap());
7810 assert_eq!(
7811 lock_snapshot(&snapshot).unwrap().state,
7812 ModuleState::Running
7813 );
7814 let mut child = child.unwrap();
7815 child.start_kill().unwrap();
7816 child.wait().await.unwrap();
7817 }
7818
7819 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7820 async fn failed_reload_spawn_clears_current_process_facts() {
7821 let supervisor =
7822 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7823 let mut runtime = supervisor.runtime_config();
7824 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7825 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7826 let mut child = None;
7827 let spec = ModuleSpec {
7828 module_id: "failed-reload-clears-facts".to_string(),
7829 program: PathBuf::from("/unused/failed-reload-module"),
7830 args: Vec::new(),
7831 env: Vec::new(),
7832 reserved: false,
7833 reserved_prefixes: Vec::new(),
7834 protocol: ModuleProtocol::Subc,
7835 overlap: Default::default(),
7836 carry_containment: false,
7837 };
7838
7839 let result = handle_reload_spawn_failure(
7840 &spec,
7841 &runtime,
7842 &supervisor.process_liveness,
7843 &snapshot,
7844 &mut child,
7845 "forced reload spawn failure".to_string(),
7846 )
7847 .await;
7848
7849 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7850 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7851 assert_snapshot_process_facts_cleared(&snapshot);
7852 }
7853
7854 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7855 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7856 let supervisor =
7857 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7858 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7859 let module = supervisor.supervised_module(
7860 ModuleSpec {
7861 module_id: "drop-clears-facts".to_string(),
7862 program: PathBuf::from("/unused/drop-module"),
7863 args: Vec::new(),
7864 env: Vec::new(),
7865 reserved: false,
7866 reserved_prefixes: Vec::new(),
7867 protocol: ModuleProtocol::Subc,
7868 overlap: Default::default(),
7869 carry_containment: false,
7870 },
7871 supervisor.runtime_config(),
7872 Arc::clone(&snapshot),
7873 None,
7874 );
7875 assert!(!module
7876 .inner
7877 .monitor
7878 .lock()
7879 .unwrap()
7880 .as_ref()
7881 .unwrap()
7882 .is_finished());
7883
7884 drop(module);
7885
7886 assert_eq!(
7887 lock_snapshot(&snapshot).unwrap().state,
7888 ModuleState::Stopped
7889 );
7890 assert_snapshot_process_facts_cleared(&snapshot);
7891 }
7892
7893 #[cfg(unix)]
7894 #[tokio::test]
7895 async fn rescan_preserves_running_protocol_until_respawn() {
7896 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7897 let initial = ModuleSpec {
7898 module_id: "rescan-protocol".into(),
7899 program: PathBuf::from("/bin/sleep"),
7900 args: vec!["60".into()],
7901 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7902 .into_iter()
7903 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7904 .collect(),
7905 reserved: false,
7906 reserved_prefixes: vec![],
7907 protocol: ModuleProtocol::None,
7908 overlap: Default::default(),
7909 carry_containment: false,
7910 };
7911 let supervisor =
7912 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7913 let module = supervisor.spawn(initial.clone()).unwrap();
7914 assert!(module.status().unwrap().live);
7915 let mut next = initial;
7916 next.protocol = ModuleProtocol::Subc;
7917 module
7918 .update_configuration(next.clone(), HealthConfig::default(), None)
7919 .await
7920 .unwrap();
7921 assert!(
7922 module.status().unwrap().live,
7923 "rescan must not require HELLO from the old non-wire process"
7924 );
7925 let runtime = supervisor.runtime_config();
7926 let action = on_child_exit(
7927 &next,
7928 RestartPolicy::default(),
7929 &supervisor.registry,
7930 &module.inner.snapshot,
7931 &runtime.terminal_ring,
7932 &runtime.spawn_events,
7933 &runtime.child_roster,
7934 ExitReport {
7935 kind: ExitKind::Clean,
7936 code: Some(0),
7937 signal: None,
7938 at_ms: unix_ms_now(),
7939 },
7940 true,
7941 )
7942 .await;
7943 assert!(
7944 matches!(action, NextAction::Restart { .. }),
7945 "the old non-wire process's clean exit must restart"
7946 );
7947 module.drain().await.unwrap();
7948 }
7949
7950 #[cfg(unix)]
7951 #[tokio::test]
7952 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
7953 use std::os::unix::fs::PermissionsExt;
7954 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
7955 let script = dir.join("module.sh");
7956 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
7957 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
7958 let record_path = dir.join("live-children.json");
7959 let supervisor =
7960 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7961 .with_live_children_record(&record_path);
7962 for (program, args) in [
7963 (PathBuf::from("sleep"), vec!["60".into()]),
7964 (script, vec![]),
7965 ] {
7966 let spec = ModuleSpec {
7967 module_id: "image-identity".into(),
7968 program: program.clone(),
7969 args,
7970 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7971 .into_iter()
7972 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
7973 .collect(),
7974 reserved: false,
7975 reserved_prefixes: vec![],
7976 protocol: ModuleProtocol::None,
7977 overlap: Default::default(),
7978 carry_containment: false,
7979 };
7980 let module = supervisor.spawn(spec).unwrap();
7981 #[cfg(target_os = "macos")]
7982 {
7983 let deadline = Instant::now() + Duration::from_secs(5);
7986 while crate::live_children::read_record(&record_path)
7987 .unwrap()
7988 .iter()
7989 .all(|entry| entry.executable.is_none())
7990 {
7991 assert!(Instant::now() < deadline, "module image was not confirmed");
7992 tokio::time::sleep(Duration::from_millis(5)).await;
7993 }
7994 }
7995 let entry = crate::live_children::read_record(&record_path)
7996 .unwrap()
7997 .pop()
7998 .unwrap();
7999 let observed = subc_os::Process::open(entry.pid)
8000 .unwrap()
8001 .unwrap()
8002 .observe()
8003 .unwrap();
8004 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
8005 module.drain().await.unwrap();
8006 assert_eq!(
8007 verdict,
8008 crate::live_children::IdentityVerdict::Matches,
8009 "program {program:?}: recorded {entry:?}, observed {observed:?}"
8010 );
8011 }
8012 }
8013
8014 #[cfg(unix)]
8015 fn http_fixture(
8016 dir: &std::path::Path,
8017 url: &str,
8018 threshold: u32,
8019 ) -> crate::daemon_config::ConfiguredModule {
8020 let path = dir.join("subc.jsonc");
8021 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
8022 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
8023 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
8024 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
8025 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
8026 }}}).to_string()).unwrap();
8027 crate::daemon_config::load(&path)
8028 .unwrap()
8029 .unwrap()
8030 .modules
8031 .pop()
8032 .unwrap()
8033 }
8034
8035 #[cfg(unix)]
8036 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
8037 timeout(Duration::from_secs(5), async {
8038 loop {
8039 if module.status().unwrap().health.status == status {
8040 break;
8041 }
8042 sleep(Duration::from_millis(5)).await;
8043 }
8044 })
8045 .await
8046 .unwrap_or_else(|_| {
8047 panic!(
8048 "expected {status:?}, got {:?}",
8049 module.status().unwrap().health
8050 )
8051 });
8052 }
8053
8054 #[cfg(unix)]
8055 #[tokio::test]
8056 async fn http_health_status_flips_ok_failing_ok() {
8057 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8058 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
8059 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8060 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
8061 let serving_status = status.clone();
8062 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8063 let server = tokio::spawn(async move {
8064 loop {
8065 let (mut stream, _) = listener.accept().await.unwrap();
8066 let mut request = [0u8; 2048];
8067 let count = stream.read(&mut request).await.unwrap();
8068 assert!(count > 0, "a probe must send an HTTP request");
8069 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
8070 let body = if code == 200 {
8071 "ready"
8072 } else {
8073 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
8074 };
8075 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
8076 let _ = stream.write_all(response.as_bytes()).await;
8077 }
8078 });
8079 let configured = http_fixture(&dir, &url, 1000);
8080 let module =
8081 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8082 .supervise_configured_with_health(
8083 configured.module_spec(),
8084 true,
8085 configured.health,
8086 configured.drain_timeout_ms,
8087 configured.restart,
8088 )
8089 .unwrap();
8090 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8091 status.store(503, std::sync::atomic::Ordering::SeqCst);
8092 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8093 assert!(module
8094 .status()
8095 .unwrap()
8096 .health
8097 .detail
8098 .unwrap()
8099 .contains("scratch failure"));
8100 status.store(200, std::sync::atomic::Ordering::SeqCst);
8101 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8102 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8103 let before = module.status().unwrap();
8104 let (spec, mut health) = module.configuration().unwrap();
8105 health.http = None;
8106 module
8107 .update_configuration(spec.clone(), health.clone(), Some(10))
8108 .await
8109 .unwrap();
8110 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8111 health.http = Some(url);
8112 module
8113 .update_configuration(spec, health, Some(10))
8114 .await
8115 .unwrap();
8116 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8117 assert_eq!(
8118 module.status().unwrap().pid,
8119 before.pid,
8120 "changing a probe must apply live, not restart its process"
8121 );
8122 let (spec, mut health) = module.configuration().unwrap();
8123 health.failure_threshold = 2;
8124 module
8125 .update_configuration(spec, health, Some(10))
8126 .await
8127 .unwrap();
8128 status.store(503, std::sync::atomic::Ordering::SeqCst);
8129 timeout(Duration::from_secs(5), async {
8130 while module.status().unwrap().spawn_generation == before.spawn_generation {
8131 sleep(Duration::from_millis(5)).await;
8132 }
8133 })
8134 .await
8135 .expect("sustained HTTP 503 responses must trigger the health restart policy");
8136 module.drain().await.unwrap();
8137 server.abort();
8138 }
8139
8140 #[cfg(unix)]
8141 #[tokio::test]
8142 async fn http_health_replacement_recovers_after_one_restart() {
8143 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8144 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8145 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8146 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8147 let mut configured = http_fixture(&dir, &url, 2);
8148 configured.restart.max_restarts = 2;
8149 let module =
8150 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8151 .supervise_configured_with_health(
8152 configured.module_spec(),
8153 true,
8154 configured.health,
8155 configured.drain_timeout_ms,
8156 configured.restart,
8157 )
8158 .unwrap();
8159 let first_generation = module.status().unwrap().spawn_generation;
8160 let observed = module.clone();
8161 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8162 let server = tokio::spawn(async move {
8163 let mut generation = first_generation;
8164 let mut probes = 0;
8165 loop {
8166 let (mut stream, _) = listener.accept().await.unwrap();
8167 let mut request = [0; 2048];
8168 assert!(stream.read(&mut request).await.unwrap() > 0);
8169 let current = observed.status().unwrap().spawn_generation;
8170 if current != generation {
8171 generation = current;
8172 probes = 0;
8173 }
8174 probes += 1;
8175 let healthy = generation != first_generation && probes > 1;
8178 let code = if healthy { 200 } else { 503 };
8179 let response = format!(
8180 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8181 );
8182 let _ = stream.write_all(response.as_bytes()).await;
8183 if healthy {
8184 healthy_tx.send(()).await.unwrap();
8185 }
8186 }
8187 });
8188 timeout(Duration::from_secs(5), async {
8189 let mut healthy_probes = 0;
8190 loop {
8191 if module.state().unwrap() == ModuleState::Failed {
8192 break;
8193 }
8194 while healthy_rx.try_recv().is_ok() {
8195 healthy_probes += 1;
8196 }
8197 if healthy_probes >= 3
8198 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8199 {
8200 break;
8201 }
8202 sleep(Duration::from_millis(5)).await;
8203 }
8204 })
8205 .await
8206 .expect("the replacement must settle healthy without exhausting its restart budget");
8207 let status = module.status().unwrap();
8208 module.drain().await.unwrap();
8209 server.abort();
8210 assert_eq!(status.state, ModuleState::Running);
8211 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8212 assert_eq!(status.health.consecutive_failures, 0);
8213 assert_eq!(status.lifetime_restarts, 1);
8214 assert_eq!(status.restart_count, 1);
8215 }
8216
8217 #[cfg(unix)]
8218 #[tokio::test]
8219 async fn http_health_sustained_failure_exhausts_restart_budget() {
8220 use std::collections::BTreeMap;
8221 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8222 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8223 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8224 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8225 let mut configured = http_fixture(&dir, &url, 2);
8226 configured.restart.max_restarts = 2;
8227 let module =
8228 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8229 .supervise_configured_with_health(
8230 configured.module_spec(),
8231 true,
8232 configured.health,
8233 configured.drain_timeout_ms,
8234 configured.restart,
8235 )
8236 .unwrap();
8237 let observed = module.clone();
8238 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8239 let served = probes.clone();
8240 let server = tokio::spawn(async move {
8241 loop {
8242 let (mut stream, _) = listener.accept().await.unwrap();
8243 let mut request = [0; 2048];
8244 assert!(stream.read(&mut request).await.unwrap() > 0);
8245 *served
8246 .lock()
8247 .unwrap()
8248 .entry(observed.status().unwrap().spawn_generation)
8249 .or_default() += 1;
8250 let _ = stream
8251 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8252 .await;
8253 }
8254 });
8255 timeout(Duration::from_secs(5), async {
8256 while module.state().unwrap() != ModuleState::Failed {
8257 sleep(Duration::from_millis(5)).await;
8258 }
8259 })
8260 .await
8261 .expect("sustained failures must still exhaust the health restart budget");
8262 let status = module.status().unwrap();
8263 server.abort();
8264 assert_eq!(status.restart_count, 2);
8265 assert_eq!(status.lifetime_restarts, 2);
8266 assert!(!status.process_alive);
8267 assert_eq!(
8268 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8269 vec![2, 2, 2]
8270 );
8271 }
8272
8273 #[cfg(unix)]
8274 #[tokio::test]
8275 async fn spawn_clears_the_previous_process_health() {
8276 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8277 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8278 configured.health.cadence = Duration::from_secs(3600);
8279 let module =
8280 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8281 .supervise_configured_with_health(
8282 configured.module_spec(),
8283 false,
8284 configured.health,
8285 configured.drain_timeout_ms,
8286 configured.restart,
8287 )
8288 .unwrap();
8289 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8290 state.health.status = SupervisorHealthStatus::Unresponsive;
8291 state.health.consecutive_failures = 3;
8292 state.health.detail = Some("old process refused connections".into());
8293 state.health.metrics = Some(serde_json::json!({"old": true}));
8294 state.health.last_probe_ms = Some(123);
8295 state.health.last_action = Some("restart".into());
8296 state.health.last_action_ms = Some(124);
8297 })
8298 .unwrap();
8299 module.set_enabled(true).await.unwrap();
8300 let health = module.status().unwrap().health;
8301 module.drain().await.unwrap();
8302 assert_eq!(health.consecutive_failures, 0);
8303 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8304 assert_eq!(health.detail, None);
8305 assert_eq!(health.metrics, None);
8306 assert_eq!(health.last_probe_ms, None);
8307 assert_eq!(health.last_action.as_deref(), Some("restart"));
8308 assert_eq!(health.last_action_ms, Some(124));
8309 }
8310
8311 #[cfg(unix)]
8312 #[tokio::test]
8313 async fn http_health_no_listener_restarts_after_consecutive_failures() {
8314 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8315 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8316 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8317 drop(unused);
8318 let configured = http_fixture(&dir, &url, 2);
8319 let module =
8320 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8321 .supervise_configured_with_health(
8322 configured.module_spec(),
8323 true,
8324 configured.health,
8325 configured.drain_timeout_ms,
8326 configured.restart,
8327 )
8328 .unwrap();
8329 let before = module.status().unwrap().spawn_generation;
8330 timeout(Duration::from_secs(5), async {
8331 loop {
8332 let status = module.status().unwrap();
8333 if status.spawn_generation > before {
8334 assert!(status.lifetime_restarts > 0);
8335 break;
8336 }
8337 sleep(Duration::from_millis(5)).await;
8338 }
8339 })
8340 .await
8341 .expect("sustained HTTP refusal must trigger the health restart policy");
8342 module.drain().await.unwrap();
8343 }
8344
8345 #[cfg(unix)]
8346 #[tokio::test]
8347 async fn http_health_timeout_honours_deadline() {
8348 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8349 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8350 let server = tokio::spawn(async move {
8351 let _held = listener.accept().await.unwrap();
8352 std::future::pending::<()>().await;
8353 });
8354 let error = timeout(
8355 Duration::from_secs(1),
8356 probe_http_health(&url, Duration::from_millis(10)),
8357 )
8358 .await
8359 .expect("the probe must enforce its own deadline")
8360 .unwrap_err();
8361 server.abort();
8362 assert!(error.to_string().contains("timed out"));
8363 }
8364
8365 #[cfg(unix)]
8366 #[tokio::test]
8367 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8368 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8369 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8370 let url = format!(
8371 "http://localhost:{}/healthz",
8372 listener.local_addr().unwrap().port()
8373 );
8374 let server = tokio::spawn(async move {
8375 let (mut stream, _) = listener.accept().await.unwrap();
8376 let mut request = [0u8; 2048];
8377 assert!(stream.read(&mut request).await.unwrap() > 0);
8378 stream
8379 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8380 .await
8381 .unwrap();
8382 });
8383 assert_eq!(
8388 probe_http_health(&url, Duration::from_secs(10))
8389 .await
8390 .unwrap()
8391 .status,
8392 HealthStatus::Ok
8393 );
8394 server.await.unwrap();
8395 }
8396
8397 #[cfg(unix)]
8398 #[tokio::test]
8399 async fn http_health_timeout_keeps_partial_status_and_body() {
8400 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8401 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8402 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8403 let server = tokio::spawn(async move {
8404 let (mut stream, _) = listener.accept().await.unwrap();
8405 let mut request = [0u8; 2048];
8406 assert!(stream.read(&mut request).await.unwrap() > 0);
8407 stream
8408 .write_all(
8409 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8410 )
8411 .await
8412 .unwrap();
8413 std::future::pending::<()>().await;
8414 });
8415 let error = probe_http_health(&url, Duration::from_secs(1))
8416 .await
8417 .unwrap_err()
8418 .to_string();
8419 server.abort();
8420 assert!(
8421 error.contains("timed out")
8422 && error.contains("503 Unavailable")
8423 && error.contains("partial diagnostic"),
8424 "{error}"
8425 );
8426 }
8427
8428 #[cfg(unix)]
8429 #[tokio::test]
8430 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8431 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8432 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8433 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8434 let server = tokio::spawn(async move {
8435 let (mut stream, _) = listener.accept().await.unwrap();
8436 let mut request = [0u8; 2048];
8437 assert!(stream.read(&mut request).await.unwrap() > 0);
8438 let body = format!("{}not-in-diagnostic", "x".repeat(400));
8439 let response = format!(
8440 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8441 body.len()
8442 );
8443 stream.write_all(response.as_bytes()).await.unwrap();
8444 });
8445 let error = probe_http_health(&url, Duration::from_secs(1))
8446 .await
8447 .unwrap_err()
8448 .to_string();
8449 server.await.unwrap();
8450 assert!(error.ends_with(&"x".repeat(200)), "{error}");
8451 assert_eq!(error.split(": ").last().unwrap().len(), 200);
8452 assert!(!error.contains("not-in-diagnostic"));
8453 }
8454
8455 #[cfg(unix)]
8456 #[tokio::test]
8457 async fn http_health_real_nats_server_monitoring() {
8458 if std::process::Command::new("nats-server")
8459 .arg("--version")
8460 .env("XDG_DATA_HOME", std::env::temp_dir())
8461 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8462 .env("XDG_CONFIG_HOME", std::env::temp_dir())
8463 .output()
8464 .is_err()
8465 {
8466 eprintln!(
8467 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8468 );
8469 return;
8470 }
8471 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8472 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8473 let port = monitor.local_addr().unwrap().port();
8474 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8475 let client_port = client.local_addr().unwrap().port();
8476 let config = dir.join("server.conf");
8477 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8478 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8479 configured.program = PathBuf::from("nats-server");
8480 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8481 drop(monitor);
8482 drop(client);
8483 let module =
8484 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8485 .supervise_configured_with_health(
8486 configured.module_spec(),
8487 true,
8488 configured.health,
8489 configured.drain_timeout_ms,
8490 configured.restart,
8491 )
8492 .unwrap();
8493 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8494 module.drain().await.unwrap();
8495 }
8496
8497 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8498 async fn configuration_update_does_not_replace_captured_running_process_facts() {
8499 let supervisor =
8500 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8501 let snapshot = stale_process_snapshot(ModuleState::Running, true);
8502 let initial = ModuleSpec {
8503 module_id: "rescan-preserves-spawn-facts".to_string(),
8504 program: PathBuf::from("/spawned/module"),
8505 args: Vec::new(),
8506 env: Vec::new(),
8507 reserved: false,
8508 reserved_prefixes: Vec::new(),
8509 protocol: ModuleProtocol::Subc,
8510 overlap: Default::default(),
8511 carry_containment: false,
8512 };
8513 let module = supervisor.supervised_module(
8514 initial.clone(),
8515 supervisor.runtime_config(),
8516 snapshot,
8517 None,
8518 );
8519 let before = module.status().unwrap();
8520 let mut replacement = initial;
8521 replacement.program = PathBuf::from("/rescanned/replacement-module");
8522
8523 module
8524 .update_configuration(replacement, HealthConfig::default(), None)
8525 .await
8526 .unwrap();
8527
8528 let after = module.status().unwrap();
8529 assert_eq!(after.pid, before.pid);
8530 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8531 assert_eq!(after.spawned_from, before.spawned_from);
8532 drop(module);
8533 }
8534}
8535
8536fn unix_ms_now() -> u64 {
8537 SystemTime::now()
8538 .duration_since(UNIX_EPOCH)
8539 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8540 .unwrap_or(0)
8541}
8542
8543async fn supervise_loop(
8544 mut spec: ModuleSpec,
8545 mut runtime: SupervisorRuntimeConfig,
8546 registry: Arc<Registry>,
8547 process_liveness: Arc<SupervisorProcessLiveness>,
8548 snapshot: SharedSnapshot,
8549 mut child: Option<SupervisedChild>,
8550 mut commands: mpsc::Receiver<SupervisorCommand>,
8551) {
8552 let mut health_probe = HealthProbeRuntime::default();
8553 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8557 Ok(changes) => Some(changes),
8558 Err(err) => {
8559 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8562 None
8563 }
8564 };
8565 let mut pending_respawn: Option<PendingRespawn> = None;
8569 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8572 let mut previous_instance_recheck = tokio::time::interval_at(
8573 Instant::now() + Duration::from_secs(5),
8574 Duration::from_secs(5),
8575 );
8576 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8577 'supervision: loop {
8578 #[cfg(test)]
8579 {
8580 lock_snapshot(&snapshot).unwrap().actor_select = None;
8581 }
8582 #[cfg(target_os = "macos")]
8583 if let Some(active) = child.as_mut() {
8584 active.confirm_privacy_exec().await;
8585 }
8586 if let Some(scheduled) = runtime
8587 .scheduled_respawn
8588 .lock()
8589 .unwrap_or_else(|p| p.into_inner())
8590 .take()
8591 {
8592 runtime.recovery_is_daemon = scheduled.daemon_initiated;
8593 pending_respawn = Some(scheduled);
8594 }
8595 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8596 pending_respawn = None;
8597 cancel_deferred_reload(
8598 &runtime,
8599 &spec.module_id,
8600 "respawn cancelled by a supervisor command",
8601 );
8602 }
8603 if child.is_none()
8604 && pending_respawn.is_none()
8605 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8606 {
8607 cancel_deferred_reload(
8608 &runtime,
8609 &spec.module_id,
8610 "respawn cancelled before a replacement was spawned",
8611 );
8612 let mut stranded = false;
8613 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8614 state.respawn_pending = false;
8615 state.coalesced_restart_pending = false;
8616 if matches!(
8617 state.state,
8618 ModuleState::Restarting
8619 | ModuleState::Starting
8620 | ModuleState::Draining
8621 | ModuleState::Unresponsive
8622 ) {
8623 error!(module_id = %spec.module_id, state = ?state.state,
8624 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8625 stranded = true;
8626 }
8627 });
8628 if stranded {
8629 fail_snapshot(
8630 &snapshot,
8631 Some(&spec.module_id),
8632 None,
8633 &runtime.failure_context(runtime.recovery_is_daemon),
8634 );
8635 }
8636 }
8637 if child.is_none()
8638 && (runtime.child_roster.is_closed()
8639 || pending_respawn.is_none()
8640 || !lock_snapshot(&snapshot)
8641 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8642 {
8643 containment::terminate(&runtime.child_roster.containment).await;
8644 }
8645 if let Some(command) = requeued.pop_front() {
8646 if !handle_supervisor_command(
8647 command,
8648 &mut spec,
8649 &mut runtime,
8650 ®istry,
8651 &process_liveness,
8652 &snapshot,
8653 &mut child,
8654 &mut commands,
8655 &mut requeued,
8656 )
8657 .await
8658 {
8659 containment::terminate(&runtime.child_roster.containment).await;
8660 return;
8661 }
8662 if child.is_some() || !respawn_still_pending(&snapshot) {
8663 pending_respawn = None;
8664 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8665 state.respawn_pending = false
8666 });
8667 }
8668 continue;
8669 }
8670 if child.is_some() {
8671 runtime.recovery_is_daemon = true;
8672 #[cfg(test)]
8673 {
8674 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8675 }
8676 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
8677 let active = child.as_mut().expect("child checked above");
8678 if active.registered_since_spawn() {
8679 if let Some(delivery) = &runtime.config_delivery {
8680 delivery.queue_registered(
8681 &spec.module_id,
8682 runtime.forwarding.as_ref(),
8683 ®istry,
8684 active,
8685 );
8686 }
8687 }
8688 if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8689 active.registration_deadline = None;
8690 }
8691 let registration_deadline = active.registration_deadline;
8692 let no_checks = runtime.health.http.is_none()
8693 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8694 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8695 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8696 state
8697 .failed_retry
8698 .uptime_reset_at(no_checks, runtime.restart_policy.window)
8699 });
8700 let wake_at = match (health_probe.wake_at(), uptime_reset) {
8701 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8702 (probe, None) => probe,
8703 };
8704 let wake_at = match (wake_at, registration_deadline) {
8705 (Some(wake), Some(register)) => Some(wake.min(register)),
8706 (wake, register) => wake.or(register),
8707 };
8708 #[cfg(test)]
8709 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8710 let probe_sleep = supervisor_sleep(wake_at);
8711 tokio::pin!(probe_sleep);
8712 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8717 #[cfg(test)]
8718 {
8719 let registration_pending = wire_child
8720 && registration_changes
8721 .as_ref()
8722 .is_some_and(|changes| changes.has_changed().unwrap());
8723 let mut state = lock_snapshot(&snapshot).unwrap();
8724 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8725 generation: state.spawn_generation,
8726 turn: state.actor_turns,
8727 registered_connection: health_probe.registered_connection,
8728 next_probe_at: health_probe.next_probe_at,
8729 wake_after,
8730 });
8731 }
8732 let registration = active
8733 .awaiting_registration()
8734 .then(|| active.registration_spawn.clone())
8735 .flatten();
8736 #[cfg(all(test, target_os = "linux"))]
8737 let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8738 &spec.module_id,
8739 "observe_registration_before_exit",
8740 Some(active.pid),
8741 ) && registration.is_some();
8742 #[cfg(not(all(test, target_os = "linux")))]
8743 let registration_before_exit = false;
8744 #[cfg(all(test, target_os = "linux"))]
8745 let registration_child_is_running = |child: &mut SupervisedChild| {
8746 registration_before_exit || registration_child_is_running(child)
8749 };
8750 let registration_failed = async {
8751 match ®istration {
8752 Some(spawn) => spawn.wait_registration_failure().await,
8753 None => std::future::pending().await,
8754 }
8755 };
8756 let active_child = child.as_mut().expect("child checked above");
8757 tokio::select! {
8758 _ = registration_failed => {
8759 if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8760 active_child.registration_deadline = None;
8761 active_child.spawn_failure = Some("registration_channel_closed".into());
8762 if let Err(error) = stop_unregistered_child(active_child).await {
8763 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8764 &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8765 &runtime.failure_context(true));
8766 child = None;
8767 }
8768 }
8769 continue;
8770 }
8771 wait_result = active_child.wait(), if !registration_before_exit => {
8772 if let Ok(mut state) = lock_snapshot(&snapshot) {
8773 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8774 }
8775 #[cfg(test)]
8776 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8777 let exit_report = match wait_result {
8786 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8787 Err(err) => {
8788 active_child.drain_stderr(&spec.module_id).await;
8789 fail_after_wait_error(
8790 &spec.module_id,
8791 &snapshot,
8792 &runtime.terminal_ring,
8793 &runtime.spawn_events,
8794 format!("failed to wait for supervised module: {err}"),
8795 &runtime.failure_context(true),
8796 );
8797 untrack_if_registration_released(
8798 &process_liveness,
8799 ®istry,
8800 &spec.module_id,
8801 &snapshot,
8802 );
8803 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8804 child = None;
8805 continue;
8806 }
8807 };
8808 active_child.drain_stderr(&spec.module_id).await;
8809
8810 let delivery=runtime.config_delivery.clone();
8811 let exited_spawn=active_child.spawned_at;
8812 let spawn=active_child.config_delivery.clone();
8813 let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8814 let module_id=spec.module_id.clone();
8815 let observed=async move {
8819 if let Some(delivery)=delivery {
8820 delivery.0.on_blocking(move |owner| {
8821 if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8822 if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8823 Ok(())
8824 }).await
8825 } else {Ok(())}
8826 };
8827 tokio::pin!(observed);
8828 loop {
8829 tokio::select! {
8830 result=&mut observed=>{
8831 if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8832 break;
8833 }
8834 command=commands.recv()=>{
8835 let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8836 let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8837 let command=match command {
8838 SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8839 SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8840 command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8841 command=>command,
8842 };
8843 if !handle_supervisor_command(command,&mut spec,&mut runtime,®istry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8844 containment::terminate(&runtime.child_roster.containment).await;return;
8845 }
8846 if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8847 }
8848 }
8849 }
8850 let active_child=child.as_mut().expect("exit observation retained its child");
8851 let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8855 let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8856 let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8857 let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8858 let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8859 && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8860 if refused_program { active_child.admission_base.as_ref() } else { None })
8861 .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8862 let next = if deferred_spawn {
8863 record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8864 let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8865 state.state=ModuleState::Restarting;
8866 clear_current_process_facts(state);
8867 state.last_exit=Some(exit_report.clone());
8868 });
8869 NextAction::Restart {schedule:None}
8870 } else if parked {
8871 record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8872 NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8873 } else { on_child_exit(
8874 &spec,
8875 runtime.restart_policy,
8876 ®istry,
8877 &snapshot,
8878 &runtime.terminal_ring,
8879 &runtime.spawn_events,
8880 &runtime.child_roster,
8881 exit_report,
8882 true,
8883 ).await };
8884 runtime.start_requester = None;
8885 runtime.pending_activity_restart = false;
8886 let carry = matches!(next, NextAction::Restart { .. })
8889 && active_child.spawn_failure.is_none()
8890 && !runtime.child_roster.is_closed();
8891 if let Err(error) = active_child.finish_exit(carry).await {
8892 containment::terminate(&runtime.child_roster.containment).await;
8893 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8894 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8895 &runtime.failure_context(true));
8896 child = None;
8897 continue;
8898 }
8899 match next {
8900 NextAction::Stop { registration_released } => {
8901 if registration_released {
8902 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8903 }
8904 child = None;
8905 }
8906 NextAction::Restart { schedule } => {
8907 let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8908 runtime.restart_policy.delay_for_restart(0),
8909 |schedule| schedule.delay,
8910 )};
8911 if let Some(schedule) = schedule {
8912 log_crash_respawn(&spec.module_id, schedule);
8913 }
8914 child = None;
8922 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8923 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8924 }
8925 }
8926 }
8927 command = commands.recv() => {
8928 #[cfg(test)]
8929 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8930 let Some(command) = command else {
8931 containment::terminate(&runtime.child_roster.containment).await;
8932 return;
8933 };
8934 if !handle_supervisor_command(
8935 command,
8936 &mut spec,
8937 &mut runtime,
8938 ®istry,
8939 &process_liveness,
8940 &snapshot,
8941 &mut child,
8942 &mut commands,
8943 &mut requeued,
8944 ).await {
8945 containment::terminate(&runtime.child_roster.containment).await;
8946 return;
8947 }
8948 }
8949 _ = async {
8950 match registration_changes.as_mut() {
8951 Some(changes) => { let _ = changes.changed().await; }
8952 None => std::future::pending().await,
8953 }
8954 }, if wire_child => {
8955 #[cfg(test)]
8956 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8957 }
8958 _ = &mut probe_sleep => {
8959 #[cfg(test)]
8960 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8961 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
8962 active_child.registration_deadline = None;
8965 if !runtime.child_roster.is_closed()
8966 && !active_child.registered_since_spawn()
8967 && registration_child_is_running(active_child)
8968 {
8969 warn!(module_id = %spec.module_id, pid = active_child.pid,
8970 wait_secs = runtime.register_timeout.as_secs(),
8971 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
8972 active_child.spawn_failure = Some("registration_timeout".into());
8973 if let Err(error) = stop_unregistered_child(active_child).await {
8974 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
8975 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8976 &runtime.spawn_events, format!("registration_timeout: {error}"),
8977 &runtime.failure_context(true));
8978 child = None;
8979 }
8980 continue;
8983 }
8984 }
8985 if health_probe.due() {
8986 run_health_probe_cycle(
8987 &spec,
8988 &runtime,
8989 ®istry,
8990 &process_liveness,
8991 &snapshot,
8992 &mut child,
8993 ).await;
8994 if child.is_none() {
8995 runtime.start_requester = None;
8996 runtime.pending_activity_restart = false;
8997 }
8998 if child.is_some() {
8999 health_probe.schedule_next(&spec, runtime.health.cadence);
9000 }
9001 }
9002 }
9003 }
9004 } else if let Some(pending) = pending_respawn.clone() {
9005 tokio::select! {
9006 _ = runtime.child_roster.closed() => {
9007 pending_respawn = None;
9008 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9009 state.state = ModuleState::Stopped;
9010 state.respawn_pending = false;
9011 });
9012 containment::terminate(&runtime.child_roster.containment).await;
9013 }
9014 _ = async {
9015 sleep_until(pending.deadline).await;
9016 if let Some(delivery) = &runtime.config_delivery {
9017 delivery.unfenced(&spec.module_id).await;
9018 }
9019 } => {
9020 pending_respawn = None;
9021 let failure = pending.failure_context(&runtime);
9022 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9023 if !respawn_still_pending(&snapshot) {
9027 continue;
9028 }
9029 if runtime.child_roster.is_closed() {
9034 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9035 state.state = ModuleState::Stopped;
9036 });
9037 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
9038 continue;
9039 }
9040 if let Err(err) = release_dead_registration(
9041 ®istry,
9042 runtime.forwarding.as_deref(),
9043 &snapshot,
9044 &spec.module_id,
9045 &failure,
9046 ).await {
9047 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9048 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
9049 continue;
9050 }
9051
9052 if matches!(pending.kind, RespawnKind::Reload) {
9053 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
9054 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
9055 if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
9056 *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
9057 continue;
9058 }
9059 if result.is_ok() && runtime.pending_activity_restart {
9060 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9061 }
9062 runtime.pending_activity_restart = false;
9063 if let Some(reply) = reply { let _ = reply.send(result); }
9064 continue;
9065 }
9066 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9067 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9068 Ok(next_child) => {
9069 child = Some(next_child);
9070 if runtime.pending_activity_restart {
9071 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9072 runtime.pending_activity_restart = false;
9073 }
9074 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
9075 }
9076 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9077 pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9078 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9079 }
9080 Err(SuperviseError::ConfigHeld { .. }) => {
9081 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9082 runtime.pending_activity_restart = false;
9083 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9084 }
9085 Err(err) => {
9086 fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9087 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9088 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9089 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9090 }
9091 }
9092 }
9093 command = commands.recv() => {
9094 let Some(command) = command else {
9095 containment::terminate(&runtime.child_roster.containment).await;
9096 return;
9097 };
9098 if !handle_supervisor_command(
9099 command,
9100 &mut spec,
9101 &mut runtime,
9102 ®istry,
9103 &process_liveness,
9104 &snapshot,
9105 &mut child,
9106 &mut commands,
9107 &mut requeued,
9108 ).await {
9109 containment::terminate(&runtime.child_roster.containment).await;
9110 return;
9111 }
9112 if child.is_some() || !respawn_still_pending(&snapshot) {
9117 pending_respawn = None;
9118 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9119 }
9120 }
9121 }
9122 } else {
9123 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9124 (state.enabled && state.state == ModuleState::Failed)
9125 .then_some(state.failed_retry.pending)
9126 .flatten()
9127 });
9128 let command = if lock_snapshot(&snapshot)
9129 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9130 {
9131 tokio::select! {
9132 command = commands.recv() => command,
9133 _ = previous_instance_recheck.tick() => {
9134 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9135 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9136 if enabled && !runtime.child_roster.is_closed() {
9137 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9138 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9139 Ok(next_child) => child = Some(next_child),
9140 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9141 let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9142 }
9143 Err(SuperviseError::ConfigHeld { .. }) => {
9144 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9145 }
9146 Err(error) => {
9147 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9148 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9149 }
9150 }
9151 } else {
9152 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9153 state.waiting_for_previous_pid = None;
9154 state.waiting_for_previous_detail = None;
9155 });
9156 }
9157 }
9158 continue;
9159 }
9160 }
9161 } else {
9162 tokio::select! {
9163 biased;
9164 command = commands.recv() => command,
9165 _ = async {
9166 match slow_retry {
9167 Some(retry) => sleep_until(retry.deadline).await,
9168 None => std::future::pending().await,
9169 }
9170 } => {
9171 let retry = slow_retry.expect("only a scheduled slow retry can wake");
9172 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9173 state.failed_retry.pending = None;
9174 });
9175 if runtime.child_roster.is_closed() {
9176 continue;
9177 }
9178 info!(module_id = %spec.module_id, attempt = retry.attempt,
9179 delay_secs = retry.delay.as_secs(), "starting failed module retry");
9180 if let Err(error) = set_child_enabled(
9181 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
9182 &runtime.failure_context(true),
9183 ).await {
9184 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9185 }
9186 continue;
9187 }
9188 }
9189 };
9190 let Some(command) = command else {
9191 containment::terminate(&runtime.child_roster.containment).await;
9192 return;
9193 };
9194 if !handle_supervisor_command(
9195 command,
9196 &mut spec,
9197 &mut runtime,
9198 ®istry,
9199 &process_liveness,
9200 &snapshot,
9201 &mut child,
9202 &mut commands,
9203 &mut requeued,
9204 )
9205 .await
9206 {
9207 containment::terminate(&runtime.child_roster.containment).await;
9208 return;
9209 }
9210 }
9211 }
9212}
9213
9214async fn supervisor_sleep(wake_at: Option<Instant>) {
9217 match wake_at {
9218 Some(deadline) => sleep_until(deadline).await,
9219 None => std::future::pending().await,
9220 }
9221}
9222
9223fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9224 info!(
9225 module_id,
9226 restart_in_window = schedule.restart_in_window,
9227 delay_ms = schedule.delay.as_millis() as u64,
9228 "respawning after crash"
9229 );
9230}
9231
9232fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9238 matches!(
9239 lock_snapshot(snapshot),
9240 Ok(state) if state.enabled && state.state == ModuleState::Restarting
9241 )
9242}
9243
9244enum NextAction {
9245 Stop {
9246 registration_released: bool,
9247 },
9248 Restart {
9249 schedule: Option<CrashRestartSchedule>,
9250 },
9251}
9252
9253#[allow(clippy::too_many_arguments)]
9254async fn handle_supervisor_command(
9255 command: SupervisorCommand,
9256 spec: &mut ModuleSpec,
9257 runtime: &mut SupervisorRuntimeConfig,
9258 registry: &Arc<Registry>,
9259 process_liveness: &SupervisorProcessLiveness,
9260 snapshot: &SharedSnapshot,
9261 child: &mut Option<SupervisedChild>,
9262 commands: &mut mpsc::Receiver<SupervisorCommand>,
9263 requeued: &mut VecDeque<SupervisorCommand>,
9264) -> bool {
9265 if let SupervisorCommand::SetEnabled {
9266 enabled: true,
9267 reset_budget: false,
9268 reply,
9269 ..
9270 } = command
9271 {
9272 let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9273 (state.config_hold_parked
9274 && state.enabled
9275 && !state.process_alive
9276 && child.is_none()
9277 && state.state == ModuleState::Stopped)
9278 .then(|| {
9279 (
9280 state.config_hold_daemon_initiated,
9281 state.config_hold_approval.clone(),
9282 )
9283 })
9284 });
9285 let result = if let Some((daemon_initiated, approval)) = parked {
9289 runtime.recovery_is_daemon = daemon_initiated;
9290 let failure = FailureContext {
9291 approval,
9292 ..runtime.failure_context(daemon_initiated)
9293 };
9294 let result = set_child_enabled_with_budget_policy(
9295 spec,
9296 runtime,
9297 registry,
9298 process_liveness,
9299 snapshot,
9300 child,
9301 true,
9302 &failure,
9303 false,
9304 )
9305 .await;
9306 if let Err(error) = &result {
9307 record_approved_attempt_failure(spec, snapshot, &failure, error);
9308 }
9309 result
9310 } else {
9311 Ok(false)
9312 };
9313 let _ = reply.send(result);
9314 return true;
9315 }
9316 let blocked = runtime
9317 .config_delivery
9318 .as_ref()
9319 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9320 .filter(|error| error.code != "config_delivery_deferred")
9321 .map(|error| {
9322 config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9323 })
9324 .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9325 .or_else(|| {
9326 runtime
9327 .child_roster
9328 .previous_instance(&spec.module_id)
9329 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9330 });
9331 let command = if let Some(error) = blocked {
9332 match command {
9333 SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9334 let _ = reply.send(Err(error));
9335 return true;
9336 }
9337 SupervisorCommand::Swap { reply, .. } => {
9338 let _ = reply.send(Err(error));
9339 return true;
9340 }
9341 SupervisorCommand::ConfigApplyRestart { .. } => return true,
9342 SupervisorCommand::SetEnabled {
9343 enabled: true,
9344 requester,
9345 reply,
9346 ..
9347 } => {
9348 if matches!(error, SuperviseError::ConfigHeld { .. })
9349 && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9350 && runtime
9351 .config_delivery
9352 .as_ref()
9353 .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9354 {
9355 let mut changed = false;
9359 let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9360 changed = !state.enabled;
9361 state.enabled = true;
9362 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9363 state.state = ModuleState::Starting;
9364 state.failed_retry.cancel();
9365 }
9366 })
9367 .and_then(|_| {
9368 if child.is_none()
9369 && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9370 {
9371 hold_config_spawn(
9372 spec,
9373 runtime,
9374 snapshot,
9375 &runtime.failure_context(false),
9376 )?;
9377 }
9378 Ok(changed)
9379 });
9380 if result.is_ok() && changed {
9381 runtime.start_requester = requester;
9382 }
9383 let _ = reply.send(result);
9384 return true;
9385 }
9386 let _ = reply.send(Err(error));
9387 return true;
9388 }
9389 other => other,
9390 }
9391 } else {
9392 command
9393 };
9394 if matches!(
9397 command,
9398 SupervisorCommand::Drain { .. }
9399 | SupervisorCommand::Retire { .. }
9400 | SupervisorCommand::SetEnabled { .. }
9401 | SupervisorCommand::Restart { .. }
9402 ) {
9403 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9404 state.failed_retry.cancel();
9405 });
9406 }
9407 match command {
9408 SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9409 let result = async {
9410 let next = approved.module_entry(&spec.module_id).map_err(|error| {
9411 SuperviseError::InvalidSpec { reason: error.to_string() }
9412 })?;
9413 let retry_approved_failure = {
9414 let mut state = lock_snapshot(snapshot)?;
9415 if let Some((generation, detail)) = &state.approved_start_failure {
9416 if *generation == approved.identity() && next.enabled {
9417 return Err(SuperviseError::Spawn {
9418 program: spec.program.clone(),
9419 source: std::io::Error::other(detail.clone()),
9420 cgroup_path: None,
9421 });
9422 }
9423 }
9424 let retry = state.approved_start_failure.is_some();
9425 state.approved_start_failure = None;
9426 retry
9427 };
9428 stage_supervisor_configuration(
9429 spec, runtime, snapshot, next.module_spec(), next.health,
9430 next.drain_timeout_ms,
9431 );
9432 runtime.restart_policy = next.restart;
9433 runtime.register_timeout = next.register_timeout;
9434 if let Some(active) = child.as_mut()
9435 .filter(|child| child.registration_deadline.is_some())
9436 {
9437 active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9438 }
9439 let enabled = lock_snapshot(snapshot)?.enabled;
9440 let retry_gate_stopped = lock_snapshot(snapshot)?
9447 .needs_approved_gate_restart(&approved.identity());
9448 let retry_failed = enabled && child.is_none()
9449 && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9450 && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9451 || retry_gate_stopped);
9452 if retry_failed {
9453 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9456 state.failed_retry.cancel();
9457 if retry_gate_stopped {
9458 state.stop_reason = None;
9459 state.config_exit_failed_approval = None;
9460 }
9461 })?;
9462 }
9463 if enabled != next.enabled || retry_failed {
9464 let failure = FailureContext {
9465 approval: next.enabled.then(|| approved.identity()),
9466 ..runtime.failure_context(false)
9467 };
9468 let held = next.enabled && runtime.config_delivery.as_ref()
9469 .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9470 || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9471 if held {
9472 reset_restart_count(snapshot, &spec.module_id)?;
9475 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9476 state.enabled = true;
9477 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9478 state.state = ModuleState::Starting;
9479 }
9480 })?;
9481 if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9482 hold_config_spawn(spec, runtime, snapshot, &failure)?;
9483 }
9484 } else {
9485 match set_child_enabled_with_budget_policy(
9486 spec, runtime, registry, process_liveness, snapshot, child,
9487 next.enabled, &failure, true,
9488 ).await {
9489 Ok(_) => {},
9490 Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9491 Err(error) => return Err(error),
9492 }
9493 if !next.enabled {
9494 containment::terminate(&runtime.child_roster.containment).await;
9495 }
9496 }
9497 }
9498 if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9499 || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9500 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9501 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9502 {
9503 let failure = FailureContext {
9504 approval: Some(approved.identity()),
9505 ..runtime.failure_context(false)
9506 };
9507 if child.is_some() {
9511 restart_child_with_budget_policy(spec, runtime, registry,
9512 process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9513 }
9514 update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9517 match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9518 Ok(replacement) => *child = Some(replacement),
9519 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9520 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9521 }
9522 Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9523 Err(error) => {
9524 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9525 return Err(error);
9526 }
9527 }
9528 }
9529 info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9530 Ok(())
9531 }
9532 .await;
9533 if let Err(error) = &result {
9534 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9535 if state.enabled
9536 && state.state == ModuleState::Failed
9537 && state
9538 .approved_start_failure
9539 .as_ref()
9540 .is_none_or(|(generation, _)| *generation != approved.identity())
9541 {
9542 state.approved_start_failure =
9543 Some((approved.identity(), error.to_string()));
9544 }
9545 });
9546 }
9547 let _ = reply.send(result);
9548 true
9549 }
9550 SupervisorCommand::ConfigPolicyProbe {
9551 connection,
9552 generation,
9553 attempt,
9554 } => {
9555 let Some(delivery) = &runtime.config_delivery else {
9556 return true;
9557 };
9558 let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9559 return true;
9560 };
9561 let guard = gate.lock().await;
9562 let pending = delivery
9563 .0
9564 .on_blocking(move |owner| {
9565 owner.policy_probe_pending(connection, generation, attempt)
9566 })
9567 .await
9568 .unwrap_or(false)
9569 && child.is_some()
9570 && registry
9571 .get_module(&spec.module_id)
9572 .ok()
9573 .flatten()
9574 .is_some_and(|registration| registration.connection_id == connection);
9575 drop(guard);
9576 if pending {
9577 run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9578 .await;
9579 }
9580 true
9581 }
9582 SupervisorCommand::ConfigApplyRestart {
9583 generation,
9584 supersedes_swap,
9585 } => {
9586 let Some(owner) = runtime
9587 .config_delivery
9588 .as_ref()
9589 .map(|delivery| delivery.0.clone())
9590 else {
9591 return true;
9592 };
9593 let Ok(gate) = owner.module_gate(&spec.module_id) else {
9594 return true;
9595 };
9596 let _guard = gate.lock().await;
9597 let spawn = child
9598 .as_ref()
9599 .and_then(|child| child.config_delivery.clone());
9600 let module = spec.module_id.clone();
9601 let current = owner
9602 .on_blocking(move |owner| {
9603 Ok((supersedes_swap
9604 || owner
9605 .pending_config_restarts()?
9606 .contains(&(module.clone(), generation)))
9607 && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9608 == Some(generation))
9609 })
9610 .await
9611 .unwrap_or(false);
9612 if !current
9613 || !lock_snapshot(snapshot).is_ok_and(|state| {
9614 state.enabled && !state.respawn_pending && state.process_alive
9615 })
9616 {
9617 return true;
9618 }
9619 drop(_guard);
9620 info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9621 if let Err(error) = restart_child_with_budget_policy(
9622 spec,
9623 runtime,
9624 registry,
9625 process_liveness,
9626 snapshot,
9627 child,
9628 runtime.drain_timeout,
9629 false,
9630 )
9631 .await
9632 {
9633 warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9634 }
9635 true
9636 }
9637 SupervisorCommand::ConfigExitHold { reply } => {
9638 let idle_start = child.is_none()
9642 && lock_snapshot(snapshot).is_ok_and(|state| {
9643 state.enabled
9644 && state.state == ModuleState::Starting
9645 && !state.process_alive
9646 && !state.respawn_pending
9647 && state.waiting_for_previous_pid.is_none()
9648 && state.waiting_for_previous_detail.is_none()
9649 && state.failed_retry.pending.is_none()
9650 })
9651 && runtime
9652 .scheduled_respawn
9653 .lock()
9654 .unwrap_or_else(|p| p.into_inner())
9655 .is_none()
9656 && runtime
9657 .deferred_reload_reply
9658 .lock()
9659 .unwrap_or_else(|p| p.into_inner())
9660 .is_none();
9661 let result = if idle_start {
9662 try_park_config_attempt(
9663 spec,
9664 runtime,
9665 snapshot,
9666 &runtime.failure_context(runtime.recovery_is_daemon),
9667 None,
9668 )
9669 .map(|_| ())
9670 } else {
9671 Ok(())
9672 };
9673 let _ = reply.send(result);
9674 true
9675 }
9676 SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9677 if registry
9678 .get_module(&spec.module_id)
9679 .ok()
9680 .flatten()
9681 .is_none_or(|registration| registration.connection_id != connection)
9682 {
9683 return true;
9684 }
9685 runtime.recovery_is_daemon = true;
9686 if hold {
9690 let notice = begin_forwarding_drain_if_configured(
9691 spec,
9692 runtime,
9693 registry,
9694 snapshot,
9695 Some(true),
9696 RouteCloseReason::Disable,
9697 )
9698 .await;
9699 if let Ok(notice) = notice {
9700 let drained = drain_optional_child(
9701 &spec.module_id,
9702 spec.protocol,
9703 notice,
9704 registry,
9705 runtime.forwarding.as_deref(),
9706 snapshot,
9707 &runtime.terminal_ring,
9708 &runtime.spawn_events,
9709 child,
9710 runtime.drain_timeout,
9711 ModuleState::Failed,
9712 Some(true),
9713 &runtime.failure_context(true),
9714 )
9715 .await;
9716 if drained.is_ok() {
9717 match try_park_config_attempt(
9718 spec,
9719 runtime,
9720 snapshot,
9721 &runtime.failure_context(true),
9722 None,
9723 ) {
9724 Ok(true) => {}
9725 Ok(false) => {
9726 if let Err(error) = budgeted_restart_child(
9727 spec,
9728 runtime,
9729 registry,
9730 process_liveness,
9731 snapshot,
9732 child,
9733 SupervisorHealthStatus::Unknown,
9734 Some("configuration delivery failed after its hold ended"),
9735 unix_ms_now(),
9736 false,
9737 )
9738 .await
9739 {
9740 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9741 }
9742 }
9743 Err(error) => {
9744 warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9745 }
9746 }
9747 }
9748 }
9749 } else if let Err(error) = budgeted_restart_child(
9750 spec,
9751 runtime,
9752 registry,
9753 process_liveness,
9754 snapshot,
9755 child,
9756 SupervisorHealthStatus::Unknown,
9757 Some("configuration delivery failed"),
9758 unix_ms_now(),
9759 false,
9760 )
9761 .await
9762 {
9763 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9764 }
9765 true
9766 }
9767 SupervisorCommand::Drain { requester, reply } => {
9768 let result = drain_optional_child(
9771 &spec.module_id,
9772 spec.protocol,
9773 StopNotice::NotSent,
9774 registry,
9775 runtime.forwarding.as_deref(),
9776 snapshot,
9777 &runtime.terminal_ring,
9778 &runtime.spawn_events,
9779 child,
9780 runtime.drain_timeout,
9781 ModuleState::Stopped,
9782 None,
9783 &FailureContext::operator(),
9784 )
9785 .await;
9786 containment::terminate(&runtime.child_roster.containment).await;
9787 let registration_released = result.is_ok();
9788 if registration_released {
9789 runtime.record_activity_transition(
9790 "subc.module_stopped",
9791 &spec.module_id,
9792 requester,
9793 );
9794 }
9795 let _ = reply.send(result);
9796 if registration_released {
9797 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9798 }
9799 false
9800 }
9801 SupervisorCommand::Retire { requester, reply } => {
9802 let result = async {
9803 let stop_notice = begin_forwarding_drain_if_configured(
9804 spec,
9805 runtime,
9806 registry,
9807 snapshot,
9808 None,
9809 RouteCloseReason::Disable,
9810 )
9811 .await?;
9812 drain_optional_child(
9813 &spec.module_id,
9814 spec.protocol,
9815 stop_notice,
9816 registry,
9817 runtime.forwarding.as_deref(),
9818 snapshot,
9819 &runtime.terminal_ring,
9820 &runtime.spawn_events,
9821 child,
9822 runtime.drain_timeout,
9823 ModuleState::Stopped,
9824 None,
9825 &FailureContext::operator(),
9826 )
9827 .await
9828 }
9829 .await;
9830 containment::terminate(&runtime.child_roster.containment).await;
9831 let registration_released = result.is_ok();
9832 if registration_released {
9833 runtime.record_activity_transition(
9834 "subc.module_stopped",
9835 &spec.module_id,
9836 requester,
9837 );
9838 }
9839 let _ = reply.send(result);
9840 if registration_released {
9841 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9842 }
9843 false
9844 }
9845 SupervisorCommand::Restart {
9846 requester,
9847 drain_timeout_ms,
9848 received_at_generation,
9849 queued_at,
9850 reply,
9851 } => {
9852 info!(
9856 module_id = %spec.module_id,
9857 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9858 "restart command dequeued"
9859 );
9860 let validation = match lock_snapshot(snapshot) {
9872 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9873 module_id: spec.module_id.clone(),
9874 }),
9875 Ok(_) => Ok(()),
9876 Err(err) => Err(err),
9877 };
9878 let initiated = validation.is_ok();
9879 let _ = reply.send(validation);
9880 let satisfied_by_generation = if initiated
9894 && child
9895 .as_ref()
9896 .is_some_and(|child| child.spawned_at > queued_at)
9897 {
9898 lock_snapshot(snapshot).ok().and_then(|state| {
9899 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9900 })
9901 } else {
9902 None
9903 };
9904 let satisfied_by_pending = initiated
9905 && child.is_none()
9906 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9907 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9911 if pending {
9912 state.coalesced_restart_pending = true;
9913 }
9914 pending
9915 });
9916 if satisfied_by_pending {
9917 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9918 } else if let Some(generation) = satisfied_by_generation {
9919 info!(
9920 module_id = %spec.module_id,
9921 received_at_generation,
9922 "restart already satisfied by generation {generation}; not restarting again"
9923 );
9924 } else if initiated {
9925 let drain_timeout = drain_timeout_ms
9928 .map(Duration::from_millis)
9929 .unwrap_or(runtime.drain_timeout);
9930 if let Err(err) = restart_child(
9931 spec,
9932 runtime,
9933 registry,
9934 process_liveness,
9935 snapshot,
9936 child,
9937 drain_timeout,
9938 )
9939 .await
9940 {
9941 warn!(
9942 module_id = %spec.module_id,
9943 error = %err,
9944 "operator restart failed after initiation ack; module state carries the outcome"
9945 );
9946 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9947 state.state = ModuleState::Failed;
9948 clear_current_process_facts(state);
9949 });
9950 } else {
9951 runtime.start_requester = requester;
9952 runtime.pending_activity_restart = true;
9953 }
9954 }
9955 true
9956 }
9957 SupervisorCommand::Reload { requester, reply } => {
9958 let result =
9959 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
9960 if result.is_ok() {
9961 runtime.start_requester = requester;
9962 runtime.pending_activity_restart = true;
9963 }
9964 if result.is_ok()
9965 && runtime
9966 .scheduled_respawn
9967 .lock()
9968 .unwrap_or_else(|p| p.into_inner())
9969 .as_ref()
9970 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
9971 {
9972 *runtime
9973 .deferred_reload_reply
9974 .lock()
9975 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
9976 } else {
9977 let _ = reply.send(result);
9978 }
9979 true
9980 }
9981 SupervisorCommand::SetEnabled {
9982 enabled,
9983 requester,
9984 reply,
9985 reset_budget,
9986 } => {
9987 let previous_requester = runtime.start_requester.clone();
9988 if enabled {
9989 runtime.start_requester = requester.clone();
9990 }
9991 let result = set_child_enabled_with_budget_policy(
9992 spec,
9993 runtime,
9994 registry,
9995 process_liveness,
9996 snapshot,
9997 child,
9998 enabled,
9999 &runtime.failure_context(false),
10000 reset_budget,
10001 )
10002 .await;
10003 if !enabled {
10004 containment::terminate(&runtime.child_roster.containment).await;
10005 }
10006 if matches!(result, Ok(true)) {
10007 runtime.pending_activity_restart = false;
10008 if enabled {
10009 runtime.start_requester = requester;
10010 } else {
10011 runtime.start_requester = None;
10012 runtime.record_activity_transition(
10013 "subc.module_disabled",
10014 &spec.module_id,
10015 requester,
10016 );
10017 }
10018 } else {
10019 runtime.start_requester = previous_requester;
10020 }
10021 let _ = reply.send(result);
10022 true
10023 }
10024 SupervisorCommand::UpdateConfiguration {
10025 spec: next_spec,
10026 health,
10027 drain_timeout_ms,
10028 reply,
10029 } => {
10030 if runtime
10031 .config_delivery
10032 .as_ref()
10033 .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
10034 {
10035 let _ = reply.send(Err(SuperviseError::ConfigHeld {
10036 module_id: spec.module_id.clone(),
10037 code: "config_gated".into(),
10038 detail: "live file reconfiguration cannot replace stamped authority".into(),
10039 }));
10040 return true;
10041 }
10042 stage_supervisor_configuration(
10043 spec,
10044 runtime,
10045 snapshot,
10046 next_spec,
10047 health,
10048 drain_timeout_ms,
10049 );
10050 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10051 state.configuration_updated_since_spawn = true;
10052 });
10053 let _ = reply.send(Ok(()));
10054 true
10055 }
10056 SupervisorCommand::Swap {
10057 requester,
10058 ready_timeout,
10059 reply,
10060 } => {
10061 if spec.carry_containment
10062 || containment::current(&runtime.child_roster.containment).is_some()
10063 {
10064 let _ = reply.send(Err(SuperviseError::InvalidSpec {
10065 reason: "a carried container cannot overlap a swap candidate".to_owned(),
10066 }));
10067 return true;
10068 }
10069 let mut swap_runtime = runtime.clone();
10070 swap_runtime.start_requester = requester;
10071 let before = lock_snapshot(snapshot)
10072 .ok()
10073 .map(|state| state.spawn_generation);
10074 let end = swap::run_swap(
10075 spec,
10076 &swap_runtime,
10077 registry,
10078 process_liveness,
10079 snapshot,
10080 child,
10081 commands,
10082 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10083 reply,
10084 )
10085 .await;
10086 if before
10087 != lock_snapshot(snapshot)
10088 .ok()
10089 .map(|state| state.spawn_generation)
10090 {
10091 runtime.start_requester = swap_runtime.start_requester;
10092 }
10093 requeued.extend(end.requeue);
10094 true
10095 }
10096 }
10097}
10098
10099fn stage_supervisor_configuration(
10100 spec: &mut ModuleSpec,
10101 runtime: &mut SupervisorRuntimeConfig,
10102 snapshot: &SharedSnapshot,
10103 next_spec: ModuleSpec,
10104 health: HealthConfig,
10105 drain_timeout_ms: Option<u64>,
10106) {
10107 if let Some(handle) = &runtime.supervisor_handle {
10108 handle.apply_identity_configuration(&next_spec);
10109 }
10110 let launch_changed = *spec != next_spec;
10111 *spec = next_spec;
10112 if launch_changed {
10113 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10114 state.configuration_updated_since_spawn = true
10115 });
10116 }
10117 let health_changed = runtime.health != health;
10118 runtime.health = health;
10119 if health_changed {
10122 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10123 state.health = ModuleHealthStatus::default();
10124 state.last_probe_evidence = None;
10125 });
10126 }
10127 runtime.drain_timeout = drain_timeout_ms
10128 .map(Duration::from_millis)
10129 .unwrap_or(runtime.default_drain_timeout);
10130 *runtime
10131 .effective_drain_timeout
10132 .lock()
10133 .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10134}
10135
10136async fn restart_child(
10137 spec: &ModuleSpec,
10138 runtime: &SupervisorRuntimeConfig,
10139 registry: &Registry,
10140 process_liveness: &SupervisorProcessLiveness,
10141 snapshot: &SharedSnapshot,
10142 child: &mut Option<SupervisedChild>,
10143 drain_timeout: Duration,
10144) -> Result<(), SuperviseError> {
10145 restart_child_with_budget_policy(
10146 spec,
10147 runtime,
10148 registry,
10149 process_liveness,
10150 snapshot,
10151 child,
10152 drain_timeout,
10153 true,
10154 )
10155 .await
10156}
10157
10158#[allow(clippy::too_many_arguments)]
10159async fn restart_child_with_budget_policy(
10160 spec: &ModuleSpec,
10161 runtime: &SupervisorRuntimeConfig,
10162 registry: &Registry,
10163 process_liveness: &SupervisorProcessLiveness,
10164 snapshot: &SharedSnapshot,
10165 child: &mut Option<SupervisedChild>,
10166 drain_timeout: Duration,
10167 reset_budget: bool,
10168) -> Result<(), SuperviseError> {
10169 if !lock_snapshot(snapshot)?.enabled {
10171 return Err(SuperviseError::Disabled {
10172 module_id: spec.module_id.clone(),
10173 });
10174 }
10175 let stop_notice = begin_forwarding_drain_with_timeout(
10176 spec,
10177 runtime,
10178 registry,
10179 snapshot,
10180 None,
10181 RouteCloseReason::Restart,
10182 drain_timeout,
10183 )
10184 .await?;
10185
10186 if child.is_some() {
10187 drain_optional_child(
10188 &spec.module_id,
10189 spec.protocol,
10190 stop_notice,
10191 registry,
10192 runtime.forwarding.as_deref(),
10193 snapshot,
10194 &runtime.terminal_ring,
10195 &runtime.spawn_events,
10196 child,
10197 drain_timeout,
10198 ModuleState::Restarting,
10199 Some(true),
10200 &FailureContext::operator(),
10201 )
10202 .await?;
10203 } else {
10204 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10205 state.enabled = true;
10206 state.state = ModuleState::Restarting;
10207 clear_current_process_facts(state);
10208 })?;
10209 release_dead_registration(
10210 registry,
10211 runtime.forwarding.as_deref(),
10212 snapshot,
10213 &spec.module_id,
10214 &runtime.failure_context(false),
10215 )
10216 .await?;
10217 }
10218
10219 if reset_budget {
10220 reset_restart_count(snapshot, &spec.module_id)?;
10221 }
10222 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10223 schedule_respawn(
10224 runtime,
10225 snapshot,
10226 &spec.module_id,
10227 runtime.restart_policy.backoff,
10228 RespawnKind::Spawn,
10229 false,
10230 )
10231}
10232
10233async fn reload_child(
10234 spec: &ModuleSpec,
10235 runtime: &SupervisorRuntimeConfig,
10236 registry: &Registry,
10237 process_liveness: &SupervisorProcessLiveness,
10238 snapshot: &SharedSnapshot,
10239 child: &mut Option<SupervisedChild>,
10240) -> Result<(), SuperviseError> {
10241 if !lock_snapshot(snapshot)?.enabled {
10243 return Err(SuperviseError::Disabled {
10244 module_id: spec.module_id.clone(),
10245 });
10246 }
10247 let stop_notice = begin_forwarding_drain(
10248 spec,
10249 runtime,
10250 registry,
10251 snapshot,
10252 Some(true),
10253 RouteCloseReason::Reload,
10254 )
10255 .await?;
10256
10257 if child.is_some() {
10258 drain_optional_child(
10259 &spec.module_id,
10260 spec.protocol,
10261 stop_notice,
10262 registry,
10263 runtime.forwarding.as_deref(),
10264 snapshot,
10265 &runtime.terminal_ring,
10266 &runtime.spawn_events,
10267 child,
10268 runtime.drain_timeout,
10269 ModuleState::Restarting,
10270 Some(true),
10271 &FailureContext::operator(),
10272 )
10273 .await?;
10274 } else {
10275 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10276 state.enabled = true;
10277 state.state = ModuleState::Restarting;
10278 clear_current_process_facts(state);
10279 })?;
10280 release_dead_registration(
10281 registry,
10282 runtime.forwarding.as_deref(),
10283 snapshot,
10284 &spec.module_id,
10285 &runtime.failure_context(false),
10286 )
10287 .await?;
10288 }
10289
10290 reset_restart_count(snapshot, &spec.module_id)?;
10291 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10292 schedule_respawn(
10293 runtime,
10294 snapshot,
10295 &spec.module_id,
10296 runtime.restart_policy.backoff,
10297 RespawnKind::Reload,
10298 false,
10299 )
10300}
10301
10302async fn finish_reload_child(
10303 spec: &ModuleSpec,
10304 runtime: &SupervisorRuntimeConfig,
10305 registry: &Registry,
10306 process_liveness: &SupervisorProcessLiveness,
10307 snapshot: &SharedSnapshot,
10308 child: &mut Option<SupervisedChild>,
10309) -> Result<(), SuperviseError> {
10310 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10311 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10312 Ok(next_child) => next_child,
10313 Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10314 defer_config_spawn(
10315 spec,
10316 runtime,
10317 snapshot,
10318 RespawnKind::Reload,
10319 &runtime.failure_context(runtime.recovery_is_daemon),
10320 )?;
10321 return Err(error);
10322 }
10323 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10324 let failure = runtime.failure_context(runtime.recovery_is_daemon);
10325 if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10326 return Err(error);
10327 }
10328 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10329 return Err(SuperviseError::ConfigDeliveryDeferred {
10330 module_id: spec.module_id.clone(),
10331 });
10332 }
10333 Err(err) => {
10334 return handle_reload_spawn_failure(
10335 spec,
10336 runtime,
10337 process_liveness,
10338 snapshot,
10339 child,
10340 format!("new child failed to spawn: {err}"),
10341 )
10342 .await;
10343 }
10344 };
10345 *child = Some(next_child);
10346
10347 let wait_outcome = {
10348 let active_child = child.as_mut().expect("new reload child was just stored");
10349 wait_for_registration_after_reload(
10350 registry,
10351 &spec.module_id,
10352 snapshot,
10353 active_child,
10354 REGISTRY_RELEASE_TIMEOUT,
10355 )
10356 .await?
10357 };
10358
10359 match wait_outcome {
10360 RegistrationWaitOutcome::Registered => {
10361 child
10362 .as_mut()
10363 .expect("registered reload child")
10364 .registration_deadline = None;
10365 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10366 Ok(())
10367 }
10368 RegistrationWaitOutcome::Exited(exit_report) => {
10369 if let Some(active_child) = child.as_mut() {
10370 active_child.drain_stderr(&spec.module_id).await;
10371 }
10372 let mut exited_child = child.take().expect("exited reload child is still stored");
10375 #[cfg(test)]
10376 if let Some(gate) = &runtime.test_reload_exit_record_gate {
10377 gate.reached.notify_one();
10378 gate.resume.notified().await;
10379 }
10380 let result = handle_reload_child_registration_failure(
10381 spec,
10382 runtime,
10383 registry,
10384 process_liveness,
10385 snapshot,
10386 child,
10387 ReloadRegistrationFailure {
10388 exit_report: registration_failure_exit_report(exit_report),
10389 reason: exited_child
10390 .spawn_failure
10391 .clone()
10392 .unwrap_or_else(|| "new child exited before registering".to_string()),
10393 },
10394 )
10395 .await;
10396 exited_child
10397 .finish_exit(false)
10398 .await
10399 .map_err(|source| SuperviseError::Wait {
10400 module_id: spec.module_id.clone(),
10401 source,
10402 })?;
10403 result
10404 }
10405 RegistrationWaitOutcome::TimedOut => {
10406 let mut timed_out_child = child
10407 .take()
10408 .expect("timed-out reload child is still running");
10409 timed_out_child
10410 .start_kill()
10411 .map_err(|source| SuperviseError::Kill {
10412 module_id: spec.module_id.clone(),
10413 source,
10414 })?;
10415 let status = timed_out_child
10416 .wait()
10417 .await
10418 .map_err(|source| SuperviseError::Wait {
10419 module_id: spec.module_id.clone(),
10420 source,
10421 })?;
10422 timed_out_child.drain_stderr(&spec.module_id).await;
10423 let result = handle_reload_child_registration_failure(
10424 spec,
10425 runtime,
10426 registry,
10427 process_liveness,
10428 snapshot,
10429 child,
10430 ReloadRegistrationFailure {
10431 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10432 snapshot,
10433 &timed_out_child,
10434 &status,
10435 )),
10436 reason: format!(
10437 "new child did not register within {:?}",
10438 REGISTRY_RELEASE_TIMEOUT
10439 ),
10440 },
10441 )
10442 .await;
10443 timed_out_child
10444 .finish_exit(false)
10445 .await
10446 .map_err(|source| SuperviseError::Wait {
10447 module_id: spec.module_id.clone(),
10448 source,
10449 })?;
10450 result
10451 }
10452 }
10453}
10454
10455#[allow(clippy::too_many_arguments)]
10456async fn set_child_enabled(
10457 spec: &ModuleSpec,
10458 runtime: &SupervisorRuntimeConfig,
10459 registry: &Registry,
10460 process_liveness: &SupervisorProcessLiveness,
10461 snapshot: &SharedSnapshot,
10462 child: &mut Option<SupervisedChild>,
10463 enabled: bool,
10464 failure: &FailureContext,
10465) -> Result<bool, SuperviseError> {
10466 set_child_enabled_with_budget_policy(
10467 spec,
10468 runtime,
10469 registry,
10470 process_liveness,
10471 snapshot,
10472 child,
10473 enabled,
10474 failure,
10475 true,
10476 )
10477 .await
10478}
10479
10480#[allow(clippy::too_many_arguments)]
10481async fn set_child_enabled_with_budget_policy(
10482 spec: &ModuleSpec,
10483 runtime: &SupervisorRuntimeConfig,
10484 registry: &Registry,
10485 process_liveness: &SupervisorProcessLiveness,
10486 snapshot: &SharedSnapshot,
10487 child: &mut Option<SupervisedChild>,
10488 enabled: bool,
10489 failure: &FailureContext,
10490 reset_budget: bool,
10491) -> Result<bool, SuperviseError> {
10492 let (current_enabled, current_state, respawn_pending) = {
10493 let state = lock_snapshot(snapshot)?;
10494 (state.enabled, state.state, state.respawn_pending)
10495 };
10496 let revive_terminal = enabled
10504 && current_enabled
10505 && child.is_none()
10506 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10507 || (current_state == ModuleState::Restarting && !respawn_pending));
10508 if current_enabled == enabled && !revive_terminal {
10509 return Ok(false);
10510 }
10511
10512 if enabled {
10513 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10514 state.enabled = true;
10515 state.state = ModuleState::Starting;
10516 clear_current_process_facts(state);
10517 })?;
10518 #[cfg(test)]
10519 if runtime.test_seed_stale_facts_before_enable_spawn {
10520 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10521 state.process_alive = true;
10522 state.pid = Some(41);
10523 state.spawned_at_ms = Some(42);
10524 state.spawned_from = Some(PathBuf::from("/spawned/module"));
10525 state.spawned_file_identity = Some(SpawnedFileIdentity {
10526 device: 43,
10527 inode: 44,
10528 #[cfg(windows)]
10529 windows_image: None,
10530 });
10531 })?;
10532 }
10533 release_dead_registration(
10534 registry,
10535 runtime.forwarding.as_deref(),
10536 snapshot,
10537 &spec.module_id,
10538 failure,
10539 )
10540 .await?;
10541 if reset_budget {
10542 reset_restart_count(snapshot, &spec.module_id)?;
10543 }
10544 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10545 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10546 Ok(next_child) => next_child,
10547 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10548 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10549 return Ok(true);
10550 }
10551 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10552 if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10553 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10554 return Err(parking_error);
10555 }
10556 return Err(error);
10557 }
10558 Err(err) => {
10559 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10560 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10561 return Err(err);
10562 }
10563 };
10564 *child = Some(next_child);
10565 debug!(module_id = %spec.module_id, "supervised module enabled");
10566 Ok(true)
10567 } else {
10568 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10569 state.config_hold_parked = false;
10570 state.config_hold_daemon_initiated = false;
10571 state.config_hold_approval = None;
10572 })?;
10573 let stop_notice = begin_forwarding_drain_if_configured(
10574 spec,
10575 runtime,
10576 registry,
10577 snapshot,
10578 Some(false),
10579 RouteCloseReason::Disable,
10580 )
10581 .await?;
10582 drain_optional_child(
10583 &spec.module_id,
10584 spec.protocol,
10585 stop_notice,
10586 registry,
10587 runtime.forwarding.as_deref(),
10588 snapshot,
10589 &runtime.terminal_ring,
10590 &runtime.spawn_events,
10591 child,
10592 runtime.drain_timeout,
10593 ModuleState::Disabled,
10594 Some(false),
10595 failure,
10596 )
10597 .await?;
10598 debug!(module_id = %spec.module_id, "supervised module disabled");
10599 Ok(true)
10600 }
10601}
10602
10603#[allow(clippy::too_many_arguments)]
10604async fn on_child_exit(
10605 spec: &ModuleSpec,
10606 policy: RestartPolicy,
10607 registry: &Registry,
10608 snapshot: &SharedSnapshot,
10609 terminal_ring: &Arc<Mutex<TerminalRing>>,
10610 spawn_events: &SpawnEventFeed,
10611 roster: &ChildRoster,
10612 exit_report: ExitReport,
10613 daemon_initiated: bool,
10614) -> NextAction {
10615 if roster.is_closed() {
10621 return on_child_exit_during_daemon_shutdown(
10622 spec,
10623 registry,
10624 snapshot,
10625 terminal_ring,
10626 spawn_events,
10627 exit_report,
10628 )
10629 .await;
10630 }
10631 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10647 && running_protocol(spec, snapshot) == ModuleProtocol::None;
10648 match exit_report.kind {
10649 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10650 info!(
10651 module_id = %spec.module_id,
10652 exit_code = ?exit_report.code,
10653 exit_signal = ?exit_report.signal,
10654 "supervised module exited cleanly"
10655 );
10656 if let Err(err) = publish_after_terminal_record(
10657 &spec.module_id,
10658 || {
10659 record_terminal(
10660 &spec.module_id,
10661 terminal_ring,
10662 spawn_events,
10663 &exit_report,
10664 TerminalDisposition::Stopped,
10665 )
10666 },
10667 || {
10668 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10669 state.state = ModuleState::Stopped;
10670 clear_current_process_facts(state);
10671 state.last_exit = Some(exit_report.clone());
10672 })
10673 },
10674 ) {
10675 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10676 }
10677 let registration_released = match wait_for_registration_release(
10678 registry,
10679 &spec.module_id,
10680 REGISTRY_RELEASE_TIMEOUT,
10681 )
10682 .await
10683 {
10684 Ok(()) => true,
10685 Err(err) => {
10686 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10687 false
10688 }
10689 };
10690 NextAction::Stop {
10691 registration_released,
10692 }
10693 }
10694 ExitKind::Clean | ExitKind::Crash => {
10695 if unrequested_clean_exit_of_protocol_none {
10696 warn!(
10697 module_id = %spec.module_id,
10698 exit_code = ?exit_report.code,
10699 exit_signal = ?exit_report.signal,
10700 "protocol-none module exited cleanly without a stop request; handling it as a crash"
10701 );
10702 } else {
10703 warn!(
10704 module_id = %spec.module_id,
10705 exit_code = ?exit_report.code,
10706 exit_signal = ?exit_report.signal,
10707 "supervised module exited abnormally (crash)"
10708 );
10709 }
10710 let mut restart_schedule = None;
10711 let mut disposition = TerminalDisposition::Disabled;
10712 let mut final_state = ModuleState::Disabled;
10713 let mut disposition_detail = lock_snapshot(snapshot)
10717 .ok()
10718 .and_then(|mut state| state.spawn_failure.take());
10719 let now = Instant::now();
10720 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10721 if state.enabled {
10722 if let Some(schedule) = state.next_crash_restart(&policy, now) {
10723 state.respawn_pending = true;
10729 final_state = ModuleState::Restarting;
10730 restart_schedule = Some(schedule);
10731 disposition = TerminalDisposition::Restarting;
10732 } else {
10733 final_state = ModuleState::Failed;
10734 disposition = TerminalDisposition::Failed;
10735 let budget = policy.budget_exhausted_detail();
10736 disposition_detail =
10737 Some(disposition_detail.take().map_or_else(
10738 || budget.clone(),
10739 |cause| format!("{cause}; {budget}"),
10740 ));
10741 }
10742 } else {
10743 disposition = TerminalDisposition::Disabled;
10744 }
10745 }) {
10746 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10747 return NextAction::Stop {
10748 registration_released: false,
10749 };
10750 }
10751 if disposition == TerminalDisposition::Failed {
10752 error!(
10757 module_id = %spec.module_id,
10758 max_restarts = policy.max_restarts,
10759 window_secs = policy.window.as_secs(),
10760 "module stopped: {}",
10761 policy.budget_exhausted_detail()
10762 );
10763 }
10764 if let Err(err) = publish_after_terminal_record(
10765 &spec.module_id,
10766 || {
10767 record_terminal_with_detail(
10768 &spec.module_id,
10769 terminal_ring,
10770 spawn_events,
10771 &exit_report,
10772 disposition,
10773 disposition_detail,
10774 )
10775 },
10776 || {
10777 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10778 state.state = final_state;
10779 clear_current_process_facts(state);
10780 if final_state == ModuleState::Failed
10781 && daemon_initiated
10782 && !roster.is_closed()
10783 {
10784 state.failed_retry.schedule(&spec.module_id);
10785 }
10786 state.last_exit = Some(exit_report.clone());
10787 })
10788 },
10789 ) {
10790 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10791 }
10792
10793 if let Some(schedule) = restart_schedule {
10794 NextAction::Restart {
10795 schedule: Some(schedule),
10796 }
10797 } else {
10798 let registration_released = match wait_for_registration_release(
10799 registry,
10800 &spec.module_id,
10801 REGISTRY_RELEASE_TIMEOUT,
10802 )
10803 .await
10804 {
10805 Ok(()) => true,
10806 Err(err) => {
10807 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10808 false
10809 }
10810 };
10811 NextAction::Stop {
10812 registration_released,
10813 }
10814 }
10815 }
10816 ExitKind::DeliberateSeverance => {
10817 warn!(
10818 module_id = %spec.module_id,
10819 exit_code = ?exit_report.code,
10820 exit_signal = ?exit_report.signal,
10821 "supervised module exited after deliberate connection severance"
10822 );
10823 let should_restart = match lock_snapshot(snapshot) {
10824 Ok(state) => state.enabled,
10825 Err(err) => {
10826 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10827 return NextAction::Stop {
10828 registration_released: false,
10829 };
10830 }
10831 };
10832 let final_state = if should_restart {
10833 ModuleState::Restarting
10834 } else {
10835 ModuleState::Disabled
10836 };
10837 if let Err(err) = publish_after_terminal_record(
10838 &spec.module_id,
10839 || {
10840 record_terminal(
10841 &spec.module_id,
10842 terminal_ring,
10843 spawn_events,
10844 &exit_report,
10845 terminal_disposition(final_state),
10846 )
10847 },
10848 || {
10849 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10850 state.state = final_state;
10851 clear_current_process_facts(state);
10852 state.last_exit = Some(exit_report.clone());
10853 state.lifetime_restarts += 1;
10854 })
10855 },
10856 ) {
10857 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10858 return NextAction::Stop {
10859 registration_released: false,
10860 };
10861 }
10862
10863 if should_restart {
10864 NextAction::Restart { schedule: None }
10865 } else {
10866 let registration_released = match wait_for_registration_release(
10867 registry,
10868 &spec.module_id,
10869 REGISTRY_RELEASE_TIMEOUT,
10870 )
10871 .await
10872 {
10873 Ok(()) => true,
10874 Err(err) => {
10875 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10876 false
10877 }
10878 };
10879 NextAction::Stop {
10880 registration_released,
10881 }
10882 }
10883 }
10884 }
10885}
10886
10887async fn on_child_exit_during_daemon_shutdown(
10888 spec: &ModuleSpec,
10889 registry: &Registry,
10890 snapshot: &SharedSnapshot,
10891 terminal_ring: &Arc<Mutex<TerminalRing>>,
10892 spawn_events: &SpawnEventFeed,
10893 exit_report: ExitReport,
10894) -> NextAction {
10895 info!(
10896 module_id = %spec.module_id,
10897 exit_code = ?exit_report.code,
10898 exit_signal = ?exit_report.signal,
10899 exit_kind = ?exit_report.kind,
10900 "supervised module exited during daemon shutdown; not restarting it"
10901 );
10902 if let Err(err) = publish_after_terminal_record(
10903 &spec.module_id,
10904 || {
10905 record_terminal(
10906 &spec.module_id,
10907 terminal_ring,
10908 spawn_events,
10909 &exit_report,
10910 TerminalDisposition::DaemonShutdown,
10911 )
10912 },
10913 || {
10914 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10915 state.state = ModuleState::Stopped;
10916 clear_current_process_facts(state);
10917 state.last_exit = Some(exit_report.clone());
10918 })
10919 },
10920 ) {
10921 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10922 }
10923 let registration_released =
10924 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10925 .await
10926 .is_ok();
10927 NextAction::Stop {
10928 registration_released,
10929 }
10930}
10931
10932fn publish_after_terminal_record<T>(
10937 module_id: &str,
10938 record: impl FnOnce(),
10939 publish: impl FnOnce() -> T,
10940) -> T {
10941 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10942 let published = publish();
10943 if let Err(panic) = recorded {
10944 error!(
10945 module_id,
10946 "terminal recording panicked; published final state before resuming unwind"
10947 );
10948 std::panic::resume_unwind(panic);
10949 }
10950 published
10951}
10952
10953fn fail_after_wait_error(
10954 module_id: &str,
10955 snapshot: &SharedSnapshot,
10956 terminal_ring: &Arc<Mutex<TerminalRing>>,
10957 spawn_events: &SpawnEventFeed,
10958 detail: String,
10959 failure: &FailureContext,
10960) {
10961 let report = wait_error_exit_report();
10962 publish_after_terminal_record(
10963 module_id,
10964 || {
10965 record_terminal_with_detail(
10966 module_id,
10967 terminal_ring,
10968 spawn_events,
10969 &report,
10970 TerminalDisposition::Failed,
10971 Some(detail),
10972 )
10973 },
10974 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
10975 );
10976}
10977
10978#[cfg(test)]
10979fn record_wait_error_terminal(
10980 module_id: &str,
10981 terminal_ring: &Arc<Mutex<TerminalRing>>,
10982 spawn_events: &SpawnEventFeed,
10983) {
10984 record_terminal(
10985 module_id,
10986 terminal_ring,
10987 spawn_events,
10988 &wait_error_exit_report(),
10989 TerminalDisposition::Failed,
10990 );
10991}
10992
10993fn record_terminal(
10994 module_id: &str,
10995 terminal_ring: &Arc<Mutex<TerminalRing>>,
10996 spawn_events: &SpawnEventFeed,
10997 exit_report: &ExitReport,
10998 disposition: TerminalDisposition,
10999) {
11000 record_terminal_with_detail(
11001 module_id,
11002 terminal_ring,
11003 spawn_events,
11004 exit_report,
11005 disposition,
11006 None,
11007 );
11008}
11009
11010fn durable_terminal_history_of(
11014 terminal_ring: &Mutex<TerminalRing>,
11015 module_id: &str,
11016) -> subc_control::TerminalHistory {
11017 let read = terminal_ring
11018 .lock()
11019 .unwrap_or_else(|p| p.into_inner())
11020 .capture_durable_history();
11021 read.read(module_id)
11022}
11023
11024fn record_terminal_with_detail(
11025 module_id: &str,
11026 terminal_ring: &Arc<Mutex<TerminalRing>>,
11027 spawn_events: &SpawnEventFeed,
11028 exit_report: &ExitReport,
11029 disposition: TerminalDisposition,
11030 disposition_detail: Option<String>,
11031) {
11032 #[cfg(test)]
11033 terminal_history_tests::before_terminal_record(spawn_events);
11034 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
11035 let record = TerminalRecord {
11036 exit_code: exit_report.code,
11037 exit_signal: exit_report.signal,
11038 at_ms: exit_report.at_ms,
11039 disposition,
11040 exit_kind: exit_report.kind.into(),
11041 disposition_detail,
11042 };
11043 terminal_ring
11044 .lock()
11045 .unwrap_or_else(|poisoned| poisoned.into_inner())
11046 .record_exit(module_id, record);
11047}
11048
11049fn untrack_if_registration_released(
11050 process_liveness: &SupervisorProcessLiveness,
11051 registry: &Registry,
11052 module_id: &str,
11053 snapshot: &SharedSnapshot,
11054) {
11055 match registry.get_module(module_id) {
11056 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
11057 Ok(Some(_)) => {}
11058 Err(err) => {
11059 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
11060 }
11061 }
11062}
11063
11064#[cfg(test)]
11078fn apply_wire_spawn_args(
11079 command: &mut Command,
11080 spec: &ModuleSpec,
11081 connection_file_path: Option<&std::path::Path>,
11082 handle: Option<&SupervisorHandle>,
11083) -> Result<Option<NonceHandoff>, SuperviseError> {
11084 apply_wire_spawn_args_for_role(
11085 command,
11086 spec,
11087 connection_file_path,
11088 handle,
11089 SpawnRole::Plain,
11090 )
11091}
11092
11093#[cfg(unix)]
11098type NonceHandoff = subc_os::LaunchNonceHandoff;
11099#[cfg(windows)]
11100type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11101#[cfg(not(any(unix, windows)))]
11102type NonceHandoff = std::convert::Infallible;
11103
11104#[cfg(test)]
11122fn apply_wire_spawn_args_for_role(
11123 command: &mut Command,
11124 spec: &ModuleSpec,
11125 connection_file_path: Option<&std::path::Path>,
11126 handle: Option<&SupervisorHandle>,
11127 role: SpawnRole,
11128) -> Result<Option<NonceHandoff>, SuperviseError> {
11129 apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11130}
11131
11132fn apply_wire_spawn_args_for_admission(
11133 command: &mut Command,
11134 spec: &ModuleSpec,
11135 connection_file_path: Option<&std::path::Path>,
11136 handle: Option<&SupervisorHandle>,
11137 role: SpawnRole,
11138 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11139) -> Result<Option<NonceHandoff>, SuperviseError> {
11140 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11141 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11146 #[cfg(windows)]
11147 {
11148 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11149 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11150 }
11151 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11153 command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11154 command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11155 command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11156 if role == SpawnRole::Plain {
11157 if let Some(handle) = handle {
11158 #[cfg(all(test, unix))]
11159 {
11160 let pause = {
11161 let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11162 if slot
11163 .as_ref()
11164 .is_some_and(|(module, _)| module == &spec.module_id)
11165 {
11166 slot.take().map(|(_, pause)| pause)
11167 } else {
11168 None
11169 }
11170 };
11171 if let Some(pause) = pause {
11172 pause.reached.send(()).unwrap();
11173 pause.release.recv().unwrap();
11174 }
11175 }
11176 handle.apply_identity_policy(spec, spec.protocol);
11180 }
11181 }
11182 if spec.protocol == ModuleProtocol::None {
11183 return Ok(None);
11184 }
11185 if let Some(connection_file_path) = connection_file_path {
11186 command.arg(SUBC_ARG).arg(connection_file_path);
11187 }
11188
11189 if admission.is_some_and(|admission| admission.launch.is_some()) {
11190 if let Some(handle) = handle {
11191 match role {
11192 SpawnRole::Plain => {
11193 handle
11194 .spawn_nonces
11195 .lock()
11196 .unwrap_or_else(|p| p.into_inner())
11197 .remove(&spec.module_id);
11198 if spec.reserved {
11199 handle
11200 .reserved_nonces
11201 .lock()
11202 .unwrap_or_else(|p| p.into_inner())
11203 .insert(spec.module_id.clone(), None);
11204 }
11205 }
11206 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11209 }
11210 }
11211 return Ok(None);
11212 }
11213 let nonce = generate_launch_nonce()?;
11217 if let Some(handle) = handle {
11218 match role {
11219 SpawnRole::Plain => {
11220 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11221 if spec.reserved {
11222 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11223 }
11224 }
11225 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11226 }
11227 }
11228 #[cfg(unix)]
11229 let handoff = {
11230 let handoff =
11231 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11232 program: spec.program.clone(),
11233 source,
11234 cgroup_path: None,
11235 })?;
11236 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11237 Some(handoff)
11238 };
11239 #[cfg(windows)]
11240 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11241 #[cfg(not(any(unix, windows)))]
11242 let handoff = None;
11243 #[cfg(not(unix))]
11247 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11248 Ok(handoff)
11249}
11250
11251pub(crate) fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11252 command.env_remove(CK_LOG_ENV);
11253 command.env_remove(SUBC_SPAWN_ROLE_ENV);
11260 for (key, value) in &spec.env {
11261 if matches!(
11265 key.as_str(),
11266 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11267 ) || key == SUBC_SPAWN_ROLE_ENV
11268 {
11269 continue;
11270 }
11271 command.env(key, value);
11272 }
11273}
11274
11275#[cfg(windows)]
11276fn prepare_windows_registration_pipe(
11277 command: &mut Command,
11278 spec: &ModuleSpec,
11279) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11280 if spec.protocol == ModuleProtocol::None {
11281 return Ok(None);
11282 }
11283 let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11284 SuperviseError::Spawn {
11285 program: spec.program.clone(),
11286 source,
11287 cgroup_path: None,
11288 }
11289 })?;
11290 command
11291 .env(
11292 subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11293 pipe.name(),
11294 )
11295 .env(
11296 subc_os::registration_channel::DAEMON_PID_ENV,
11297 std::process::id().to_string(),
11298 );
11299 Ok(Some(pipe))
11300}
11301
11302#[cfg(windows)]
11303fn prepare_windows_nonce_pipe(
11304 command: &mut Command,
11305 spec: &ModuleSpec,
11306 nonce: &str,
11307) -> Option<NonceHandoff> {
11308 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11309 Ok(handoff) => {
11310 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11311 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11316 Some(handoff)
11317 }
11318 Err(error) => {
11319 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11320 None
11321 }
11322 }
11323}
11324
11325#[cfg(windows)]
11326fn start_windows_nonce_pipe(
11327 handoff: Option<NonceHandoff>,
11328 pid: u32,
11329 spec: &ModuleSpec,
11330) -> Option<subc_os::LaunchNoncePipeDelivery> {
11331 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11335 Ok(delivery) => Some(delivery),
11336 Err(error) => {
11337 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11338 None
11339 }
11340 })
11341}
11342
11343fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11344 #[cfg(windows)]
11345 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11346 return Some(
11347 if snapshot
11348 .pipe_consumed
11349 .as_ref()
11350 .is_some_and(|consumed| consumed.get().is_some())
11351 {
11352 "pipe"
11353 } else {
11354 "env"
11355 }
11356 .to_owned(),
11357 );
11358 }
11359 let _ = snapshot;
11360 None
11361}
11362
11363#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11366enum SpawnRole {
11367 Plain,
11368 SwapCandidate,
11369}
11370
11371fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11374 if role == SpawnRole::SwapCandidate {
11375 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11376 }
11377}
11378
11379fn previous_instance_error(
11380 spec: &ModuleSpec,
11381 pid: u32,
11382 reason: crate::live_children::RetainedReason,
11383) -> SuperviseError {
11384 SuperviseError::Spawn {
11385 program: spec.program.clone(),
11386 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11387 cgroup_path: None,
11388 }
11389}
11390
11391fn carries_containment(configured: bool, has_container: bool) -> bool {
11392 configured || has_container
11393}
11394
11395fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11396 if !carries_containment(
11397 spec.carry_containment,
11398 containment::current(&roster.containment).is_some(),
11399 ) {
11400 return None;
11401 }
11402 Some(SuperviseError::Spawn {
11403 program: spec.program.clone(),
11404 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11405 cgroup_path: None,
11406 })
11407}
11408
11409#[allow(clippy::too_many_arguments)]
11410fn spawn_child(
11411 spec: &ModuleSpec,
11412 connection_file_path: Option<&std::path::Path>,
11413 handle: Option<&SupervisorHandle>,
11414 ring: &Arc<Mutex<StderrRing>>,
11415 capture_logs_dir: Option<&std::path::Path>,
11416 roster: &ChildRoster,
11417 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11418 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11419 register_timeout: Duration,
11420) -> Result<SupervisedChild, SuperviseError> {
11421 #[cfg(all(test, unix))]
11422 if let Some(source) = containment_tests::join_failure(spec, roster) {
11423 containment::terminate_blocking(&roster.containment);
11424 return Err(SuperviseError::Spawn {
11425 program: spec.program.clone(),
11426 source,
11427 cgroup_path: None,
11428 });
11429 }
11430 let result = spawn_child_in_slot(
11431 spec,
11432 connection_file_path,
11433 handle,
11434 ring,
11435 capture_logs_dir,
11436 roster,
11437 config_delivery,
11438 #[cfg(target_os = "linux")]
11439 cgroup_placement,
11440 SpawnRole::Plain,
11441 false,
11442 register_timeout,
11443 );
11444 if result.is_err() {
11445 if roster.admission_pending(&spec.module_id) {
11446 containment::terminate_deferred(&roster.containment);
11447 } else {
11448 containment::terminate_blocking(&roster.containment);
11449 }
11450 }
11451 result
11452}
11453
11454#[allow(clippy::too_many_arguments)]
11467fn spawn_child_in_slot(
11468 spec: &ModuleSpec,
11469 connection_file_path: Option<&std::path::Path>,
11470 handle: Option<&SupervisorHandle>,
11471 ring: &Arc<Mutex<StderrRing>>,
11472 capture_logs_dir: Option<&std::path::Path>,
11473 roster: &ChildRoster,
11474 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11475 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11476 role: SpawnRole,
11477 alternate_slot: bool,
11478 register_timeout: Duration,
11479) -> Result<SupervisedChild, SuperviseError> {
11480 let prepared = prepare_child_in_slot(
11481 spec,
11482 connection_file_path,
11483 handle,
11484 capture_logs_dir,
11485 roster,
11486 config_delivery,
11487 #[cfg(target_os = "linux")]
11488 cgroup_placement,
11489 role,
11490 alternate_slot,
11491 false,
11492 )?;
11493 let gate = config_delivery
11494 .map(|delivery| delivery.module_gate(spec))
11495 .transpose()?;
11496 let _gate = gate
11497 .as_ref()
11498 .map(|gate| {
11499 gate.try_lock()
11500 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11501 module_id: spec.module_id.clone(),
11502 })
11503 })
11504 .transpose()?;
11505 if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11506 delivery.check(spec, admission)?;
11507 }
11508 let checked_base = prepared
11509 .spawn_admission
11510 .as_ref()
11511 .and_then(|admission| admission.approved_base())
11512 .cloned();
11513 let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11514 drop(_gate);
11515 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11516 (config_delivery, &result)
11517 {
11518 if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11519 if !delivery.record_refusal(spec, base, detail)? {
11520 result = Err(spawn_plan_stale(spec));
11521 }
11522 }
11523 }
11524 result
11525}
11526
11527struct AsyncInitialIdentity<'a> {
11528 supervisor: &'a Supervisor,
11529 module: String,
11530 committed: bool,
11531}
11532impl Drop for AsyncInitialIdentity<'_> {
11533 fn drop(&mut self) {
11534 if !self.committed {
11535 self.supervisor.abandon_unrostered(&self.module);
11536 }
11537 }
11538}
11539
11540struct PreparedChild {
11544 registration_spawn: Option<Arc<RegistrationSpawn>>,
11545 registration_handle: Option<SupervisorHandle>,
11546 #[cfg(any(unix, windows))]
11547 registration_channel: Option<SpawnRegistrationChannel>,
11548 config_context: Option<config_delivery::ConfigDelivery>,
11549 binding_publication: Option<ProgramBindingPublication>,
11550 launch_spec: ModuleSpec,
11551 spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11552 command: Command,
11553 carrying: bool,
11554 carried: Option<Arc<containment::CarriedContainment>>,
11555 #[cfg(unix)]
11556 launch_identity: Option<containment::LaunchIdentity>,
11557 #[cfg(unix)]
11558 group_target: containment::GroupTarget,
11559 #[cfg(target_os = "linux")]
11560 cgroup_placement: Option<subc_cgroup::Placement>,
11561 #[cfg(target_os = "linux")]
11562 cgroup_name: String,
11563 #[cfg(target_os = "linux")]
11564 cgroup_cleanup: PreparedCgroup,
11565 cgroup_path: Option<PathBuf>,
11566 #[cfg(unix)]
11567 admission_gate: Option<admission::Gate>,
11568 output_sink: ChildOutputSink,
11569 #[cfg(target_os = "macos")]
11570 privacy_exec: Option<PrivacyExec>,
11571 #[cfg(windows)]
11572 image_capture: subc_os::ExecutableCapture,
11573 #[cfg(windows)]
11574 nonce_handoff: Option<NonceHandoff>,
11575 spawned_at: Instant,
11576 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11577 delivery_registration: Option<config_delivery::RegistrationGuard>,
11578 program_binding: Option<SpawnProgramBinding>,
11579}
11580
11581#[cfg(target_os = "linux")]
11582struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11583#[cfg(target_os = "linux")]
11584impl Drop for PreparedCgroup {
11585 fn drop(&mut self) {
11586 if let Some((placement, name)) = &self.0 {
11587 remove_module_cgroup(placement, name);
11588 }
11589 }
11590}
11591
11592pub(crate) struct SpawnProgramBinding {
11593 pub(crate) path: PathBuf,
11594 _file: std::fs::File,
11597 pub(crate) identity: subc_os::RetainedFileIdentity,
11598}
11599
11600impl SpawnProgramBinding {
11601 pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11602 Self::capture_after_resolution(spec, |_| {})
11603 }
11604
11605 fn capture_after_resolution(
11606 spec: &ModuleSpec,
11607 before_open: impl FnOnce(&std::path::Path),
11608 ) -> Result<Self, SuperviseError> {
11609 require_program_search_path(spec)?;
11610 #[cfg(windows)]
11611 let windows_capture =
11612 windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11613 program: spec.program.clone(),
11614 source,
11615 cgroup_path: None,
11616 })?;
11617 #[cfg(windows)]
11618 let path = windows_capture.path().to_path_buf();
11619 #[cfg(not(windows))]
11620 let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11621 spec.program.clone()
11622 } else {
11623 let path = spec
11624 .env
11625 .iter()
11626 .rev()
11627 .find(|(key, _)| key == "PATH")
11628 .map(|(_, path)| std::ffi::OsString::from(path))
11629 .or_else(|| std::env::var_os("PATH"))
11630 .unwrap_or_else(|| {
11631 if cfg!(target_os = "macos") {
11632 "/usr/bin:/bin".into()
11635 } else {
11636 Default::default()
11637 }
11638 });
11639 std::env::split_paths(&path)
11640 .map(|dir| dir.join(&spec.program))
11641 .find(|path| {
11642 if !path.is_file() {
11643 return false;
11644 }
11645 #[cfg(unix)]
11646 {
11647 rustix::fs::accessat(
11648 rustix::fs::CWD,
11649 path,
11650 rustix::fs::Access::EXEC_OK,
11651 rustix::fs::AtFlags::EACCESS,
11652 )
11653 .is_ok()
11654 }
11655 #[cfg(not(unix))]
11656 {
11657 true
11658 }
11659 })
11660 .ok_or_else(|| SuperviseError::Spawn {
11661 program: spec.program.clone(),
11662 source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11663 cgroup_path: None,
11664 })?
11665 };
11666 let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11667 program: spec.program.clone(),
11668 source,
11669 cgroup_path: None,
11670 })?;
11671 before_open(&resolved);
11672 let file =
11675 subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11676 program: spec.program.clone(),
11677 source: io::Error::other(source),
11678 cgroup_path: None,
11679 })?;
11680 let identity =
11681 subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11682 program: spec.program.clone(),
11683 source,
11684 cgroup_path: None,
11685 })?;
11686 Ok(Self {
11687 path,
11688 _file: file,
11689 identity,
11690 })
11691 }
11692 fn current(&self) -> bool {
11693 #[cfg(unix)]
11694 {
11695 subc_os::file_identity(&self.path) == Some(self.identity)
11696 }
11697 #[cfg(windows)]
11698 {
11699 std::fs::canonicalize(&self.path)
11700 .ok()
11701 .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11702 .and_then(|file| subc_os::retained_file_identity(&file).ok())
11703 == Some(self.identity)
11704 }
11705 }
11706}
11707
11708fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11709 SuperviseError::ConfigDeliveryDeferred {
11710 module_id: spec.module_id.clone(),
11711 }
11712}
11713
11714fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11715 if !cfg!(target_os = "macos")
11716 && !spec.program.is_absolute()
11717 && spec.program.components().count() == 1
11718 && !spec.env.iter().any(|(key, _)| {
11719 if cfg!(windows) {
11720 key.eq_ignore_ascii_case("PATH")
11721 } else {
11722 key == "PATH"
11723 }
11724 })
11725 && std::env::var_os("PATH").is_none()
11726 {
11727 return Err(SuperviseError::Spawn {
11730 program: spec.program.clone(),
11731 source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11732 cgroup_path: None,
11733 });
11734 }
11735 Ok(())
11736}
11737
11738async fn spawn_child_async(
11739 spec: &ModuleSpec,
11740 runtime: &SupervisorRuntimeConfig,
11741) -> Result<SupervisedChild, SuperviseError> {
11742 #[cfg(all(test, unix))]
11743 if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11744 containment::terminate_blocking(&runtime.child_roster.containment);
11745 return Err(SuperviseError::Spawn {
11746 program: spec.program.clone(),
11747 source,
11748 cgroup_path: None,
11749 });
11750 }
11751 let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11752 if result.is_err() {
11753 if runtime.child_roster.admission_pending(&spec.module_id) {
11754 containment::terminate_deferred(&runtime.child_roster.containment);
11755 } else {
11756 containment::terminate_blocking(&runtime.child_roster.containment);
11757 }
11758 }
11759 result
11760}
11761
11762async fn spawn_child_in_slot_async(
11763 spec: &ModuleSpec,
11764 runtime: &SupervisorRuntimeConfig,
11765 role: SpawnRole,
11766 alternate_slot: bool,
11767) -> Result<SupervisedChild, SuperviseError> {
11768 let gate = runtime
11769 .config_delivery
11770 .as_ref()
11771 .map(|delivery| delivery.0.module_gate(&spec.module_id))
11772 .transpose()
11773 .map_err(|error| SuperviseError::InvalidSpec {
11774 reason: error.to_string(),
11775 })?;
11776 if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11777 return Err(SuperviseError::ConfigDeliveryDeferred {
11778 module_id: spec.module_id.clone(),
11779 });
11780 }
11781 let prepared_spec = spec.clone();
11782 let prepared_runtime = runtime.clone();
11783 let prepared = tokio::task::spawn_blocking(move || {
11784 let prepared = prepare_child_in_slot(
11785 &prepared_spec,
11786 prepared_runtime.connection_file_path.as_deref(),
11787 prepared_runtime.supervisor_handle.as_ref(),
11788 prepared_runtime.capture_logs_dir.as_deref(),
11789 &prepared_runtime.child_roster,
11790 prepared_runtime.spawn_delivery(),
11791 #[cfg(target_os = "linux")]
11792 prepared_runtime.cgroup_placement.as_ref(),
11793 role,
11794 alternate_slot,
11795 true,
11796 );
11797 prepared
11798 })
11799 .await
11800 .map_err(|_| SuperviseError::InvalidSpec {
11801 reason: "configuration launch preparation stopped".into(),
11802 })??;
11803 if let Some(registration) = &prepared.delivery_registration {
11804 if !registration.launch_current()? {
11805 return Err(spawn_plan_stale(spec));
11806 }
11807 }
11808 let _gate = match gate {
11809 Some(gate) => {
11810 Some(
11811 gate.try_lock_owned()
11812 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11813 module_id: spec.module_id.clone(),
11814 })?,
11815 )
11816 }
11817 None => None,
11818 };
11819 if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11820 {
11821 delivery.check(spec, admission)?;
11822 }
11823 if prepared
11824 .spawn_admission
11825 .as_ref()
11826 .is_none_or(|admission| admission.launch.is_none())
11827 && prepared
11828 .program_binding
11829 .as_ref()
11830 .is_some_and(|binding| !binding.current())
11831 {
11832 return Err(spawn_plan_stale(spec));
11833 }
11834 let checked_base = prepared
11836 .spawn_admission
11837 .as_ref()
11838 .and_then(|admission| admission.approved_base())
11839 .cloned();
11840 let mut result = launch_child_in_slot(
11841 spec,
11842 &runtime.stderr_ring,
11843 &runtime.child_roster,
11844 prepared,
11845 runtime.register_timeout,
11846 );
11847 drop(_gate);
11848 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11849 (runtime.config_delivery.as_ref(), &result)
11850 {
11851 if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11852 let module = spec.module_id.clone();
11853 let identity = subc_config::raw_digest(detail.as_bytes());
11854 let recorded = delivery
11855 .0
11856 .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11857 .await;
11858 match recorded {
11859 Ok(true) => {}
11860 Ok(false) => result = Err(spawn_plan_stale(spec)),
11861 Err(error) => {
11862 result = Err(config_delivery::gate_hold_error(
11863 spec,
11864 error,
11865 Some(delivery),
11866 ))
11867 }
11868 }
11869 }
11870 }
11871 result
11872}
11873
11874fn validate_spawn_program(
11875 spec: &ModuleSpec,
11876 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11877 delivery: Option<config_delivery::SpawnDelivery<'_>>,
11878) -> Result<(), SuperviseError> {
11879 if let Some(admission) = admission {
11880 if let Some(program) = &admission.launch {
11881 if let Err(error) = program.validate_for_spawn() {
11882 if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11886 if error.code == "program_refused"
11887 && !context.record_refusal(spec, base, &error.detail)?
11888 {
11889 return Err(spawn_plan_stale(spec));
11890 }
11891 }
11892 return Err(config_delivery::gate_hold_error(
11893 spec,
11894 error,
11895 delivery.map(|context| context.runtime()).as_ref(),
11896 ));
11897 }
11898 }
11899 }
11900 Ok(())
11901}
11902
11903#[allow(clippy::too_many_arguments)]
11904fn prepare_child_in_slot(
11905 spec: &ModuleSpec,
11906 connection_file_path: Option<&std::path::Path>,
11907 handle: Option<&SupervisorHandle>,
11908 capture_logs_dir: Option<&std::path::Path>,
11909 roster: &ChildRoster,
11910 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11911 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11912 role: SpawnRole,
11913 alternate_slot: bool,
11914 asynchronous: bool,
11915) -> Result<PreparedChild, SuperviseError> {
11916 require_program_search_path(spec)?;
11917 let spawn_admission = config_delivery
11918 .map(|delivery| delivery.prepare(spec))
11919 .transpose()?;
11920 #[cfg(target_os = "macos")]
11921 let gated = spawn_admission
11922 .as_ref()
11923 .is_some_and(|admission| admission.approved_base().is_some());
11924 let launch_spec = spawn_admission
11925 .as_ref()
11926 .and_then(|admission| admission.launch.as_ref())
11927 .map(|launch| launch.spec.clone())
11928 .unwrap_or_else(|| spec.clone());
11929 let spec = &launch_spec;
11930 if let Some(error) = recovery_block_error(spec, roster) {
11931 return Err(error);
11932 }
11933 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
11934 return Err(previous_instance_error(spec, pid, reason));
11935 }
11936 if roster.is_closed() {
11937 return Err(SuperviseError::Spawn {
11938 program: spec.program.clone(),
11939 source: io::Error::other("the daemon is shutting down; not starting a new process"),
11940 cgroup_path: None,
11941 });
11942 }
11943 let carried = containment::current(&roster.containment);
11944 let carrying = carries_containment(spec.carry_containment, carried.is_some());
11945 if role != SpawnRole::Plain && carrying {
11946 return Err(SuperviseError::InvalidSpec {
11947 reason: "a carried container cannot overlap a swap candidate".to_owned(),
11948 });
11949 }
11950 #[cfg(unix)]
11951 let launch_identity = if carrying && carried.is_none() {
11952 Some(
11953 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
11954 program: spec.program.clone(),
11955 source,
11956 cgroup_path: None,
11957 })?,
11958 )
11959 } else {
11960 None
11961 };
11962 #[cfg(target_os = "linux")]
11963 let cgroup_placement = match &carried {
11964 Some(container) => container.cgroup().map(|(placement, _)| placement),
11965 None => cgroup_placement,
11966 };
11967 #[cfg(unix)]
11968 let group_target = if let Some(container) = carried
11969 .as_ref()
11970 .filter(|container| container.group().is_some())
11971 {
11972 containment::GroupTarget::Pinned(Arc::clone(container))
11973 } else {
11974 #[cfg(target_os = "linux")]
11975 let cgroup_container = carrying && cgroup_placement.is_some();
11976 #[cfg(not(target_os = "linux"))]
11977 let cgroup_container = false;
11978 if cgroup_container {
11979 containment::GroupTarget::NoGroup
11980 } else {
11981 containment::GroupTarget::OwnLeader
11982 }
11983 };
11984 #[cfg(target_os = "linux")]
11985 let cgroup_name = {
11986 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
11991 name.to_owned()
11992 } else if cgroup_placement.is_none() {
11993 swap::cgroup_name(&spec.module_id, alternate_slot)
11994 } else {
11995 let nonce = generate_launch_nonce()?;
11996 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
11997 let mut end = spec.module_id.len().min(64);
12000 while !spec.module_id.is_char_boundary(end) {
12001 end -= 1;
12002 }
12003 format!(
12004 "{}_{suffix}",
12005 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
12006 )
12007 }
12008 };
12009 #[cfg(not(target_os = "linux"))]
12010 let _ = alternate_slot;
12011 #[cfg(windows)]
12012 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
12013 program: spec.program.clone(),
12014 source,
12015 cgroup_path: None,
12016 })?;
12017 #[cfg(windows)]
12018 let program_binding = if asynchronous {
12019 let mut resolved = spec.clone();
12020 resolved.program = image_capture.path().to_path_buf();
12021 Some(SpawnProgramBinding::capture(&resolved)?)
12022 } else {
12023 None
12024 };
12025 #[cfg(not(windows))]
12026 let program_binding = (asynchronous || cfg!(target_os = "macos"))
12027 .then(|| SpawnProgramBinding::capture(spec))
12028 .transpose()?;
12029 #[cfg(target_os = "macos")]
12030 validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
12031 #[cfg(target_os = "macos")]
12032 let (mut command, privacy_exec, exec_ack) = privacy_command(
12033 spec,
12034 program_binding
12035 .as_ref()
12036 .expect("macOS captures every launch"),
12037 roster,
12038 gated,
12042 )?;
12043 #[cfg(windows)]
12044 let mut command = Command::new(image_capture.path());
12045 #[cfg(not(any(target_os = "macos", windows)))]
12046 let mut command = Command::new(
12047 program_binding
12048 .as_ref()
12049 .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
12050 );
12051 command.args(&spec.args);
12052 apply_child_env(&mut command, spec);
12082 let origin_witness = crate::program_origin::LaunchWitness::capture_command(spec, &command)
12083 .map_err(|source| SuperviseError::Spawn {
12084 program: spec.program.clone(),
12085 source,
12086 cgroup_path: None,
12087 })?;
12088 command.current_dir(&origin_witness.working_directory);
12089 apply_spawn_role(&mut command, role);
12090 let nonce_handoff = apply_wire_spawn_args_for_admission(
12091 &mut command,
12092 spec,
12093 connection_file_path,
12094 handle,
12095 role,
12096 spawn_admission.as_ref(),
12097 )?;
12098 #[cfg(unix)]
12099 let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12100 #[cfg(unix)]
12101 let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12102 let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12103 .map_err(|source| SuperviseError::Spawn {
12104 program: spec.program.clone(),
12105 source,
12106 cgroup_path: None,
12107 })?;
12108 slots
12109 .channel(child)
12110 .map_err(|source| SuperviseError::Spawn {
12111 program: spec.program.clone(),
12112 source,
12113 cgroup_path: None,
12114 })?;
12115 command
12116 .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12117 .env(
12118 subc_os::registration_channel::DAEMON_PID_ENV,
12119 std::process::id().to_string(),
12120 );
12121 Some(parent)
12122 } else {
12123 None
12124 };
12125 #[cfg(windows)]
12126 let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12127 let binding_publication = if spawn_admission
12128 .as_ref()
12129 .is_some_and(|admission| admission.launch.is_some())
12130 {
12131 handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12132 } else {
12133 None
12134 };
12135 let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12136 .then(|| {
12137 handle.map(|handle| {
12138 let spawn = RegistrationSpawn::new_with_binding_requirement(
12139 handle,
12140 &spec.module_id,
12141 role,
12142 binding_publication.clone(),
12143 spawn_admission
12144 .as_ref()
12145 .is_some_and(|admission| admission.launch.is_some()),
12146 );
12147 if let Some(admission) = &spawn_admission {
12148 admission.initialize_origin(&spawn.authority, &origin_witness);
12149 }
12150 handle.record_nonce_launch(&spawn);
12151 spawn
12152 })
12153 })
12154 .flatten();
12155
12156 #[cfg(target_os = "linux")]
12157 let cgroup_path = cgroup_placement
12158 .map(|placement| {
12159 if let Some(container) = &carried {
12160 let path = container
12161 .cgroup_path()
12162 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12163 placement.existing_module_path(path)?.ok_or_else(|| {
12164 io::Error::new(
12165 io::ErrorKind::NotFound,
12166 "carried cgroup disappeared before replacement",
12167 )
12168 })
12169 } else {
12170 if carrying {
12171 containment::prepare_cgroup_path(placement, &cgroup_name)
12172 } else {
12173 placement.module_path(&cgroup_name)
12174 }
12175 }
12176 })
12177 .transpose()
12178 .map_err(|source| SuperviseError::Cgroup {
12179 module_id: spec.module_id.clone(),
12180 source,
12181 })?;
12182 #[cfg(not(target_os = "linux"))]
12183 let cgroup_path: Option<PathBuf> = None;
12184 #[cfg(target_os = "linux")]
12187 let cgroup_cleanup = PreparedCgroup(
12188 cgroup_placement
12189 .filter(|_| carried.is_none())
12190 .map(|placement| (placement.clone(), cgroup_name.clone())),
12191 );
12192 #[cfg(unix)]
12193 let mut admission_gate = if carrying {
12194 Some(
12195 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12196 SuperviseError::Spawn {
12197 program: spec.program.clone(),
12198 source,
12199 cgroup_path: cgroup_path.clone(),
12200 }
12201 })?,
12202 )
12203 } else {
12204 None
12205 };
12206 #[cfg(target_os = "linux")]
12207 if let Some(path) = &cgroup_path {
12208 apply_cgroup_placement(&mut command, spec, path)?;
12209 }
12210
12211 let output_sink = if let Some(logs_dir) = capture_logs_dir {
12212 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12213 match ChildOutputSink::open(&path, capture_retention(spec)) {
12214 Ok(sink) => sink,
12215 Err(error) => {
12216 warn!(
12217 module_id = %spec.module_id,
12218 path = %path.display(),
12219 error = %error,
12220 "could not open child output capture file; forwarding to stderr"
12221 );
12222 ChildOutputSink::Stderr
12223 }
12224 }
12225 } else {
12226 ChildOutputSink::Stderr
12227 };
12228
12229 command.stdout(Stdio::piped());
12230 command.stderr(Stdio::piped());
12231 command.kill_on_drop(!carrying);
12232 #[cfg(unix)]
12249 command.process_group(
12250 carried
12251 .as_ref()
12252 .map(|container| container.verified_join_group())
12253 .transpose()
12254 .map_err(|source| SuperviseError::Spawn {
12255 program: spec.program.clone(),
12256 source,
12257 cgroup_path: cgroup_path.clone(),
12258 })?
12259 .flatten()
12260 .unwrap_or(0) as i32,
12261 );
12262 command.stdin(Stdio::null());
12263 #[cfg(unix)]
12264 if let Some(gate) = &mut admission_gate {
12265 gate.install_placement(&mut command);
12266 }
12267 #[cfg(unix)]
12270 {
12271 if let Some(handoff) = nonce_handoff {
12272 slots.nonce(handoff);
12273 }
12274 #[cfg(target_os = "macos")]
12275 slots
12276 .acknowledgement(exec_ack.into_descriptor())
12277 .map_err(|source| SuperviseError::Spawn {
12278 program: spec.program.clone(),
12279 source,
12280 cgroup_path: cgroup_path.clone(),
12281 })?;
12282 slots.install(command.as_std_mut());
12283 }
12284 #[cfg(not(any(unix, windows)))]
12285 let _ = nonce_handoff;
12286
12287 #[cfg(windows)]
12292 {
12293 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12294 program: spec.program.clone(),
12295 source,
12296 cgroup_path: None,
12297 })?;
12298 subc_jobobject::suspend_on_create_async(&mut command);
12299 }
12300 let spawned_at = Instant::now();
12304 let delivery_context = config_delivery;
12305 let config_delivery = config_delivery
12306 .map(|delivery| {
12307 delivery.spawned(
12308 spec,
12309 spawn_admission
12310 .as_ref()
12311 .expect("delivery prepares spawn admission"),
12312 spawned_at,
12313 role,
12314 registration_spawn
12315 .as_ref()
12316 .map(|spawn| spawn.authority.clone()),
12317 )
12318 })
12319 .transpose()?
12320 .flatten();
12321 let mut delivery_registration =
12322 delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12323 if let Some(registration) = &mut delivery_registration {
12324 registration.defer_cleanup();
12325 }
12326 if asynchronous {
12327 #[cfg(test)]
12328 if let Some(context) = delivery_context {
12329 context.pause_preflight_for_test();
12330 }
12331 }
12332 validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12333 Ok(PreparedChild {
12334 registration_spawn,
12335 registration_handle: handle.cloned(),
12336 #[cfg(any(unix, windows))]
12337 registration_channel,
12338 config_context: delivery_context.map(|delivery| delivery.runtime()),
12339 binding_publication,
12340 launch_spec,
12341 spawn_admission,
12342 command,
12343 carrying,
12344 carried: carried.clone(),
12345 #[cfg(unix)]
12346 launch_identity,
12347 #[cfg(unix)]
12348 group_target,
12349 #[cfg(target_os = "linux")]
12350 cgroup_placement: cgroup_placement.cloned(),
12351 #[cfg(target_os = "linux")]
12352 cgroup_cleanup,
12353 #[cfg(target_os = "linux")]
12354 cgroup_name,
12355 cgroup_path,
12356 #[cfg(unix)]
12357 admission_gate,
12358 output_sink,
12359 #[cfg(target_os = "macos")]
12360 privacy_exec,
12361 #[cfg(windows)]
12362 image_capture,
12363 #[cfg(windows)]
12364 nonce_handoff,
12365 spawned_at,
12366 config_delivery,
12367 delivery_registration,
12368 program_binding,
12369 })
12370}
12371
12372fn launch_child_in_slot(
12373 _spec: &ModuleSpec,
12374 ring: &Arc<Mutex<StderrRing>>,
12375 roster: &ChildRoster,
12376 prepared: PreparedChild,
12377 register_timeout: Duration,
12378) -> Result<SupervisedChild, SuperviseError> {
12379 let launch_spec = prepared.launch_spec.clone();
12380 let registration_declaration = prepared
12381 .spawn_admission
12382 .as_ref()
12383 .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12384 let admission_base = prepared
12385 .spawn_admission
12386 .as_ref()
12387 .and_then(|admission| admission.approved_base())
12388 .cloned();
12389 let spec = &launch_spec;
12390 let expected_image = prepared
12391 .spawn_admission
12392 .as_ref()
12393 .and_then(|admission| admission.launch.as_ref())
12394 .map(|launch| launch.image);
12395 let program_launch = prepared
12396 .spawn_admission
12397 .and_then(|admission| admission.launch);
12398 let config_owner = prepared
12399 .config_context
12400 .as_ref()
12401 .map(|context| context.0.clone());
12402 let binding_publication = prepared.binding_publication;
12403 #[cfg(test)]
12404 if let Some(context) = &prepared.config_context {
12405 context.0.before_spawn_exec_for_test();
12406 }
12407 #[cfg(feature = "test-support")]
12408 if let Some(context) = &prepared.config_context {
12409 context
12410 .0
12411 .pause_spawn_before_exec()
12412 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12413 }
12414 let mut command = prepared.command;
12415 let carrying = prepared.carrying;
12416 let carried = prepared.carried;
12417 #[cfg(unix)]
12418 let launch_identity = prepared.launch_identity;
12419 #[cfg(unix)]
12420 let group_target = prepared.group_target;
12421 #[cfg(target_os = "linux")]
12422 let cgroup_placement = prepared.cgroup_placement.as_ref();
12423 #[cfg(target_os = "linux")]
12424 let cgroup_name = prepared.cgroup_name;
12425 #[cfg(target_os = "linux")]
12426 let mut cgroup_cleanup = prepared.cgroup_cleanup;
12427 #[cfg(target_os = "linux")]
12428 let _ = cgroup_cleanup.0.take();
12429 let cgroup_path = prepared.cgroup_path;
12430 #[cfg(unix)]
12431 let admission_gate = prepared.admission_gate;
12432 let output_sink = prepared.output_sink;
12433 #[cfg(target_os = "macos")]
12434 let mut privacy_exec = prepared.privacy_exec;
12435 #[cfg(target_os = "macos")]
12436 if let Some(pending) = &mut privacy_exec {
12437 pending.admission_binding = binding_publication.clone();
12438 if expected_image.is_some() {
12439 pending.expected = expected_image;
12440 }
12441 }
12442 #[cfg(any(unix, windows))]
12443 let registration_channel = prepared.registration_channel;
12444 let registration_spawn = prepared.registration_spawn;
12445 let registration_handle = prepared.registration_handle;
12446 #[cfg(windows)]
12447 let image_capture = prepared.image_capture;
12448 #[cfg(windows)]
12449 let nonce_handoff = prepared.nonce_handoff;
12450 let spawned_at = prepared.spawned_at;
12451 let config_delivery = prepared.config_delivery;
12452 let delivery_registration = prepared.delivery_registration;
12453 let _program_binding = prepared.program_binding;
12454 #[cfg(all(test, unix))]
12455 containment_tests::record_spawn_attempt(spec, &mut command);
12456 #[cfg(unix)]
12457 let spawned = match admission_gate {
12458 Some(gate) => gate
12459 .spawn(
12460 command,
12461 spec,
12462 roster,
12463 launch_identity,
12464 #[cfg(target_os = "linux")]
12465 cgroup_placement
12466 .zip(cgroup_path.as_ref())
12467 .map(|(placement, path)| {
12468 (placement.clone(), cgroup_name.clone(), path.clone())
12469 }),
12470 )
12471 .map(|(child, admitted)| (child, Some(admitted))),
12472 None => {
12473 let spawned =
12474 subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12475 drop(command);
12478 spawned
12479 }
12480 };
12481 #[cfg(not(unix))]
12482 let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12483 let spawned = match spawned {
12484 Ok(child) => child,
12485 Err(source) => {
12486 if carried
12487 .as_ref()
12488 .and_then(|container| container.group())
12489 .is_some()
12490 {
12491 containment::terminate_blocking(&roster.containment);
12492 }
12493 #[cfg(target_os = "linux")]
12494 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12495 remove_module_cgroup(placement, &cgroup_name);
12496 }
12497 return Err(SuperviseError::Spawn {
12498 program: spec.program.clone(),
12499 source,
12500 cgroup_path,
12501 });
12502 }
12503 };
12504 #[cfg(unix)]
12505 let (mut child, mut admitted) = spawned;
12506 #[cfg(unix)]
12507 let group_target = admitted
12508 .as_ref()
12509 .map(|attempt| attempt.group_target())
12510 .unwrap_or(group_target);
12511 #[cfg(not(unix))]
12512 let mut child = spawned;
12513 #[cfg(test)]
12514 spawn_record_tests::after_create(&child, spec);
12515
12516 #[cfg(windows)]
12521 let windows_image = match image_capture.bind_tokio(&child) {
12522 Ok(image) => Arc::new(image),
12523 Err(source) => {
12524 kill_and_reap_rejected_child(&mut child);
12525 return Err(SuperviseError::Spawn {
12526 program: spec.program.clone(),
12527 source,
12528 cgroup_path: None,
12529 });
12530 }
12531 };
12532 #[cfg(windows)]
12535 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12536 let spawned_at_ms = unix_ms_now();
12537 #[cfg(not(windows))]
12538 let spawned_from = spec.program.clone();
12539 #[cfg(windows)]
12540 let spawned_from = windows_image.path().to_path_buf();
12541 #[cfg(not(windows))]
12542 let spawned_file_identity = spawned_file_identity(&spawned_from);
12543 #[cfg(windows)]
12544 let spawned_file_identity = Some(SpawnedFileIdentity {
12545 device: 0,
12546 inode: 0,
12547 windows_image: Some(Arc::clone(&windows_image)),
12548 });
12549 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12550 program: spec.program.clone(),
12551 source: io::Error::other("spawned child exposed no live pid"),
12552 cgroup_path: cgroup_path.clone(),
12553 })?;
12554 #[cfg(windows)]
12555 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12556 #[cfg(not(windows))]
12557 let process_start_time = admitted
12558 .as_ref()
12559 .map(|admitted| admitted.birth)
12560 .or_else(|| crate::provenance::process_start_time(pid));
12561 #[cfg(windows)]
12562 let process_start_time = Some(windows_image.start_time());
12563 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12564 #[cfg(all(test, target_os = "macos"))]
12565 privacy_exec_boundary_tests::before_image_sample(spec, pid);
12566 #[cfg(all(test, target_os = "linux"))]
12567 assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12568 #[cfg(not(windows))]
12573 let recorded_image = observe_spawned_image(pid);
12574 #[cfg(all(test, target_os = "linux"))]
12575 let recorded_image = if roster.admission_event_for_test(
12576 &spec.module_id,
12577 "after_spawn_image_sample",
12578 Some(pid),
12579 ) {
12580 recorded_image
12581 } else {
12582 None
12583 };
12584 #[cfg(target_os = "linux")]
12585 let exited_before_binding = expected_image.is_some()
12588 && recorded_image.and_then(|image| image.executable) != expected_image
12589 && child.try_wait().ok().flatten().is_some();
12590 #[cfg(target_os = "linux")]
12591 let unavailable_initial_image = expected_image.is_some()
12592 && recorded_image.and_then(|image| image.executable).is_none()
12593 && !exited_before_binding;
12594 #[cfg(not(target_os = "linux"))]
12595 let unavailable_initial_image = false;
12596 #[cfg(target_os = "linux")]
12597 if expected_image
12598 .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12599 && !exited_before_binding
12600 && !unavailable_initial_image
12601 {
12602 kill_module_cgroup(cgroup_placement, &cgroup_name);
12603 kill_and_reap_rejected_child(&mut child, group_target.clone());
12604 return Err(SuperviseError::ConfigHeld {
12605 module_id: spec.module_id.clone(),
12606 code: "program_refused".into(),
12607 detail: format!(
12608 "spawned executable {:?} does not match hashed program {:?}",
12609 recorded_image.and_then(|image| image.executable),
12610 expected_image
12611 ),
12612 });
12613 }
12614 #[cfg(not(target_os = "linux"))]
12615 let _ = expected_image;
12616 #[cfg(target_os = "macos")]
12617 let binding_ready = privacy_exec.is_none();
12618 #[cfg(target_os = "linux")]
12619 let binding_ready = !exited_before_binding;
12620 #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12621 let binding_ready = cfg!(windows);
12622 #[cfg(target_os = "macos")]
12626 let recorded_image = if privacy_exec.is_some() {
12627 None
12628 } else {
12629 recorded_image
12630 };
12631 #[cfg(target_os = "linux")]
12632 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12633 #[cfg(not(target_os = "linux"))]
12634 let recorded_cgroup_name = None;
12635 #[cfg(unix)]
12636 let preadmitted = admitted
12637 .as_mut()
12638 .and_then(|admitted| admitted.leader.take());
12639 #[cfg(not(unix))]
12640 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12641 let roster_guard = if let Some(guard) = preadmitted {
12642 Ok(guard)
12643 } else {
12644 roster.admit(
12645 spec.module_id.clone(),
12646 pid,
12647 spec.protocol,
12648 process_start_time,
12649 crate::child_roster::RecordedIdentity {
12650 container: None,
12651 #[cfg(unix)]
12652 carried: None,
12653 #[cfg(not(windows))]
12654 start_time: recorded_image.map(|image| image.start_time),
12655 #[cfg(windows)]
12656 start_time: process_start_time,
12657 #[cfg(not(windows))]
12658 executable: recorded_image
12659 .and_then(|image| image.executable)
12660 .map(crate::live_children::ExecutableIdentity::from),
12661 #[cfg(windows)]
12662 executable: Some(crate::live_children::ExecutableIdentity::from(
12663 windows_image.identity(),
12664 )),
12665 cgroup_name: recorded_cgroup_name,
12666 #[cfg(target_os = "linux")]
12667 cgroup_placement: cgroup_placement.cloned(),
12668 },
12669 )
12670 };
12671 let roster_guard = match roster_guard {
12672 Ok(guard) => guard,
12673 Err(failure) => {
12674 return failure.with_cleanup(|source| {
12675 #[cfg(windows)]
12676 if let Some(job) = &job {
12677 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12678 }
12679 #[cfg(target_os = "linux")]
12680 kill_module_cgroup(cgroup_placement, &cgroup_name);
12681 #[cfg(unix)]
12682 let rejected_group = group_target.clone();
12683 kill_and_reap_rejected_child(
12684 &mut child,
12685 #[cfg(unix)]
12686 rejected_group,
12687 );
12688 #[cfg(target_os = "linux")]
12689 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12690 remove_module_cgroup(placement, &cgroup_name);
12691 }
12692 Err(SuperviseError::Spawn {
12693 program: spec.program.clone(),
12694 source: io::Error::other(format!(
12695 "could not durably record module child: {source}"
12696 )),
12697 cgroup_path,
12698 })
12699 });
12700 }
12701 };
12702 #[cfg(unix)]
12703 if admitted.is_some() {
12704 if let Some(image) = recorded_image {
12705 if let Err(source) = roster_guard.confirm_executable(image) {
12706 kill_and_reap_rejected_child(&mut child, group_target.clone());
12707 return Err(SuperviseError::Spawn {
12708 program: spec.program.clone(),
12709 source,
12710 cgroup_path,
12711 });
12712 }
12713 }
12714 }
12715 if roster.is_closed() {
12720 if carrying {
12721 #[cfg(unix)]
12722 let rejected_group = admitted
12723 .as_ref()
12724 .map(|admitted| admitted.rejection_target())
12725 .unwrap_or_else(|| group_target.clone());
12726 #[cfg(windows)]
12727 if let Some(job) = &job {
12728 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12729 }
12730 kill_and_reap_rejected_child(
12731 &mut child,
12732 #[cfg(unix)]
12733 rejected_group,
12734 );
12735 #[cfg(all(test, unix))]
12736 let _ =
12737 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12738 drop(roster_guard);
12739 return Err(SuperviseError::Spawn {
12740 program: spec.program.clone(),
12741 source: io::Error::other(
12742 "the daemon began shutting down during carried admission; ended the container",
12743 ),
12744 cgroup_path,
12745 });
12746 }
12747 #[cfg(target_os = "linux")]
12749 kill_module_cgroup(cgroup_placement, &cgroup_name);
12750 if let Err(error) = child.start_kill() {
12751 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12752 }
12753 #[cfg(target_os = "linux")]
12754 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12755 while matches!(child.try_wait(), Ok(None)) {
12759 std::thread::yield_now();
12760 }
12761 if matches!(
12762 subc_cgroup::kill_module(Some(placement), &cgroup_name),
12763 subc_cgroup::KillOutcome::Killed
12764 ) {
12765 if let Ok(path) = placement.module_path(&cgroup_name) {
12766 while std::fs::read_to_string(path.join("cgroup.events"))
12767 .ok()
12768 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12769 {
12770 std::thread::yield_now();
12771 }
12772 }
12773 }
12774 remove_module_cgroup(placement, &cgroup_name);
12775 }
12776 drop(roster_guard);
12777 return Err(SuperviseError::Spawn {
12778 program: spec.program.clone(),
12779 source: io::Error::other(
12780 "the daemon began shutting down while this process was starting; ended it",
12781 ),
12782 cgroup_path,
12783 });
12784 }
12785
12786 let stdout_pump = match child.stdout.take() {
12787 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12788 None => {
12789 warn!(
12790 module_id = %spec.module_id,
12791 "spawned child exposed no stdout pipe; file capture will be incomplete"
12792 );
12793 None
12794 }
12795 };
12796 let stderr_pump = match child.stderr.take() {
12797 Some(stderr) => {
12798 let generation = ring
12799 .lock()
12800 .unwrap_or_else(|poisoned| poisoned.into_inner())
12801 .begin_process();
12802 Some(StderrPump {
12803 task: tokio::spawn(pump_stderr_to(
12804 stderr,
12805 Arc::clone(ring),
12806 generation,
12807 output_sink,
12808 )),
12809 generation,
12810 })
12811 }
12812 None => {
12813 ring.lock()
12817 .unwrap_or_else(|poisoned| poisoned.into_inner())
12818 .mark_not_captured("stderr pipe was not available on spawn");
12819 warn!(
12820 module_id = %spec.module_id,
12821 "spawned child exposed no stderr pipe; tail will be unavailable"
12822 );
12823 None
12824 }
12825 };
12826
12827 let exec = Arc::new(crate::activity::ExecOutcome::new());
12828 #[cfg(target_os = "macos")]
12829 let pending_exec = privacy_exec.is_some();
12830 #[cfg(not(target_os = "macos"))]
12831 let pending_exec = false;
12832 if !pending_exec {
12833 exec.finish(crate::activity::ExecState::CommittedExec);
12834 }
12835 #[cfg(windows)]
12836 let subtree_target = job
12837 .as_ref()
12838 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12839 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12840 #[cfg(target_os = "linux")]
12841 let subtree_target = cgroup_path
12842 .map(crate::subtree_resources::Target::Cgroup)
12843 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12844 #[cfg(not(any(target_os = "linux", windows)))]
12845 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12846 let mut supervised = SupervisedChild {
12847 registration_spawn,
12848 #[cfg(any(unix, windows))]
12849 registration_channel,
12850 live_program_binding: None,
12851 registered_config_connection: None,
12852 admission_base,
12853 registration_declaration,
12854 _program_binding: binding_publication,
12855 subtree_target,
12856 child: containment::ProcessChild::plain(child, pid),
12857 exec: crate::activity::ExecOwner(exec),
12858 #[cfg(windows)]
12859 nonce_delivery,
12860 protocol: spec.protocol,
12861 #[cfg(target_os = "linux")]
12862 module_id: cgroup_name,
12863 #[cfg(target_os = "linux")]
12864 cgroup_placement: cgroup_placement.cloned(),
12865 #[cfg(windows)]
12866 job,
12867 stdout_pump,
12868 stderr_pump,
12869 stderr_ring: Arc::clone(ring),
12870 spawned_at,
12871 config_delivery,
12872 registration_deadline: None,
12873 registration_watch: None,
12874 spawned_at_ms,
12875 spawned_from,
12876 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12877 spawned_file_identity,
12878 process_start_time,
12879 process_identity,
12880 pid,
12881 roster_guard: Some(roster_guard),
12882 #[cfg(target_os = "macos")]
12883 privacy_exec,
12884 #[cfg(target_os = "macos")]
12885 report_ready: Arc::new(OnceLock::new()),
12886 spawn_failure: None,
12887 admission_kill_sent: false,
12888 };
12889 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
12890 supervised.child.reject(
12891 #[cfg(unix)]
12892 group_target,
12893 );
12894 #[cfg(all(test, unix))]
12895 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12896 return Err(error);
12897 }
12898 if let (Some(launch), Some(owner), Some(base)) = (
12899 program_launch,
12900 config_owner,
12901 supervised.admission_base.clone(),
12902 ) {
12903 supervised.live_program_binding =
12904 content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
12905 if let (Some(publication), Some(live)) = (
12906 &supervised._program_binding,
12907 &supervised.live_program_binding,
12908 ) {
12909 publication.attach(live);
12910 }
12911 }
12912 #[cfg(unix)]
12913 if let Some(admitted) = admitted.take() {
12914 admitted.disarm();
12915 }
12916 #[cfg(windows)]
12917 if let Err(error) = resume_suspended_child(pid, spec) {
12918 containment::terminate_blocking(&roster.containment);
12919 let _ = supervised.child.start_kill();
12920 return Err(error);
12921 }
12922 if binding_ready {
12923 let checked = if unavailable_initial_image {
12927 Err(crate::config_gate::GateError::new(
12928 "config_delivery_deferred",
12929 "initial process image is unavailable",
12930 ))
12931 } else {
12932 match (
12933 &supervised._program_binding,
12934 &supervised.live_program_binding,
12935 ) {
12936 (Some(binding), _) => binding.verify(),
12937 (None, Some(live)) => live.check(),
12938 _ if supervised.admission_base.is_some() => {
12939 Err(crate::config_gate::GateError::new(
12940 "config_delivery_deferred",
12941 "live program witness is unavailable",
12942 ))
12943 }
12944 _ => Ok(()),
12945 }
12946 };
12947 #[cfg(all(test, target_os = "linux"))]
12948 let checked = if roster.admission_event_for_test(
12949 &spec.module_id,
12950 "after_content_witness_check",
12951 Some(pid),
12952 ) {
12953 checked
12954 } else {
12955 Err(crate::config_gate::GateError::new(
12956 "config_delivery_deferred",
12957 "current process image is unavailable",
12958 ))
12959 };
12960 if let Err(error) = checked {
12961 if let Some(binding) = &mut supervised._program_binding {
12962 binding.reject();
12963 }
12964 if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
12967 supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
12968 #[cfg(all(test, target_os = "linux"))]
12969 assert!(roster.admission_event_for_test(
12970 &spec.module_id,
12971 if unavailable_initial_image {
12972 "before_initial_image_kill"
12973 } else {
12974 "before_content_witness_kill"
12975 },
12976 Some(pid),
12977 ));
12978 let _ = supervised.start_admission_kill();
12979 #[cfg(all(test, target_os = "linux"))]
12980 if supervised.admission_kill_sent {
12981 assert!(roster.admission_event_for_test(
12982 &spec.module_id,
12983 "after_admission_kill_sent",
12984 Some(pid),
12985 ));
12986 }
12987 }
12988 }
12989 }
12990 if let Some(registration) = delivery_registration {
12991 registration.commit();
12992 }
12993 if let Some(spawn) = &supervised.registration_spawn {
12994 if let Some(identity) = supervised.process_identity {
12995 let _ = spawn.identity.set(identity);
12996 }
12997 if supervised.spawn_failure.is_some() {
12998 spawn.refuse();
12999 } else if binding_ready {
13000 spawn.ready();
13001 }
13002 #[cfg(any(unix, windows))]
13003 if let (Some(handle), Some(channel)) =
13004 (registration_handle, supervised.registration_channel.take())
13005 {
13006 handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
13007 }
13008 }
13009 Ok(supervised)
13010}
13011
13012#[cfg(test)]
13013mod spawn_record_tests {
13014 use super::*;
13015
13016 thread_local! {
13017 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
13018 }
13019
13020 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
13021 LAST_PID.set(child.id());
13022 #[cfg(windows)]
13023 if let Some((_, address)) = _spec
13024 .env
13025 .iter()
13026 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
13027 {
13028 use std::io::{Read, Write};
13029 let mut stream = std::net::TcpStream::connect(address).unwrap();
13030 stream
13031 .write_all(&child.id().unwrap().to_ne_bytes())
13032 .unwrap();
13033 let mut release = [0];
13034 stream.read_exact(&mut release).unwrap();
13035 }
13036 }
13037
13038 #[cfg(windows)]
13039 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
13040 if spec
13041 .env
13042 .iter()
13043 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
13044 {
13045 Err(io::Error::other("injected module job assignment failure"))
13046 } else {
13047 Ok(())
13048 }
13049 }
13050
13051 fn spec() -> ModuleSpec {
13052 ModuleSpec {
13053 module_id: "spawn-record-worker".into(),
13054 #[cfg(unix)]
13055 program: "/bin/sleep".into(),
13056 #[cfg(windows)]
13057 program: std::env::current_exe().unwrap(),
13058 #[cfg(unix)]
13059 args: vec!["60".into()],
13060 #[cfg(windows)]
13061 args: vec![
13062 "--exact".into(),
13063 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
13064 "--nocapture".into(),
13065 ],
13066 env: vec![],
13067 reserved: false,
13068 reserved_prefixes: vec![],
13069 protocol: ModuleProtocol::None,
13070 overlap: Default::default(),
13071 carry_containment: false,
13072 }
13073 }
13074
13075 #[tokio::test]
13076 async fn record_sync_failure_fails_spawn_and_reaps_child() {
13077 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
13078 let supervisor =
13079 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
13080 .with_live_children_record(home.join("live-children.json"));
13081 crate::live_children::fail_next_file_sync();
13082 let result = supervisor.spawn(spec());
13083 let failed = result.is_err();
13084 if let Ok(module) = result {
13085 module.stop().await.unwrap();
13086 }
13087 assert!(failed, "record failure must fail spawn");
13088 let pid = LAST_PID.get().expect("the real OS child was created");
13089 assert!(
13090 subc_os::Process::open(pid).unwrap().is_none(),
13091 "child {pid} survived rejected admission or was not reaped"
13092 );
13093 }
13094
13095 #[cfg(target_os = "macos")]
13096 #[test]
13097 #[ignore = "manual on-device latency measurement"]
13098 fn durable_record_write_latency_50() {
13099 let home = cortexkit_test_support::ScratchDir::new("record-latency");
13100 let path = home.join("live-children.json");
13101 let children = vec![crate::live_children::LiveChild {
13102 module_id: "latency-worker".into(),
13103 pid: std::process::id(),
13104 protocol: ModuleProtocol::None,
13105 start_time: Some(1),
13106 executable: None,
13107 cgroup_name: None,
13108 container: None,
13109 }];
13110 let mut samples = Vec::new();
13111 for _ in 0..50 {
13112 let start = std::time::Instant::now();
13113 crate::live_children::write_record(&path, &children).unwrap();
13114 samples.push(start.elapsed());
13115 }
13116 samples.sort();
13117 eprintln!(
13118 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13119 (samples[24] + samples[25]) / 2,
13120 samples[47]
13121 );
13122 }
13123
13124 #[cfg(windows)]
13125 #[test]
13126 fn windows_store_touch_fixture() {
13127 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13128 std::fs::write(path, b"module ran").unwrap();
13129 }
13130 }
13131
13132 #[cfg(windows)]
13133 #[tokio::test]
13134 async fn windows_daemon_fixture() {
13135 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13136 return;
13137 };
13138 let mut spec = spec();
13139 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13140 let supervisor =
13141 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13142 let _module = supervisor.spawn(spec).unwrap();
13143 }
13144
13145 #[cfg(windows)]
13146 #[test]
13147 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13148 use std::io::Read;
13149 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13150 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13153 command
13154 .args([
13155 "--exact",
13156 "supervise::spawn_record_tests::windows_daemon_fixture",
13157 "--nocapture",
13158 ])
13159 .env(
13160 "SUBC_TEST_DAEMON_BARRIER",
13161 listener.local_addr().unwrap().to_string(),
13162 );
13163 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13164 listener.set_nonblocking(true).unwrap();
13165 let deadline = std::time::Instant::now() + Duration::from_secs(30);
13166 let mut stream = loop {
13167 if let Ok((stream, _)) = listener.accept() {
13168 break stream;
13169 }
13170 assert!(
13171 daemon.child.try_wait().unwrap().is_none(),
13172 "daemon failed before rendezvous"
13173 );
13174 assert!(
13175 std::time::Instant::now() < deadline,
13176 "daemon did not reach CreateProcess failpoint"
13177 );
13178 std::thread::yield_now();
13179 };
13180 stream
13181 .set_read_timeout(Some(Duration::from_secs(30)))
13182 .unwrap();
13183 let mut pid = [0; 4];
13184 stream.read_exact(&mut pid).unwrap();
13185 let pid = u32::from_ne_bytes(pid);
13186 assert!(subc_jobobject::process_exists(pid));
13187 daemon.child.kill().unwrap();
13190 daemon.child.wait().unwrap();
13191 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13192 if !exited {
13193 daemon.job.terminate().unwrap();
13194 }
13195 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13196 }
13197
13198 #[cfg(windows)]
13199 #[tokio::test]
13200 async fn windows_containment_failure_never_resumes() {
13201 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13202 let sentinel = home.join("store-touched");
13203 let mut spec = spec();
13204 spec.env
13205 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13206 spec.env.push((
13207 "SUBC_TEST_STORE_TOUCH".into(),
13208 sentinel.display().to_string(),
13209 ));
13210 let supervisor =
13211 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13212 let result = supervisor.spawn(spec);
13213 if let Ok(module) = result {
13216 let _ = tokio::time::timeout(Duration::from_secs(10), async {
13217 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13218 tokio::task::yield_now().await;
13219 }
13220 })
13221 .await;
13222 let _ = module.stop().await;
13223 panic!(
13224 "assignment failure resumed a module; store touched={}",
13225 sentinel.exists()
13226 );
13227 }
13228 assert!(
13229 !sentinel.exists(),
13230 "refused module executed store-touch sentinel"
13231 );
13232 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13233 }
13234
13235 #[cfg(windows)]
13236 #[tokio::test]
13237 async fn windows_nested_daemon_fixture() {
13238 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13239 return;
13240 };
13241 let mut spec = spec();
13242 spec.env.push((
13243 "SUBC_TEST_STORE_TOUCH".into(),
13244 sentinel.to_str().unwrap().into(),
13245 ));
13246 let supervisor =
13247 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13248 let module = supervisor
13249 .spawn(spec)
13250 .expect("per-module job must nest below daemon and host jobs");
13251 tokio::time::timeout(Duration::from_secs(10), async {
13252 while !PathBuf::from(&sentinel).exists() {
13253 tokio::task::yield_now().await;
13254 }
13255 })
13256 .await
13257 .unwrap();
13258 module.stop().await.unwrap();
13259 }
13260
13261 #[cfg(windows)]
13262 #[test]
13263 fn windows_module_job_nests_under_host_job() {
13264 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13265 let sentinel = home.join("nested-store-touched");
13266 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13267 command
13268 .args([
13269 "--exact",
13270 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13271 "--nocapture",
13272 ])
13273 .env("SUBC_TEST_NESTED_STORE", &sentinel);
13274 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13275 assert!(daemon.child.wait().unwrap().success());
13276 assert!(sentinel.exists(), "nested protocol:none module never ran");
13277 }
13278}
13279
13280#[cfg(target_os = "linux")]
13281pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13282 use subc_cgroup::KillOutcome;
13283 match subc_cgroup::kill_module(placement, module_id) {
13284 KillOutcome::Killed => {}
13285 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13286 debug!(
13287 module_id,
13288 "cgroup tree kill unavailable; using direct-child kill"
13289 );
13290 }
13291 KillOutcome::IoError { path, error } => {
13292 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13293 }
13294 }
13295}
13296
13297#[cfg(windows)]
13305fn contain_spawned_child(
13306 child: &mut Child,
13307 spec: &ModuleSpec,
13308 carried: Option<&Arc<containment::CarriedContainment>>,
13309) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13310 let assignment = || {
13311 let job = match carried {
13312 Some(container) => container.job(),
13313 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13314 };
13315 #[cfg(test)]
13316 spawn_record_tests::assignment_failpoint(spec)?;
13317 job.lock()
13318 .unwrap_or_else(|p| p.into_inner())
13319 .assign(child)?;
13320 Ok::<_, io::Error>(job)
13321 };
13322 match assignment() {
13323 Ok(job) => Ok(Some(job)),
13324 Err(source) => {
13325 kill_and_reap_rejected_child(child);
13326 Err(SuperviseError::Spawn {
13327 program: spec.program.clone(),
13328 source: io::Error::other(format!(
13329 "Windows module job containment failed: {source}"
13330 )),
13331 cgroup_path: None,
13332 })
13333 }
13334 }
13335}
13336
13337fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13340 #[cfg(unix)]
13341 group.signal_rejected(child);
13342 let _ = child.start_kill();
13343 while matches!(child.try_wait(), Ok(None)) {
13344 std::thread::yield_now();
13345 }
13346}
13347
13348#[cfg(windows)]
13353fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13354 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13355 let _ = subc_os::child_spawn::status(
13359 std::process::Command::new("taskkill.exe")
13360 .args(["/PID", &pid.to_string(), "/T", "/F"])
13361 .stdin(Stdio::null())
13362 .stdout(Stdio::null())
13363 .stderr(Stdio::null()),
13364 );
13365 return Err(SuperviseError::Spawn {
13366 program: spec.program.clone(),
13367 source,
13368 cgroup_path: None,
13369 });
13370 }
13371 Ok(())
13372}
13373
13374#[cfg(target_os = "linux")]
13375fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13376 match placement.remove_module(module_id) {
13377 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13378 Err(error) => warn!(
13379 module_id,
13380 error = %error,
13381 "could not remove module cgroup after process exit; continuing teardown"
13382 ),
13383 }
13384}
13385
13386#[cfg(target_os = "linux")]
13387const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13388#[cfg(target_os = "linux")]
13389const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13390
13391#[cfg(target_os = "linux")]
13392async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13393 let path = placement.module_path(module_id).ok();
13394 if path.as_ref().is_some_and(|path| {
13395 std::fs::read_to_string(path.join("cgroup.events"))
13396 .ok()
13397 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13398 }) {
13399 remove_module_cgroup(placement, module_id);
13400 return;
13401 }
13402
13403 if terminate_reaped_cgroup(placement, module_id).await {
13406 if let Some(path) = &path {
13407 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13408 while cgroup_is_populated(path) && Instant::now() < deadline {
13409 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13410 }
13411 }
13412 }
13413
13414 if matches!(
13418 subc_cgroup::kill_module(Some(placement), module_id),
13419 subc_cgroup::KillOutcome::Killed
13420 ) {
13421 if let Some(path) = &path {
13422 while cgroup_is_populated(path) {
13423 sleep(Duration::from_millis(1)).await;
13424 }
13425 }
13426 }
13427 remove_module_cgroup(placement, module_id);
13428}
13429
13430#[cfg(target_os = "linux")]
13431fn cgroup_is_populated(path: &std::path::Path) -> bool {
13432 std::fs::read_to_string(path.join("cgroup.events"))
13433 .ok()
13434 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13435}
13436
13437#[cfg(target_os = "linux")]
13438async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13439 let frozen =
13440 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13441 .await
13442 {
13443 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13444 outcome => {
13445 tracing::debug!(
13446 module_id,
13447 ?outcome,
13448 "cgroup freeze failed; skipping SIGTERM grace"
13449 );
13450 return false;
13451 }
13452 };
13453 let listed = match frozen.processes() {
13454 subc_cgroup::ProcsOutcome::Listed(pids) => {
13455 for pid in pids.into_iter().filter(|pid| *pid > 1) {
13456 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13457 if let Err(error) =
13458 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13459 {
13460 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13461 }
13462 }
13463 }
13464 true
13465 }
13466 outcome => {
13467 tracing::debug!(
13468 module_id,
13469 ?outcome,
13470 "cgroup process listing failed; skipping SIGTERM grace"
13471 );
13472 false
13473 }
13474 };
13475 match frozen.thaw() {
13476 subc_cgroup::ThawOutcome::Thawed => listed,
13477 outcome => {
13478 tracing::debug!(
13479 module_id,
13480 ?outcome,
13481 "cgroup thaw failed; proceeding to cgroup.kill"
13482 );
13483 false
13484 }
13485 }
13486}
13487
13488#[cfg(target_os = "linux")]
13489fn apply_cgroup_placement(
13490 command: &mut Command,
13491 spec: &ModuleSpec,
13492 path: &std::path::Path,
13493) -> Result<(), SuperviseError> {
13494 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13495 module_id: spec.module_id.clone(),
13496 source,
13497 })
13498}
13499
13500fn capture_retention(spec: &ModuleSpec) -> Retention {
13501 let defaults = Retention::default();
13502 let value = |name: &str| {
13503 spec.env
13504 .iter()
13505 .rev()
13506 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13507 };
13508 Retention {
13509 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13510 .and_then(|value| value.parse().ok())
13511 .unwrap_or(defaults.max_file_mb),
13512 keep: value(CAPTURE_KEEP_ENV)
13513 .and_then(|value| value.parse().ok())
13514 .unwrap_or(defaults.keep),
13515 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13516 .and_then(|value| value.parse().ok())
13517 .unwrap_or(defaults.max_age_days),
13518 }
13519}
13520
13521fn generate_launch_nonce() -> Result<String, SuperviseError> {
13524 let mut bytes = [0u8; 32];
13525 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13526 reason: source.to_string(),
13527 })?;
13528 let mut hex = String::with_capacity(64);
13529 for b in bytes {
13530 use std::fmt::Write;
13531 let _ = write!(hex, "{b:02x}");
13532 }
13533 Ok(hex)
13534}
13535
13536fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13539 if a.len() != b.len() {
13540 return false;
13541 }
13542 let mut diff = 0u8;
13543 for (x, y) in a.iter().zip(b.iter()) {
13544 diff |= x ^ y;
13545 }
13546 diff == 0
13547}
13548
13549#[cfg(not(windows))]
13553fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13554 let process = subc_os::Process::open(pid).ok().flatten()?;
13555 #[cfg(target_os = "linux")]
13556 return sample_after_linux_exec(
13557 || {
13558 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13559 let committed = linux_exec_committed(&stat);
13560 #[cfg(test)]
13561 spawn_image_tests::report_exec_state(committed);
13562 committed
13563 },
13564 || process.observe(),
13565 );
13566 #[cfg(not(target_os = "linux"))]
13567 process.observe()
13568}
13569
13570#[cfg(target_os = "linux")]
13574fn linux_exec_committed(stat: &str) -> Option<bool> {
13575 const PF_FORKNOEXEC: u64 = 0x40;
13576 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13577 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13578}
13579
13580#[cfg(target_os = "linux")]
13581fn sample_after_linux_exec(
13582 mut exec_committed: impl FnMut() -> Option<bool>,
13583 observe: impl FnOnce() -> Option<subc_os::Observation>,
13584) -> Option<subc_os::Observation> {
13585 let deadline = std::time::Instant::now() + Duration::from_secs(1);
13586 while !exec_committed()? {
13587 if std::time::Instant::now() >= deadline {
13591 return None;
13592 }
13593 std::thread::sleep(Duration::from_micros(50));
13594 }
13595 observe()
13596}
13597
13598#[cfg(all(test, target_os = "linux"))]
13599mod spawn_image_tests {
13600 use super::*;
13601 use std::cell::{Cell, RefCell};
13602
13603 enum ExecEvent {
13604 Uncommitted,
13605 Image(Option<subc_os::Observation>),
13606 }
13607
13608 thread_local! {
13609 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13610 }
13611
13612 pub(super) fn report_exec_state(committed: Option<bool>) {
13613 if committed == Some(false) {
13614 EXEC_EVENTS.with(|events| {
13615 if let Some(events) = events.borrow_mut().take() {
13616 let _ = events.send(ExecEvent::Uncommitted);
13617 }
13618 });
13619 }
13620 }
13621
13622 #[test]
13623 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13624 use std::io::{Read, Write};
13625 use std::os::fd::AsRawFd;
13626
13627 let descriptor = std::fs::File::open("/dev/null").unwrap();
13628 let mut command = std::process::Command::new("/bin/sleep");
13629 command.arg("30");
13630 let (mut ready, mut release) =
13631 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13632 .unwrap();
13633 let spawn = std::thread::spawn(move || command.spawn());
13634 let mut message = [0u8; 8];
13635 ready.read_exact(&mut message).unwrap();
13636 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13637 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13638 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13639 let (events, received) = std::sync::mpsc::channel();
13640 let observer = std::thread::spawn(move || {
13641 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13642 let image = observe_spawned_image(pid);
13643 let _ = events.send(ExecEvent::Image(image));
13644 });
13645 let first = received.recv_timeout(Duration::from_secs(5));
13649 release.write_all(&[1]).unwrap();
13650 let mut child = spawn.join().unwrap().unwrap();
13651 let after = match first {
13652 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13653 Ok(ExecEvent::Image(image)) => image,
13654 _ => None,
13655 },
13656 Ok(ExecEvent::Image(image)) => image,
13657 Err(_) => None,
13658 };
13659 observer.join().unwrap();
13660 child.kill().unwrap();
13661 child.wait().unwrap();
13662
13663 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13664 assert_eq!(
13665 inherited.unwrap().executable,
13666 subc_os::file_identity(&std::env::current_exe().unwrap())
13667 );
13668 assert_eq!(
13669 after.unwrap().executable,
13670 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13671 );
13672 }
13673
13674 #[test]
13675 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13676 let committed = Cell::new(false);
13677 let inherited = subc_os::Observation {
13678 start_time: 23,
13679 executable: Some(subc_os::FileIdentity {
13680 device: 29,
13681 inode: 31,
13682 }),
13683 };
13684 let module = subc_os::Observation {
13685 start_time: 23,
13686 executable: Some(subc_os::FileIdentity {
13687 device: 29,
13688 inode: 37,
13689 }),
13690 };
13691 let polls = Cell::new(0);
13692 let image = sample_after_linux_exec(
13693 || {
13694 polls.set(polls.get() + 1);
13695 committed.set(polls.get() == 2);
13698 Some(committed.get())
13699 },
13700 || Some(if committed.get() { module } else { inherited }),
13701 );
13702 assert_eq!(image, Some(module));
13703 assert_eq!(polls.get(), 2);
13704 }
13705
13706 #[test]
13707 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13708 assert_eq!(
13709 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13710 Some(false)
13711 );
13712 assert_eq!(
13713 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13714 Some(true)
13715 );
13716 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13717 assert_eq!(linux_exec_committed("17 (a) R"), None);
13718 }
13719
13720 #[test]
13721 fn unreadable_exec_state_never_samples_an_inherited_image() {
13722 assert_eq!(
13723 sample_after_linux_exec(
13724 || None,
13725 || panic!("must not sample before exec confirmation")
13726 ),
13727 None
13728 );
13729 }
13730
13731 #[test]
13732 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13733 assert_eq!(
13734 sample_after_linux_exec(
13735 || Some(false),
13736 || panic!("must not sample after an exec timeout")
13737 ),
13738 None
13739 );
13740 }
13741}
13742
13743#[cfg(windows)]
13746pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13747 if let Err(error) = require_program_search_path(spec) {
13748 return Err(match error {
13749 SuperviseError::Spawn { source, .. } => source,
13750 error => io::Error::other(error),
13751 });
13752 }
13753 let mut program = spec.program.clone();
13754 if program.is_absolute() || program.components().count() > 1 {
13755 match subc_os::ExecutableCapture::open(&program) {
13758 Ok(capture) => return Ok(capture),
13759 Err(error)
13760 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13761 Err(error) => return Err(error),
13762 }
13763 program.set_extension("exe");
13764 return subc_os::ExecutableCapture::open(&program);
13765 }
13766 if program.extension().is_none() {
13767 program.set_extension("exe");
13768 }
13769 let custom_path = spec
13770 .env
13771 .iter()
13772 .rev()
13773 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13774 let mut directories = Vec::new();
13775 if let Some(parent) = std::env::current_exe()?.parent() {
13778 directories.push(parent.to_path_buf());
13779 }
13780 directories.push(std::env::current_dir()?);
13781 if let Some(windows) = std::env::var_os("SystemRoot") {
13782 let windows = PathBuf::from(windows);
13783 directories.push(windows.join("System32"));
13784 directories.push(windows.clone());
13785 }
13786 let path = custom_path
13787 .map(|(_, value)| std::ffi::OsString::from(value))
13788 .or_else(|| std::env::var_os("PATH"));
13789 if let Some(path) = path {
13790 directories.extend(std::env::split_paths(&path));
13791 }
13792 for directory in directories {
13793 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13794 Ok(capture) => return Ok(capture),
13795 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13796 Err(error) => return Err(error),
13797 }
13798 }
13799 Err(io::Error::new(
13800 io::ErrorKind::NotFound,
13801 "resolved executable not found",
13802 ))
13803}
13804
13805async fn spawn_and_mark_running_async(
13806 spec: &ModuleSpec,
13807 runtime: &SupervisorRuntimeConfig,
13808 snapshot: &SharedSnapshot,
13809) -> Result<SupervisedChild, SuperviseError> {
13810 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13811 let registration_watch = watch_registration(&runtime.registry, spec);
13812 #[cfg(unix)]
13813 let joining_group = containment::current(&runtime.child_roster.containment)
13814 .is_some_and(|container| container.group().is_some());
13815 let result = spawn_child_async(spec, runtime).await;
13816 #[cfg(unix)]
13817 let result = if result.is_err()
13818 && joining_group
13819 && !runtime.child_roster.is_closed()
13820 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13821 && runtime
13822 .child_roster
13823 .previous_instance(&spec.module_id)
13824 .is_none()
13825 {
13826 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13829 spawn_child_async(spec, runtime).await
13830 } else {
13831 result
13832 };
13833 let mut child = result?;
13834 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13835 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13836 containment::terminate_blocking(&runtime.child_roster.containment);
13837 return Err(error);
13838 }
13839 if let Some(spawn) = activity_spawn {
13840 spawn.complete(snapshot);
13841 }
13842 Ok(child)
13843}
13844
13845#[cfg(test)]
13846fn spawn_and_mark_running(
13847 spec: &ModuleSpec,
13848 runtime: &SupervisorRuntimeConfig,
13849 snapshot: &SharedSnapshot,
13850) -> Result<SupervisedChild, SuperviseError> {
13851 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13852 let registration_watch = watch_registration(&runtime.registry, spec);
13853 #[cfg(unix)]
13854 let joining_group = containment::current(&runtime.child_roster.containment)
13855 .is_some_and(|container| container.group().is_some());
13856 let spawn = || {
13857 spawn_child(
13858 spec,
13859 runtime.connection_file_path.as_deref(),
13860 runtime.supervisor_handle.as_ref(),
13861 &runtime.stderr_ring,
13862 runtime.capture_logs_dir.as_deref(),
13863 &runtime.child_roster,
13864 runtime.spawn_delivery(),
13865 #[cfg(target_os = "linux")]
13866 runtime.cgroup_placement.as_ref(),
13867 runtime.register_timeout,
13868 )
13869 };
13870 let result = spawn();
13871 #[cfg(unix)]
13872 let result = if result.is_err()
13873 && joining_group
13874 && !runtime.child_roster.is_closed()
13875 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13876 && runtime
13877 .child_roster
13878 .previous_instance(&spec.module_id)
13879 .is_none()
13880 {
13881 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13885 spawn()
13886 } else {
13887 result
13888 };
13889 let child = result?;
13890 let mut child = child;
13891 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13892 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13893 containment::terminate_blocking(&runtime.child_roster.containment);
13894 return Err(error);
13895 }
13896 if let Some(spawn) = activity_spawn {
13897 spawn.complete(snapshot);
13898 }
13899 Ok(child)
13900}
13901
13902struct RegistrationWatch {
13903 changes: watch::Receiver<u64>,
13904 baseline: u64,
13905}
13906
13907fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
13911 if spec.protocol != ModuleProtocol::Subc {
13912 return None;
13913 }
13914 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
13915 let baseline = *changes.borrow();
13916 Some(RegistrationWatch { changes, baseline })
13917}
13918
13919fn arm_registration_deadline(
13920 child: &mut SupervisedChild,
13921 budget: Duration,
13922 registration_watch: Option<RegistrationWatch>,
13923) {
13924 child.registration_watch = registration_watch;
13925 child.registration_deadline = (child.protocol == ModuleProtocol::Subc
13926 && child.awaiting_registration())
13927 .then_some(child.spawned_at + budget);
13928}
13929
13930async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
13931 child.retire_registration();
13932 if let Some(container) = child.child.container() {
13933 container.stop_members().await;
13934 child.kill_leader()?;
13935 child.wait().await?;
13936 return Ok(());
13937 }
13938 #[cfg(unix)]
13939 {
13940 let group = unreaped_registration_group(child)?;
13943 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
13944 sleep(crate::child_roster::TERM_TO_KILL).await;
13949 let group = unreaped_registration_group(child)?;
13950 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
13951 }
13952 child.start_kill()?;
13955 child.wait().await?;
13956 Ok(())
13957}
13958
13959#[cfg(unix)]
13960fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
13961 if let Some(container) = child.child.container() {
13962 return container
13963 .group()
13964 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
13965 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
13966 }
13967 let group = i32::try_from(child.pid)
13970 .ok()
13971 .filter(|pid| *pid > 1)
13972 .and_then(rustix::process::Pid::from_raw)
13973 .ok_or_else(|| {
13974 io::Error::new(
13975 io::ErrorKind::InvalidInput,
13976 "invalid supervised process group",
13977 )
13978 })?;
13979 if child.child.id() != Some(child.pid) {
13980 return Err(io::Error::new(
13981 io::ErrorKind::InvalidInput,
13982 "refusing process-group signal after supervised child was reaped",
13983 ));
13984 }
13985 Ok(group)
13986}
13987
13988enum RegistrationWaitOutcome {
13989 Registered,
13990 Exited(ExitReport),
13991 TimedOut,
13992}
13993
13994struct ReloadRegistrationFailure {
13995 exit_report: ExitReport,
13996 reason: String,
13997}
13998
13999#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14000enum BusyGaugeObservation {
14001 Quiescent,
14002 Busy,
14003 Omitted,
14004}
14005
14006#[derive(Debug, Clone, PartialEq, Eq)]
14008enum GaugeVerdict {
14009 Zero,
14011 NonZero(u64),
14013 NotACount(String),
14017 Missing,
14019 ProbeFailed(&'static str),
14022}
14023
14024impl GaugeVerdict {
14025 fn reads_quiet(&self) -> bool {
14026 matches!(self, Self::Zero)
14027 }
14028}
14029
14030impl fmt::Display for GaugeVerdict {
14031 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14032 match self {
14033 Self::Zero => f.write_str("zero"),
14034 Self::NonZero(value) => write!(f, "non_zero({value})"),
14035 Self::NotACount(value) => write!(f, "not_a_count({value})"),
14036 Self::Missing => f.write_str("missing_from_health_body"),
14037 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
14038 }
14039 }
14040}
14041
14042const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
14045
14046fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
14048 let metrics = metrics.and_then(Value::as_object);
14049 gauges
14050 .iter()
14051 .map(
14052 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
14053 None => GaugeVerdict::Missing,
14054 Some(value) => match value.as_u64() {
14055 Some(0) => GaugeVerdict::Zero,
14056 Some(count) => GaugeVerdict::NonZero(count),
14057 None => {
14058 let mut text = value.to_string();
14059 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
14060 let mut end = GAUGE_VALUE_TEXT_LIMIT;
14061 while !text.is_char_boundary(end) {
14062 end -= 1;
14063 }
14064 text.truncate(end);
14065 text.push_str("...");
14066 }
14067 GaugeVerdict::NotACount(text)
14068 }
14069 },
14070 },
14071 )
14072 .collect()
14073}
14074
14075fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
14078 if verdicts
14079 .iter()
14080 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
14081 {
14082 BusyGaugeObservation::Omitted
14083 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
14084 BusyGaugeObservation::Quiescent
14085 } else {
14086 BusyGaugeObservation::Busy
14087 }
14088}
14089
14090#[derive(Debug, Clone, Default, PartialEq, Eq)]
14095struct GaugeTrace {
14096 latest: Option<(GaugeVerdict, Duration)>,
14097 last_value: Option<(GaugeVerdict, Duration)>,
14098}
14099
14100#[derive(Debug, Clone, Default, PartialEq, Eq)]
14106struct DrainGaugeTrace {
14107 probes: u32,
14109 gauges: Vec<(String, GaugeTrace)>,
14111}
14112
14113impl DrainGaugeTrace {
14114 fn new(gauges: &[String]) -> Self {
14115 Self {
14116 probes: 0,
14117 gauges: gauges
14118 .iter()
14119 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14120 .collect(),
14121 }
14122 }
14123
14124 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14126 self.probes = self.probes.saturating_add(1);
14127 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14128 trace.latest = Some((verdict.clone(), at));
14129 if matches!(
14130 verdict,
14131 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14132 ) {
14133 trace.last_value = Some((verdict.clone(), at));
14134 }
14135 }
14136 }
14137}
14138
14139impl fmt::Display for DrainGaugeTrace {
14144 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14145 if self.gauges.is_empty() {
14146 return f.write_str("none declared");
14147 }
14148 for (index, (name, trace)) in self.gauges.iter().enumerate() {
14149 if index > 0 {
14150 f.write_str(" | ")?;
14151 }
14152 write!(f, "{name}: ")?;
14153 match &trace.latest {
14154 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14155 None => f.write_str("never probed")?,
14156 }
14157 match &trace.last_value {
14158 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14159 None => f.write_str("; no value ever read")?,
14160 }
14161 }
14162 Ok(())
14163 }
14164}
14165
14166pub(crate) fn declared_busy_gauges(
14167 registry: &Registry,
14168 module_id: &str,
14169) -> Result<Vec<String>, SuperviseError> {
14170 busy_gauges_of(
14171 registry
14172 .get_module(module_id)
14173 .map_err(SuperviseError::Registry)?,
14174 )
14175}
14176
14177fn declared_busy_gauges_for_connection(
14181 registry: &Registry,
14182 connection_id: ConnectionId,
14183) -> Result<Vec<String>, SuperviseError> {
14184 busy_gauges_of(
14185 registry
14186 .get_module_by_connection(connection_id)
14187 .map_err(SuperviseError::Registry)?,
14188 )
14189}
14190
14191fn busy_gauges_of(
14192 registration: Option<crate::registry::ModuleRegistration>,
14193) -> Result<Vec<String>, SuperviseError> {
14194 let Some(registration) = registration else {
14195 return Ok(Vec::new());
14196 };
14197 let Some(self_signals) = registration.manifest.self_signals else {
14198 return Ok(Vec::new());
14199 };
14200
14201 let mut gauges = Vec::new();
14202 for declaration in self_signals {
14203 if declaration.kind != SelfSignalKind::Busy {
14204 continue;
14205 }
14206 match declaration.anchored_to {
14207 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14208 gauges.extend(declared)
14209 }
14210 _ => {
14211 gauges.push(String::new());
14214 }
14215 }
14216 }
14217 Ok(gauges)
14218}
14219
14220#[allow(clippy::too_many_arguments)]
14229async fn wait_for_forwarding_quiescence(
14230 forwarding: &ForwardingTable,
14231 module_id: &str,
14232 runtime: &SupervisorRuntimeConfig,
14233 endpoint: crate::ModuleEndpointId,
14234 deadline: Instant,
14235 busy_gauges: &[String],
14236 scope: DrainScope,
14237 trace: &mut DrainGaugeTrace,
14238) -> Result<bool, SuperviseError> {
14239 let wait_started = Instant::now();
14240 let mut gauges_quiescent = busy_gauges.is_empty();
14241 let mut next_probe_at = wait_started;
14242 let mut omission_reported = false;
14243
14244 loop {
14245 let now = Instant::now();
14246 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14247 let report = match scope {
14248 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14249 DrainScope::Endpoint(endpoint) => {
14250 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14251 }
14252 };
14253 let verdicts = match &report {
14254 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14255 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14256 };
14257 trace.record(&verdicts, wait_started.elapsed());
14258 gauges_quiescent = match report {
14259 Ok(_) => match busy_gauge_observation(&verdicts) {
14260 BusyGaugeObservation::Quiescent => true,
14261 BusyGaugeObservation::Busy => false,
14262 BusyGaugeObservation::Omitted => {
14263 if !omission_reported {
14266 forwarding
14267 .counters()
14268 .increment_drains_with_undeclared_gauge();
14269 let missing = busy_gauges
14270 .iter()
14271 .zip(&verdicts)
14272 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14273 .map(|(gauge, _)| gauge.as_str())
14274 .collect::<Vec<_>>()
14275 .join(",");
14276 warn!(
14277 module_id,
14278 missing_gauges = %missing,
14279 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14280 );
14281 omission_reported = true;
14282 }
14283 false
14284 }
14285 },
14286 Err(err) => {
14287 warn!(
14288 module_id,
14289 error = %err,
14290 "drain health.check did not produce declared busy gauges; treating module as busy"
14291 );
14292 false
14293 }
14294 };
14295 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14298 }
14299
14300 let in_flight = forwarding
14301 .endpoint_in_flight_count(endpoint)
14302 .map_err(SuperviseError::Forwarding)?;
14303 if in_flight == 0 && gauges_quiescent {
14304 return Ok(true);
14305 }
14306
14307 let now = Instant::now();
14308 if now >= deadline {
14309 return Ok(false);
14310 }
14311 let mut wait = deadline
14312 .saturating_duration_since(now)
14313 .min(REGISTRY_RELEASE_POLL);
14314 if !busy_gauges.is_empty() {
14315 wait = wait.min(next_probe_at.saturating_duration_since(now));
14316 }
14317 sleep(wait).await;
14318 }
14319}
14320
14321fn drain_busy_reprobe_interval() -> Duration {
14325 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14326}
14327
14328fn warn_drain_timed_out(
14343 module_id: &str,
14344 drain_timeout: Duration,
14345 reason: RouteCloseReason,
14346 holdouts: &crate::forwarding::DrainHoldouts,
14347 gauges: &DrainGaugeTrace,
14348) {
14349 warn!(
14350 module_id = %module_id,
14351 waited = ?drain_timeout,
14352 ?reason,
14353 held_requests = holdouts.requests,
14354 held_routes = holdouts.routes,
14355 total_routes = holdouts.total_routes,
14356 top_connections = ?holdouts.top_connections,
14357 held = %holdouts
14360 .held
14361 .iter()
14362 .map(|(channel, corr)| format!("{channel}:{corr}"))
14363 .collect::<Vec<_>>()
14364 .join(","),
14365 gauge_probes = gauges.probes,
14366 busy_gauges = %gauges,
14367 "route drain timed out before request quiescence; forcing teardown"
14368 );
14369}
14370
14371fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14379 match wait_result {
14380 Ok(drained) => *drained,
14381 Err(_) => false,
14382 }
14383}
14384
14385fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14386 for released in released_routes {
14387 let frame = match Frame::build_with_version(
14388 released.negotiated_ver,
14389 FrameType::Goodbye,
14390 control_flags(),
14391 released.channel,
14392 released.epoch,
14393 0,
14394 Vec::new(),
14395 ) {
14396 Ok(frame) => frame,
14397 Err(err) => {
14398 warn!(
14399 route_channel = released.channel,
14400 error = %err,
14401 "failed to build supervisor drain route GOODBYE frame"
14402 );
14403 continue;
14404 }
14405 };
14406 if !released.close_on_delivery_failure() {
14407 crate::forwarding::send_module_route_goodbye(
14408 &forwarding.counters(),
14409 &released.sink,
14410 frame,
14411 released.module_id.as_deref(),
14412 "supervisor drain",
14413 );
14414 continue;
14415 }
14416 if let Err(err) = released.sink.try_send(frame) {
14417 warn!(
14418 target_connection_id = released.connection_id.get(),
14419 route_channel = released.channel,
14420 error = %err,
14421 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14422 );
14423 let _ = forwarding.escalate_client_delivery_failure(
14424 released.connection_id,
14425 released.channel,
14426 released.epoch,
14427 CloseReason::new(
14428 "route_goodbye_delivery_failed",
14429 format!(
14430 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14431 released.channel
14432 ),
14433 ),
14434 crate::forwarding::UndeliveredFrame {
14435 module_id: released.module_id.as_deref(),
14436 sink: &released.sink,
14437 },
14438 );
14439 }
14440 }
14441}
14442
14443async fn send_module_draining(
14444 module_id: &str,
14445 reason: RouteCloseReason,
14446 deadline_ms: u64,
14447 target: &ModuleDrainTarget,
14448 _wait_for_flush: bool,
14449) -> bool {
14450 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14451 reason,
14452 deadline_ms,
14453 }) {
14454 Ok(body) => body,
14455 Err(err) => {
14456 warn!(
14457 module_id,
14458 error = %err,
14459 "failed to encode module draining command"
14460 );
14461 return false;
14462 }
14463 };
14464 let frame = match Frame::build_with_version(
14465 target.negotiated_ver,
14466 FrameType::Push,
14467 control_flags(),
14468 0,
14469 0,
14470 0,
14471 body,
14472 ) {
14473 Ok(frame) => frame,
14474 Err(err) => {
14475 warn!(
14476 module_id,
14477 error = %err,
14478 "failed to build module draining command frame"
14479 );
14480 return false;
14481 }
14482 };
14483 #[cfg(unix)]
14484 if _wait_for_flush {
14485 return target.sink.send_flushed(frame).await.is_ok();
14486 }
14487 if let Err(err) = target.sink.try_send(frame) {
14488 warn!(
14489 module_id,
14490 target_connection_id = target.endpoint.connection_id.get(),
14491 error = %err,
14492 "module draining command was not delivered to peer"
14493 );
14494 return false;
14495 }
14496 true
14497}
14498
14499fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14501 match Frame::build_with_version(
14502 negotiated_ver,
14503 FrameType::Goodbye,
14504 control_flags(),
14505 0,
14506 0,
14507 0,
14508 Vec::new(),
14509 ) {
14510 Ok(frame) => Some(frame),
14511 Err(err) => {
14512 warn!(
14513 module_id,
14514 error = %err,
14515 "failed to build module GOODBYE frame"
14516 );
14517 None
14518 }
14519 }
14520}
14521
14522#[cfg(unix)]
14536async fn send_module_goodbyes_for_daemon_shutdown(
14537 forwarding: &Arc<ForwardingTable>,
14538 reason: &CloseReason,
14539 wait_for_flush: bool,
14540) {
14541 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14542 let targets = match forwarding.module_connections() {
14543 Ok(targets) => targets,
14544 Err(err) => {
14545 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14546 return;
14547 }
14548 };
14549 let deadline = Instant::now() + GOODBYE_BUDGET;
14550 let mut sends = tokio::task::JoinSet::new();
14551 for target in targets {
14552 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14553 continue;
14554 };
14555 if !wait_for_flush {
14556 if let Err(err) = target.sink.try_send(frame) {
14557 debug!(
14558 module_id = %target.module_id,
14559 error = %err,
14560 "shutdown module GOODBYE was not queued"
14561 );
14562 }
14563 continue;
14564 }
14565 let forwarding = Arc::clone(forwarding);
14566 let reason = reason.clone();
14567 sends.spawn(async move {
14568 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14569 Ok(Ok(())) => {}
14570 Ok(Err(err)) => debug!(
14571 module_id = %target.module_id,
14572 error = %err,
14573 "module connection closed before its shutdown GOODBYE was written"
14574 ),
14575 Err(_) => warn!(
14576 module_id = %target.module_id,
14577 budget = ?GOODBYE_BUDGET,
14578 "shutdown module GOODBYE was not written within its budget; closing anyway"
14579 ),
14580 }
14581 forwarding.request_connection_close(target.endpoint.connection_id, reason);
14582 });
14583 }
14584 while sends.join_next().await.is_some() {}
14586}
14587
14588async fn send_module_goodbye(
14589 module_id: &str,
14590 forwarding: &ForwardingTable,
14591 target: &ModuleDrainTarget,
14592 wait_for_flush: bool,
14593) {
14594 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14595 return;
14596 };
14597 let sent = if wait_for_flush {
14600 timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14601 .await
14602 .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14603 .and_then(|result| result.map_err(|err| err.to_string()))
14604 } else {
14605 target.sink.try_send(frame).map_err(|err| err.to_string())
14606 };
14607 if let Err(err) = sent {
14608 warn!(
14609 module_id,
14610 target_connection_id = target.endpoint.connection_id.get(),
14611 error = %err,
14612 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14613 );
14614 forwarding.request_connection_close(
14615 target.endpoint.connection_id,
14616 CloseReason::new(
14617 "module_goodbye_delivery_failed",
14618 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14619 ),
14620 );
14621 }
14622}
14623
14624#[derive(Clone, Copy)]
14625struct ForwardingDrainContext<'a> {
14626 spec: &'a ModuleSpec,
14627 runtime: &'a SupervisorRuntimeConfig,
14628 registry: &'a Registry,
14629 scope: DrainScope,
14630}
14631
14632#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14634enum DrainScope {
14635 Active,
14638 Endpoint(crate::ModuleEndpointId),
14643}
14644
14645#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14653enum StopNotice {
14654 SentOverConnection,
14657 NoConnection,
14661 NotSent,
14665}
14666
14667async fn begin_forwarding_drain(
14668 spec: &ModuleSpec,
14669 runtime: &SupervisorRuntimeConfig,
14670 registry: &Registry,
14671 snapshot: &SharedSnapshot,
14672 enabled: Option<bool>,
14673 reason: RouteCloseReason,
14674) -> Result<StopNotice, SuperviseError> {
14675 let Some(forwarding) = runtime.forwarding.as_ref() else {
14676 return Err(SuperviseError::ReloadUnavailable {
14677 module_id: spec.module_id.clone(),
14678 reason: "supervisor was not configured with a forwarding table".to_string(),
14679 });
14680 };
14681
14682 begin_forwarding_drain_with(
14683 forwarding,
14684 ForwardingDrainContext {
14685 spec,
14686 runtime,
14687 registry,
14688 scope: DrainScope::Active,
14689 },
14690 snapshot,
14691 enabled,
14692 reason,
14693 runtime.drain_timeout,
14694 )
14695 .await
14696}
14697
14698async fn begin_forwarding_drain_if_configured(
14699 spec: &ModuleSpec,
14700 runtime: &SupervisorRuntimeConfig,
14701 registry: &Registry,
14702 snapshot: &SharedSnapshot,
14703 enabled: Option<bool>,
14704 reason: RouteCloseReason,
14705) -> Result<StopNotice, SuperviseError> {
14706 begin_forwarding_drain_with_timeout(
14707 spec,
14708 runtime,
14709 registry,
14710 snapshot,
14711 enabled,
14712 reason,
14713 runtime.drain_timeout,
14714 )
14715 .await
14716}
14717
14718async fn begin_forwarding_drain_with_timeout(
14722 spec: &ModuleSpec,
14723 runtime: &SupervisorRuntimeConfig,
14724 registry: &Registry,
14725 snapshot: &SharedSnapshot,
14726 enabled: Option<bool>,
14727 reason: RouteCloseReason,
14728 drain_timeout: Duration,
14729) -> Result<StopNotice, SuperviseError> {
14730 let Some(forwarding) = runtime.forwarding.as_ref() else {
14731 return Ok(StopNotice::NotSent);
14732 };
14733
14734 begin_forwarding_drain_with(
14735 forwarding,
14736 ForwardingDrainContext {
14737 spec,
14738 runtime,
14739 registry,
14740 scope: DrainScope::Active,
14741 },
14742 snapshot,
14743 enabled,
14744 reason,
14745 drain_timeout,
14746 )
14747 .await
14748}
14749
14750async fn begin_forwarding_drain_with(
14751 forwarding: &ForwardingTable,
14752 context: ForwardingDrainContext<'_>,
14753 snapshot: &SharedSnapshot,
14754 enabled: Option<bool>,
14755 reason: RouteCloseReason,
14756 drain_timeout: Duration,
14757) -> Result<StopNotice, SuperviseError> {
14758 let ForwardingDrainContext {
14759 spec,
14760 runtime,
14761 registry,
14762 scope,
14763 } = context;
14764 debug_assert_ne!(reason, RouteCloseReason::Crash);
14765 let terminal = matches!(reason, RouteCloseReason::Disable);
14766 let drain_started_at = Instant::now();
14767 let drain_deadline = drain_started_at + drain_timeout;
14768 let deadline_ms =
14769 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14770 let busy_gauges = match scope {
14771 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14772 DrainScope::Endpoint(endpoint) => {
14773 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14774 }
14775 };
14776
14777 let gate_started = Instant::now();
14780 let drain_target = match scope {
14781 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14782 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14783 }
14784 .map_err(SuperviseError::Forwarding)?;
14785 info!(
14790 module_id = %spec.module_id,
14791 ?reason,
14792 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14793 connected = drain_target.is_some(),
14794 "module drain began; route admission closed"
14795 );
14796 if scope == DrainScope::Active {
14797 update_snapshot(snapshot, Some(&spec.module_id), |state| {
14798 state.state = ModuleState::Draining;
14799 state.draining_to_replace =
14800 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14801 if let Some(enabled) = enabled {
14802 state.enabled = enabled;
14803 }
14804 })?;
14805 }
14806
14807 let Some(target) = drain_target.as_ref() else {
14808 return Ok(StopNotice::NoConnection);
14812 };
14813 let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14814 handle
14815 .registration_spawn(target.endpoint.connection_id)
14816 .is_some()
14817 });
14818 {
14819 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14820 #[cfg(feature = "test-support")]
14821 forwarding.wait_before_route_closing_for_test().await;
14822 let routes = forwarding
14823 .endpoint_routes(target.endpoint)
14824 .map_err(SuperviseError::Forwarding)?;
14825 let routes_notified = routes.len();
14826 crate::control::send_route_control_pushes(
14827 forwarding,
14828 routes.clone(),
14829 ClientControlPush::RouteClosing {
14830 module_id: spec.module_id.clone(),
14831 channels: Vec::new(),
14832 reason,
14833 },
14834 );
14835 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14836
14837 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14843 let wait_result = wait_for_forwarding_quiescence(
14844 forwarding,
14845 &spec.module_id,
14846 runtime,
14847 target.endpoint,
14848 drain_deadline,
14849 &busy_gauges,
14850 scope,
14851 &mut gauge_trace,
14852 )
14853 .await;
14854 let drained = drained_after_quiescence_wait(&wait_result);
14855 if let Err(err) = &wait_result {
14856 error!(
14857 module_id = %spec.module_id,
14858 ?reason,
14859 error = %err,
14860 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14861 );
14862 } else if !drained {
14863 let holdouts = forwarding
14865 .endpoint_drain_holdouts(target.endpoint)
14866 .unwrap_or_default();
14867 warn_drain_timed_out(
14868 &spec.module_id,
14869 drain_timeout,
14870 reason,
14871 &holdouts,
14872 &gauge_trace,
14873 );
14874 }
14875 crate::control::send_route_control_pushes(
14876 forwarding,
14877 routes,
14878 ClientControlPush::RouteClosed {
14879 module_id: spec.module_id.clone(),
14880 channels: Vec::new(),
14881 reason,
14882 drained,
14883 abandoned: target.abandoned_bindings.len() as u32,
14884 excluded_subscriptions: target.excluded_subscriptions,
14885 terminal: Some(terminal),
14886 },
14887 );
14888 wait_result?;
14889
14890 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
14896 Ok(routes) => routes,
14897 Err(err) => {
14898 warn!(
14899 module_id = %spec.module_id,
14900 ?reason,
14901 error = %err,
14902 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
14903 );
14904 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14905 return Err(SuperviseError::Forwarding(err));
14906 }
14907 };
14908 let route_goodbye_count = released_routes.len();
14909 send_route_goodbyes(forwarding, released_routes);
14910 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14911
14912 info!(
14918 module_id = %spec.module_id,
14919 ?reason,
14920 routes_notified,
14921 route_goodbyes = route_goodbye_count,
14922 abandoned_reservations = target.abandoned_bindings.len(),
14923 excluded_subscriptions = target.excluded_subscriptions,
14924 drained,
14925 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
14926 );
14927 }
14928
14929 Ok(StopNotice::SentOverConnection)
14930}
14931
14932async fn wait_for_registration_after_reload(
14935 registry: &Registry,
14936 module_id: &str,
14937 snapshot: &SharedSnapshot,
14938 child: &mut SupervisedChild,
14939 wait: Duration,
14940) -> Result<RegistrationWaitOutcome, SuperviseError> {
14941 wait_for_slot_registration(
14942 registry,
14943 crate::registry::RegistrationSlot::Active(module_id),
14944 module_id,
14945 snapshot,
14946 child,
14947 wait,
14948 )
14949 .await
14950}
14951
14952async fn wait_for_slot_registration(
14960 registry: &Registry,
14961 slot: crate::registry::RegistrationSlot<'_>,
14962 module_id: &str,
14963 snapshot: &SharedSnapshot,
14964 child: &mut SupervisedChild,
14965 wait: Duration,
14966) -> Result<RegistrationWaitOutcome, SuperviseError> {
14967 let deadline = Instant::now() + wait;
14968 loop {
14969 if registry
14970 .registration(slot)
14971 .map_err(SuperviseError::Registry)?
14972 .is_some()
14973 {
14974 return Ok(RegistrationWaitOutcome::Registered);
14975 }
14976
14977 let now = Instant::now();
14978 if now >= deadline {
14979 return Ok(RegistrationWaitOutcome::TimedOut);
14980 }
14981 let remaining = deadline.saturating_duration_since(now);
14982 let poll = remaining.min(REGISTRY_RELEASE_POLL);
14983
14984 tokio::select! {
14985 wait_result = child.wait() => {
14986 let status = wait_result.map_err(|source| SuperviseError::Wait {
14987 module_id: module_id.to_string(),
14988 source,
14989 })?;
14990 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
14991 snapshot,
14992 child,
14993 &status,
14994 )));
14995 }
14996 _ = sleep(poll) => {}
14997 }
14998 }
14999}
15000
15001fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
15002 if exit_report.kind != ExitKind::DeliberateSeverance {
15006 exit_report.kind = ExitKind::Crash;
15007 }
15008 exit_report
15009}
15010
15011async fn handle_reload_child_registration_failure(
15012 spec: &ModuleSpec,
15013 runtime: &SupervisorRuntimeConfig,
15014 registry: &Registry,
15015 process_liveness: &SupervisorProcessLiveness,
15016 snapshot: &SharedSnapshot,
15017 _child: &mut Option<SupervisedChild>,
15018 failure: ReloadRegistrationFailure,
15019) -> Result<(), SuperviseError> {
15020 let ReloadRegistrationFailure {
15021 exit_report,
15022 reason,
15023 } = failure;
15024 match on_child_exit(
15025 spec,
15026 runtime.restart_policy,
15027 registry,
15028 snapshot,
15029 &runtime.terminal_ring,
15030 &runtime.spawn_events,
15031 &runtime.child_roster,
15032 exit_report,
15033 runtime.recovery_is_daemon,
15034 )
15035 .await
15036 {
15037 NextAction::Stop {
15038 registration_released,
15039 } => {
15040 if registration_released {
15041 process_liveness.untrack_if_current(&spec.module_id, snapshot);
15042 }
15043 }
15044 NextAction::Restart { schedule } => {
15045 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
15046 schedule.delay
15047 });
15048 if let Some(schedule) = schedule {
15049 log_crash_respawn(&spec.module_id, schedule);
15050 }
15051 schedule_respawn(
15052 runtime,
15053 snapshot,
15054 &spec.module_id,
15055 delay,
15056 RespawnKind::Spawn,
15057 true,
15058 )?;
15059 }
15060 }
15061 Err(SuperviseError::ReloadFailed {
15062 module_id: spec.module_id.clone(),
15063 reason,
15064 })
15065}
15066
15067async fn handle_reload_spawn_failure(
15068 spec: &ModuleSpec,
15069 runtime: &SupervisorRuntimeConfig,
15070 process_liveness: &SupervisorProcessLiveness,
15071 snapshot: &SharedSnapshot,
15072 _child: &mut Option<SupervisedChild>,
15073 reason: String,
15074) -> Result<(), SuperviseError> {
15075 let now = Instant::now();
15076 let mut schedule = None;
15077 update_snapshot(snapshot, Some(&spec.module_id), |state| {
15078 clear_current_process_facts(state);
15079 if state.enabled {
15080 schedule = state.next_crash_restart(&runtime.restart_policy, now);
15081 state.state = if schedule.is_some() {
15082 ModuleState::Restarting
15083 } else {
15084 if runtime
15085 .failure_context(runtime.recovery_is_daemon)
15086 .may_retry()
15087 {
15088 state.failed_retry.schedule(&spec.module_id);
15089 }
15090 ModuleState::Failed
15091 };
15092 } else {
15093 state.state = ModuleState::Disabled;
15094 }
15095 })?;
15096 if let Some(schedule) = schedule {
15097 schedule_respawn(
15098 runtime,
15099 snapshot,
15100 &spec.module_id,
15101 schedule.delay,
15102 RespawnKind::Spawn,
15103 true,
15104 )?;
15105 } else {
15106 process_liveness.untrack_if_current(&spec.module_id, snapshot);
15107 }
15108 Err(SuperviseError::ReloadFailed {
15109 module_id: spec.module_id.clone(),
15110 reason,
15111 })
15112}
15113
15114fn control_flags() -> Flags {
15115 Flags::new(false, Priority::Passive, false)
15116}
15117
15118#[allow(clippy::too_many_arguments)]
15119async fn drain_optional_child(
15120 module_id: &str,
15121 protocol: ModuleProtocol,
15122 stop_notice: StopNotice,
15123 registry: &Registry,
15124 forwarding: Option<&ForwardingTable>,
15125 snapshot: &SharedSnapshot,
15126 terminal_ring: &Arc<Mutex<TerminalRing>>,
15127 spawn_events: &SpawnEventFeed,
15128 child: &mut Option<SupervisedChild>,
15129 drain_timeout: Duration,
15130 final_state: ModuleState,
15131 enabled: Option<bool>,
15132 failure: &FailureContext,
15133) -> Result<(), SuperviseError> {
15134 if let Some(child) = child.take() {
15135 drain_child_to_state(
15136 module_id,
15137 protocol,
15138 stop_notice,
15139 registry,
15140 forwarding,
15141 snapshot,
15142 terminal_ring,
15143 spawn_events,
15144 child,
15145 drain_timeout,
15146 final_state,
15147 enabled,
15148 failure,
15149 )
15150 .await
15151 } else {
15152 update_snapshot(snapshot, Some(module_id), |state| {
15153 state.state = final_state;
15154 if final_state == ModuleState::Failed {
15155 state.failed_retry.failed_at = Some(Instant::now());
15156 }
15157 if let Some(enabled) = enabled {
15158 state.enabled = enabled;
15159 }
15160 clear_current_process_facts(state);
15161 })?;
15162 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15163 }
15164}
15165
15166#[allow(clippy::too_many_arguments)]
15167async fn drain_child_to_state(
15168 module_id: &str,
15169 _protocol: ModuleProtocol,
15170 stop_notice: StopNotice,
15171 registry: &Registry,
15172 forwarding: Option<&ForwardingTable>,
15173 snapshot: &SharedSnapshot,
15174 terminal_ring: &Arc<Mutex<TerminalRing>>,
15175 spawn_events: &SpawnEventFeed,
15176 mut child: SupervisedChild,
15177 drain_timeout: Duration,
15178 final_state: ModuleState,
15179 enabled: Option<bool>,
15180 failure: &FailureContext,
15181) -> Result<(), SuperviseError> {
15182 let protocol = child.protocol;
15183 child.retire_registration();
15184 update_snapshot(snapshot, Some(module_id), |state| {
15185 state.state = ModuleState::Draining;
15186 state.draining_to_replace = final_state == ModuleState::Restarting;
15187 if let Some(enabled) = enabled {
15188 state.enabled = enabled;
15189 }
15190 })?;
15191
15192 if stop_notice != StopNotice::SentOverConnection {
15203 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15204 info!(
15205 module_id,
15206 pid = child.pid,
15207 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15208 "module has no connection yet; requesting stop by signal"
15209 );
15210 }
15211 request_graceful_stop(module_id, &child);
15212 }
15213
15214 let exit_report = match timeout(drain_timeout, child.wait()).await {
15215 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15216 Ok(Err(source)) => {
15217 fail_after_wait_error(
15218 module_id,
15219 snapshot,
15220 terminal_ring,
15221 spawn_events,
15222 format!("failed to wait during drain: {source}"),
15223 failure,
15224 );
15225 return Err(SuperviseError::Wait {
15226 module_id: module_id.to_string(),
15227 source,
15228 });
15229 }
15230 Err(_) => {
15231 warn!(
15244 module_id,
15245 pid = child.pid,
15246 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15247 reason = ?final_state,
15248 ?stop_notice,
15249 "drain budget expired before the module exited; killing it"
15250 );
15251 let kill =
15252 if final_state == ModuleState::Restarting && child.child.container().is_some() {
15253 child.kill_leader()
15254 } else {
15255 child.start_kill()
15256 };
15257 kill.map_err(|source| {
15258 fail_after_wait_error(
15259 module_id,
15260 snapshot,
15261 terminal_ring,
15262 spawn_events,
15263 format!("failed to kill during drain: {source}"),
15264 failure,
15265 );
15266 SuperviseError::Kill {
15267 module_id: module_id.to_string(),
15268 source,
15269 }
15270 })?;
15271 let status = child.wait().await.map_err(|source| {
15272 fail_after_wait_error(
15273 module_id,
15274 snapshot,
15275 terminal_ring,
15276 spawn_events,
15277 format!("failed to wait after drain kill: {source}"),
15278 failure,
15279 );
15280 SuperviseError::Wait {
15281 module_id: module_id.to_string(),
15282 source,
15283 }
15284 })?;
15285 classify_reaped_child_exit(snapshot, &child, &status)
15286 }
15287 };
15288
15289 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15290 publish_after_terminal_record(
15291 module_id,
15292 || {
15293 record_terminal_with_detail(
15294 module_id,
15295 terminal_ring,
15296 spawn_events,
15297 &exit_report,
15298 terminal_disposition(final_state),
15299 detail,
15300 )
15301 },
15302 || {
15303 update_snapshot(snapshot, Some(module_id), |state| {
15304 state.state = final_state;
15305 if final_state == ModuleState::Failed {
15306 state.failed_retry.failed_at = Some(Instant::now());
15307 }
15308 if let Some(enabled) = enabled {
15309 state.enabled = enabled;
15310 }
15311 clear_current_process_facts(state);
15312 state.last_exit = Some(exit_report.clone());
15313 if exit_report.kind == ExitKind::DeliberateSeverance {
15314 state.lifetime_restarts += 1;
15315 }
15316 })
15317 },
15318 )?;
15319 child.drain_stderr(module_id).await;
15320
15321 let released =
15322 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15323 child
15324 .finish_exit(
15325 released.is_ok()
15326 && final_state == ModuleState::Restarting
15327 && child.spawn_failure.is_none(),
15328 )
15329 .await
15330 .map_err(|source| SuperviseError::Wait {
15331 module_id: module_id.to_owned(),
15332 source,
15333 })?;
15334 released
15335}
15336
15337#[cfg(unix)]
15357fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15358 let Some(pid) = child
15359 .id()
15360 .and_then(|pid| i32::try_from(pid).ok())
15361 .and_then(rustix::process::Pid::from_raw)
15362 else {
15363 debug!(
15364 module_id,
15365 "no pid to signal for teardown; falling through to the drain wait"
15366 );
15367 return;
15368 };
15369 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15370 Ok(()) => debug!(
15371 module_id,
15372 "sent SIGTERM to a module nothing else asked to stop"
15373 ),
15374 Err(err) => debug!(
15375 module_id,
15376 error = %err,
15377 "SIGTERM to module failed; the drain wait and kill still apply"
15378 ),
15379 }
15380}
15381
15382#[cfg(not(unix))]
15390fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15391 debug!(
15392 module_id,
15393 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15394 );
15395}
15396
15397fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15398 match final_state {
15399 ModuleState::Stopped => TerminalDisposition::Stopped,
15400 ModuleState::Disabled => TerminalDisposition::Disabled,
15401 ModuleState::Restarting => TerminalDisposition::Restarting,
15402 ModuleState::Failed => TerminalDisposition::Failed,
15403 ModuleState::Starting
15404 | ModuleState::Running
15405 | ModuleState::Unresponsive
15406 | ModuleState::Draining => {
15407 unreachable!("terminal exits only finish in terminal or restarting states")
15408 }
15409 }
15410}
15411
15412async fn release_dead_registration(
15420 registry: &Registry,
15421 forwarding: Option<&ForwardingTable>,
15422 snapshot: &SharedSnapshot,
15423 module_id: &str,
15424 failure: &FailureContext,
15425) -> Result<(), SuperviseError> {
15426 let result = async {
15427 let registration = registry
15428 .get_module(module_id)
15429 .map_err(SuperviseError::Registry)?;
15430 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15431 Ok(()) => return Ok(()),
15432 Err(SuperviseError::RegistrationStillActive { .. }) => {}
15433 Err(err) => return Err(err),
15434 }
15435 let pid = lock_snapshot(snapshot)?.reaped_pid;
15436 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15437 warn!(
15438 module_id,
15439 pid,
15440 connection_id = registration.connection_id.get(),
15441 "reaped module registration outlived release grace; closing dead connection"
15442 );
15443 forwarding.request_connection_close(
15444 registration.connection_id,
15445 CloseReason::new(
15446 "supervised_process_reaped",
15447 format!("module '{module_id}' pid {pid} exited"),
15448 ),
15449 );
15450 wait_for_slot_registration_release(
15451 registry,
15452 crate::registry::RegistrationSlot::Connection(registration.connection_id),
15453 REGISTRY_RELEASE_TIMEOUT,
15454 )
15455 .await?;
15456 }
15457 wait_for_registration_release(registry, module_id, Duration::ZERO).await
15458 }
15459 .await;
15460 if let Err(err) = &result {
15461 fail_snapshot(snapshot, Some(module_id), None, failure);
15462 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15463 }
15464 result
15465}
15466
15467async fn wait_for_registration_release(
15470 registry: &Registry,
15471 module_id: &str,
15472 wait: Duration,
15473) -> Result<(), SuperviseError> {
15474 wait_for_slot_registration_release(
15475 registry,
15476 crate::registry::RegistrationSlot::Active(module_id),
15477 wait,
15478 )
15479 .await
15480}
15481
15482async fn wait_for_slot_registration_release(
15490 registry: &Registry,
15491 slot: crate::registry::RegistrationSlot<'_>,
15492 wait: Duration,
15493) -> Result<(), SuperviseError> {
15494 let deadline = Instant::now() + wait;
15495 let mut release_events = registration_release_events().subscribe();
15496 let still_active = |registration: &crate::registry::ModuleRegistration| {
15497 SuperviseError::RegistrationStillActive {
15498 module_id: registration.manifest.module_id.clone(),
15499 waited: wait,
15500 }
15501 };
15502 loop {
15503 let _observed_generation = *release_events.borrow_and_update();
15504 let Some(registration) = registry
15505 .registration(slot)
15506 .map_err(SuperviseError::Registry)?
15507 else {
15508 return Ok(());
15509 };
15510
15511 let now = Instant::now();
15512 if now >= deadline {
15513 return Err(still_active(®istration));
15514 }
15515
15516 let remaining = deadline.saturating_duration_since(now);
15517 match timeout(remaining, release_events.changed()).await {
15518 Ok(Ok(())) | Ok(Err(_)) => {}
15519 Err(_) => return Err(still_active(®istration)),
15520 }
15521 }
15522}
15523
15524#[cfg(test)]
15525mod slot_registration_wait_tests {
15526 use super::*;
15527 use crate::registry::{ConnectionId, RegistrationSlot};
15528 use subc_protocol::manifest::ModuleManifest;
15529
15530 #[tokio::test]
15531 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15532 let registry = Arc::new(Registry::default());
15533 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
15534 let runtime = supervisor.runtime_config();
15535 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15536 let spec = ModuleSpec {
15537 module_id: "enable-stale-registration".to_string(),
15538 program: PathBuf::from("/missing/enable-retry-test"),
15539 args: Vec::new(),
15540 env: Vec::new(),
15541 reserved: false,
15542 reserved_prefixes: Vec::new(),
15543 protocol: ModuleProtocol::Subc,
15544 overlap: Default::default(),
15545 carry_containment: false,
15546 };
15547 let connection = ConnectionId::new(90);
15548 registry
15549 .register_with_control_ops(
15550 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15551 1,
15552 connection,
15553 Vec::new(),
15554 )
15555 .unwrap();
15556 let mut child = None;
15557 let err = set_child_enabled(
15558 &spec,
15559 &runtime,
15560 ®istry,
15561 &supervisor.process_liveness,
15562 &snapshot,
15563 &mut child,
15564 true,
15565 &runtime.failure_context(false),
15566 )
15567 .await
15568 .unwrap_err();
15569 assert!(matches!(
15570 err,
15571 SuperviseError::RegistrationStillActive { .. }
15572 ));
15573 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15574 assert!(child.is_none());
15575 registry.deregister_connection(connection).unwrap();
15576 let err = set_child_enabled(
15577 &spec,
15578 &runtime,
15579 ®istry,
15580 &supervisor.process_liveness,
15581 &snapshot,
15582 &mut child,
15583 true,
15584 &runtime.failure_context(false),
15585 )
15586 .await
15587 .unwrap_err();
15588 assert!(
15589 matches!(err, SuperviseError::Spawn { .. }),
15590 "second enable must attempt a spawn: {err}"
15591 );
15592 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15593 }
15594
15595 const INCUMBENT: u64 = 1;
15596 const CANDIDATE: u64 = 2;
15597
15598 fn swapped_registry() -> Arc<Registry> {
15599 let registry = Arc::new(Registry::default());
15600 let manifest = ModuleManifest::builder("m", "0.1.0").build();
15601 registry
15602 .register_with_control_ops(
15603 manifest.clone(),
15604 1,
15605 ConnectionId::new(INCUMBENT),
15606 Vec::new(),
15607 )
15608 .unwrap();
15609 registry
15610 .register_candidate_with_control_ops(
15611 manifest,
15612 1,
15613 ConnectionId::new(CANDIDATE),
15614 Vec::new(),
15615 )
15616 .unwrap();
15617 registry
15618 }
15619
15620 #[tokio::test]
15624 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15625 let registry = swapped_registry();
15626 registry.promote_candidate("m").unwrap().unwrap();
15627
15628 assert!(matches!(
15629 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
15630 Err(SuperviseError::RegistrationStillActive { .. })
15631 ));
15632
15633 assert!(matches!(
15635 wait_for_slot_registration_release(
15636 ®istry,
15637 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15638 Duration::from_millis(50),
15639 )
15640 .await,
15641 Err(SuperviseError::RegistrationStillActive { .. })
15642 ));
15643
15644 let releaser = Arc::clone(®istry);
15645 let release = tokio::spawn(async move {
15646 sleep(Duration::from_millis(20)).await;
15647 releaser
15648 .deregister_connection(ConnectionId::new(INCUMBENT))
15649 .unwrap();
15650 notify_registration_release();
15651 });
15652 wait_for_slot_registration_release(
15653 ®istry,
15654 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15655 Duration::from_secs(5),
15656 )
15657 .await
15658 .expect("the incumbent's own registration is released");
15659 release.await.unwrap();
15660 assert!(registry.get_module("m").unwrap().is_some());
15661 }
15662
15663 #[tokio::test]
15666 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15667 let registry = swapped_registry();
15668 assert!(matches!(
15669 wait_for_slot_registration_release(
15670 ®istry,
15671 RegistrationSlot::Candidate("m"),
15672 Duration::from_millis(50),
15673 )
15674 .await,
15675 Err(SuperviseError::RegistrationStillActive { .. })
15676 ));
15677 registry
15678 .deregister_connection(ConnectionId::new(CANDIDATE))
15679 .unwrap();
15680 wait_for_slot_registration_release(
15681 ®istry,
15682 RegistrationSlot::Candidate("m"),
15683 Duration::from_millis(50),
15684 )
15685 .await
15686 .expect("a candidate slot with no candidate is released");
15687 assert!(registry
15688 .registration(RegistrationSlot::Active("m"))
15689 .unwrap()
15690 .is_some());
15691 }
15692}
15693
15694fn classify_exit(status: &ExitStatus) -> ExitReport {
15695 ExitReport {
15696 kind: if status.success() {
15697 ExitKind::Clean
15698 } else {
15699 ExitKind::Crash
15700 },
15701 code: status.code(),
15702 signal: exit_signal(status),
15703 at_ms: unix_ms_now(),
15704 }
15705}
15706
15707fn wait_error_exit_report() -> ExitReport {
15713 ExitReport {
15714 kind: ExitKind::Crash,
15715 code: None,
15716 signal: None,
15717 at_ms: unix_ms_now(),
15718 }
15719}
15720
15721#[cfg(unix)]
15722fn exit_signal(status: &ExitStatus) -> Option<i32> {
15723 use std::os::unix::process::ExitStatusExt;
15724
15725 status.signal()
15726}
15727
15728#[cfg(not(unix))]
15729fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15730 None
15731}
15732
15733fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15739 update_snapshot(snapshot, Some(module_id), |state| {
15740 state.clear_crash_restarts();
15741 })
15742}
15743
15744fn set_running(
15745 snapshot: &SharedSnapshot,
15746 child: &SupervisedChild,
15747 module_id: &str,
15748 spawn_events: &SpawnEventFeed,
15749) -> Result<(), SuperviseError> {
15750 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15751 module_id: Some(module_id.to_string()),
15752 })?;
15753 state.spawn_generation = spawn_events.emit_spawned(
15754 module_id,
15755 child
15756 .registration_spawn
15757 .as_ref()
15758 .map(|spawn| spawn.authority.spawn_number()),
15759 child.pid,
15760 child.spawned_at_ms,
15761 );
15762 if std::mem::take(&mut state.coalesced_restart_pending) {
15763 let generation = state.spawn_generation;
15764 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15765 }
15766 state.drain_disposition_detail = None;
15767 state.spawn_failure = None;
15768 state.admission_refused = None;
15769 state.config_hold_parked = false;
15770 state.config_hold_daemon_initiated = false;
15771 state.config_hold_approval = None;
15772 state.legacy_registration_spawn =
15773 child.registration_spawn.is_some() && child.admission_base.is_none();
15774 state.registration_declaration = child.registration_declaration.clone();
15775 state.stop_reason = None;
15776 state.config_exit_failed_approval = None;
15777 state.waiting_for_previous_pid = None;
15778 state.waiting_for_previous_detail = None;
15779 state.in_alternate_slot = false;
15782 state.configuration_updated_since_spawn = false;
15783 state.spawned_protocol = Some(child.protocol);
15784 state.state = ModuleState::Running;
15785 state.approved_start_failure = None;
15786 state.enabled = true;
15787 state.process_alive = true;
15788 state.pid = child.id();
15789 state.subtree_target = child.subtree_target.clone();
15790 state.health.status = SupervisorHealthStatus::Unknown;
15793 state.last_probe_evidence = None;
15794 state.health.consecutive_failures = 0;
15795 state.health.detail = None;
15796 state.health.metrics = None;
15797 state.health.last_probe_ms = None;
15798 state.failed_retry.healthy_since = None;
15799 state.failed_retry.failed_at = None;
15800 state.failed_retry.running_since = Some(child.spawned_at);
15801 #[cfg(windows)]
15802 {
15803 state.pipe_consumed = child
15804 .nonce_delivery
15805 .as_ref()
15806 .map(|delivery| delivery.consumed());
15807 }
15808 #[cfg(target_os = "macos")]
15809 {
15810 state.report_ready = Some(Arc::clone(&child.report_ready));
15811 }
15812 state.spawned_at_ms = Some(child.spawned_at_ms);
15813 state.exec_outcome = Some(Arc::clone(&child.exec.0));
15814 state.spawned_from = Some(child.spawned_from.clone());
15815 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15816 state.spawned_file_identity = child.spawned_file_identity.clone();
15817 state.process_start_time = child.process_start_time;
15818 Ok(())
15819}
15820
15821fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15822 state.legacy_registration_spawn = false;
15823 state.registration_declaration = None;
15824 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15825 state.failed_retry.healthy_since = None;
15826 state.failed_retry.running_since = None;
15827 state.waiting_for_previous_pid = None;
15828 state.waiting_for_previous_detail = None;
15829 state.process_alive = false;
15830 state.spawned_protocol = None;
15831 state.pid = None;
15832 #[cfg(windows)]
15833 {
15834 state.pipe_consumed = None;
15835 }
15836 #[cfg(target_os = "macos")]
15837 {
15838 state.report_ready = None;
15839 }
15840 state.spawned_at_ms = None;
15841 state.exec_outcome = None;
15842 state.spawned_from = None;
15843 state.launch_spec_digest = None;
15844 state.spawned_file_identity = None;
15845 state.process_start_time = None;
15846 state.deliberate_severance = None;
15847}
15848
15849#[cfg(test)]
15850fn record_deliberate_severance(
15851 snapshot: &SharedSnapshot,
15852 identity: ProcessIdentity,
15853) -> Result<(), SuperviseError> {
15854 update_snapshot(snapshot, None, |state| {
15855 state.deliberate_severance = Some(identity);
15856 })
15857}
15858
15859fn apply_deliberate_severance_marker(
15860 snapshot: &SharedSnapshot,
15861 exited_identity: Option<ProcessIdentity>,
15862 mut exit_report: ExitReport,
15863) -> ExitReport {
15864 let marker = lock_snapshot(snapshot)
15865 .ok()
15866 .and_then(|mut state| state.deliberate_severance.take());
15867 if marker.is_some() && marker == exited_identity {
15868 exit_report.kind = ExitKind::DeliberateSeverance;
15869 }
15870 exit_report
15871}
15872
15873fn classify_reaped_child_exit(
15874 snapshot: &SharedSnapshot,
15875 child: &SupervisedChild,
15876 status: &ExitStatus,
15877) -> ExitReport {
15878 let _ = update_snapshot(snapshot, None, |state| {
15879 state.reaped_pid = Some(child.pid);
15880 state.spawn_failure = child.spawn_failure.clone();
15881 });
15882 if matches!(
15883 child.spawn_failure.as_deref(),
15884 Some("registration_timeout" | "registration_channel_closed")
15885 ) {
15886 registration_failure_exit_report(classify_exit(status))
15887 } else {
15888 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
15889 }
15890}
15891
15892fn fail_snapshot(
15893 snapshot: &SharedSnapshot,
15894 module_id: Option<&str>,
15895 last_exit: Option<ExitReport>,
15896 failure: &FailureContext,
15897) {
15898 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
15899 state.state = ModuleState::Failed;
15900 state.config_hold_parked = false;
15901 state.config_hold_daemon_initiated = false;
15902 state.config_hold_approval = None;
15903 clear_current_process_facts(state);
15904 if failure.may_retry() && state.enabled {
15905 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
15906 } else if failure.shutdown.is_set() {
15907 state.failed_retry.cancel();
15908 }
15909 if let Some(last_exit) = last_exit {
15910 state.last_exit = Some(last_exit);
15911 }
15912 }) {
15913 error!(error = %err, "failed to mark supervisor state failed");
15914 }
15915}
15916
15917fn update_snapshot(
15918 snapshot: &SharedSnapshot,
15919 module_id: Option<&str>,
15920 update: impl FnOnce(&mut SupervisorSnapshot),
15921) -> Result<(), SuperviseError> {
15922 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15923 module_id: module_id.map(ToOwned::to_owned),
15924 })?;
15925 update(&mut state);
15926 Ok(())
15927}
15928
15929const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
15930
15931fn lock_snapshot_for_control<'a>(
15932 snapshot: &'a SharedSnapshot,
15933 module_id: &str,
15934 caller: &'static str,
15935) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
15936 let started_at = Instant::now();
15937 let guard = lock_snapshot(snapshot)?;
15938 let waited = started_at.elapsed();
15939 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
15940 warn!(
15941 module_id = %module_id,
15942 waited_ms = waited.as_millis() as u64,
15943 caller = %caller,
15944 "slow snapshot lock"
15945 );
15946 }
15947 Ok(guard)
15948}
15949
15950fn lock_snapshot(
15951 snapshot: &SharedSnapshot,
15952) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
15953 snapshot
15954 .lock()
15955 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
15956}
15957
15958#[cfg(test)]
15959mod terminal_history_tests {
15960 include!("supervise_route_open_recovery_tests.rs");
15961 use std::{
15962 path::PathBuf,
15963 sync::Arc,
15964 time::{Duration, Instant},
15965 };
15966
15967 use tokio::{sync::mpsc, time::sleep};
15968
15969 use super::{
15970 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
15971 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
15972 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
15973 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
15974 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
15975 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
15976 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
15977 SupervisorHealthStatus, SupervisorSnapshot,
15978 };
15979 use super::Instant as ClockInstant;
15984 use crate::{
15985 registry::Registry,
15986 terminal_ring::{TerminalRing, TerminalRingConfig},
15987 };
15988 use std::sync::Mutex;
15989 use subc_control::TerminalDisposition;
15990
15991 pub(super) fn fake_aft_stub_path() -> PathBuf {
15996 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
15997 assert!(
15998 path.exists(),
15999 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16000 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16001 path.display()
16002 );
16003 path
16004 }
16005
16006 #[test]
16007 fn reserved_never_spawned_refuses_every_hello() {
16008 let supervisor = SupervisorHandle::default();
16013 supervisor.apply_identity_configuration(&ModuleSpec {
16014 module_id: "never-spawned".to_string(),
16015 program: PathBuf::from("/usr/bin/false"),
16016 args: Vec::new(),
16017 env: Vec::new(),
16018 reserved: true,
16019 reserved_prefixes: Vec::new(),
16020 protocol: ModuleProtocol::Subc,
16021 overlap: Default::default(),
16022 carry_containment: false,
16023 });
16024 assert!(
16025 supervisor
16026 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
16027 .is_err(),
16028 "forged nonce must refuse on a reserved never-spawned id"
16029 );
16030 assert!(
16031 supervisor
16032 .reserved_hello_verification("never-spawned", None)
16033 .is_err(),
16034 "absent nonce must refuse on a reserved never-spawned id"
16035 );
16036 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
16038 supervisor.apply_identity_configuration(&ModuleSpec {
16039 module_id: "never-spawned".to_string(),
16040 program: PathBuf::from("/usr/bin/false"),
16041 args: Vec::new(),
16042 env: Vec::new(),
16043 reserved: true,
16044 reserved_prefixes: Vec::new(),
16045 protocol: ModuleProtocol::Subc,
16046 overlap: Default::default(),
16047 carry_containment: false,
16048 });
16049 assert!(supervisor
16050 .reserved_hello_verification("never-spawned", Some("minted"))
16051 .is_ok());
16052 assert!(supervisor
16053 .reserved_hello_verification("never-spawned", Some("forged"))
16054 .is_err());
16055 }
16056
16057 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
16060 let now = ClockInstant::now();
16061 for _ in 0..count {
16062 state.crash_restarts.push_back(now);
16063 }
16064 }
16065
16066 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
16070 let aged = state
16071 .crash_restarts
16072 .front()
16073 .expect("a crash restart must be recorded before it can be aged")
16074 .checked_sub(window + Duration::from_secs(1))
16075 .expect("the test clock is far enough from its origin to age an instant");
16076 state.crash_restarts[0] = aged;
16077 }
16078
16079 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
16080 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
16081 seed_crash_restarts(&mut state, count);
16082 state
16083 }
16084
16085 #[test]
16086 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
16087 let policy = RestartPolicy::new(3, Duration::ZERO);
16088 let now = ClockInstant::now();
16089 assert!(daemon_will_restart(
16090 &mut snapshot_with_restarts(true, 2),
16091 &policy,
16092 now
16093 ));
16094 assert!(!daemon_will_restart(
16095 &mut snapshot_with_restarts(true, 3),
16096 &policy,
16097 now
16098 ));
16099 assert!(!daemon_will_restart(
16100 &mut snapshot_with_restarts(false, 0),
16101 &policy,
16102 now
16103 ));
16104 }
16105
16106 #[test]
16107 fn crash_restart_backoff_escalates_with_in_window_count() {
16108 let policy = RestartPolicy::new(4, Duration::from_millis(100))
16109 .with_max_backoff(Duration::from_secs(30));
16110 let now = ClockInstant::now();
16111 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16112 let schedules = (0..4)
16113 .map(|_| {
16114 state
16115 .next_crash_restart(&policy, now)
16116 .expect("the test policy allows four crash restarts")
16117 })
16118 .collect::<Vec<_>>();
16119
16120 assert_eq!(
16121 schedules
16122 .iter()
16123 .map(|schedule| schedule.restart_in_window)
16124 .collect::<Vec<_>>(),
16125 vec![0, 1, 2, 3]
16126 );
16127 assert_eq!(
16128 schedules
16129 .iter()
16130 .map(|schedule| schedule.delay)
16131 .collect::<Vec<_>>(),
16132 vec![
16133 Duration::from_millis(100),
16134 Duration::from_secs(1),
16135 Duration::from_secs(10),
16136 Duration::from_secs(30),
16137 ]
16138 );
16139 }
16140
16141 #[test]
16142 fn crash_restart_backoff_resets_after_ring_clear() {
16143 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16144 let now = ClockInstant::now();
16145 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16146 assert_eq!(
16147 state.next_crash_restart(&policy, now).unwrap().delay,
16148 Duration::from_millis(100)
16149 );
16150 assert_eq!(
16151 state.next_crash_restart(&policy, now).unwrap().delay,
16152 Duration::from_secs(1)
16153 );
16154
16155 state.clear_crash_restarts();
16156 let schedule = state
16157 .next_crash_restart(&policy, now)
16158 .expect("a cleared ring must allow another restart");
16159 assert_eq!(schedule.restart_in_window, 0);
16160 assert_eq!(schedule.delay, Duration::from_millis(100));
16161 }
16162
16163 #[test]
16164 fn crash_restart_backoff_ignores_aged_restarts() {
16165 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16166 let now = ClockInstant::now();
16167 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16168 state
16169 .next_crash_restart(&policy, now)
16170 .expect("the first restart is allowed");
16171 state
16172 .next_crash_restart(&policy, now)
16173 .expect("the second restart is allowed");
16174 state.crash_restarts[0] = now
16175 .checked_sub(policy.window + Duration::from_secs(1))
16176 .expect("the fake clock can age a restart past the window");
16177
16178 let schedule = state
16179 .next_crash_restart(&policy, now)
16180 .expect("an aged restart must release its slot");
16181 assert_eq!(schedule.restart_in_window, 1);
16182 assert_eq!(schedule.delay, Duration::from_secs(1));
16183 assert_eq!(state.crash_restarts.len(), 2);
16184 }
16185
16186 #[test]
16190 fn a_budget_spent_before_the_window_no_longer_refuses() {
16191 let policy = RestartPolicy::new(3, Duration::ZERO);
16192 let mut state = snapshot_with_restarts(true, 3);
16193 let now = ClockInstant::now();
16194 assert!(!daemon_will_restart(&mut state, &policy, now));
16195
16196 assert!(daemon_will_restart(
16197 &mut state,
16198 &policy,
16199 now + policy.window + Duration::from_secs(1)
16200 ));
16201 assert!(
16202 state.crash_restarts.is_empty(),
16203 "reading the budget must drop the instants that left the window"
16204 );
16205 }
16206
16207 fn module_with_recovery_snapshot(
16208 state: ModuleState,
16209 enabled: bool,
16210 restart_count: u32,
16211 ) -> SupervisedModule {
16212 let registry = Arc::new(Registry::default());
16213 let supervisor =
16214 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
16215 let runtime = supervisor.runtime_config();
16216 let spec = ModuleSpec {
16217 module_id: "recovery-snapshot".to_string(),
16218 program: fake_aft_stub_path(),
16219 args: Vec::new(),
16220 env: Vec::new(),
16221 reserved: false,
16222 reserved_prefixes: Vec::new(),
16223 protocol: ModuleProtocol::Subc,
16224 overlap: Default::default(),
16225 carry_containment: false,
16226 };
16227 let mut snapshot = SupervisorSnapshot::new(state, enabled);
16228 seed_crash_restarts(&mut snapshot, restart_count);
16229 let (commands, _rx) = mpsc::channel(4);
16232 let configured_enabled = snapshot.enabled;
16233 SupervisedModule {
16234 inner: Arc::new(SupervisedModuleInner {
16235 module_id: spec.module_id.clone(),
16236 registry,
16237 snapshot: Arc::new(Mutex::new(snapshot)),
16238 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16239 spec,
16240 enabled: configured_enabled,
16241 health: runtime.health,
16242 register_timeout: runtime.register_timeout,
16243 restart_policy: runtime.restart_policy,
16244 })),
16245 stderr_ring: runtime.stderr_ring,
16246 terminal_ring: runtime.terminal_ring,
16247 commands,
16248 monitor: Mutex::new(None),
16249 effective_drain_timeout: runtime.effective_drain_timeout,
16250 provenance_probe: supervisor.provenance_probe.clone(),
16251 retained: runtime.child_roster.retained_view(),
16252 }),
16253 }
16254 }
16255
16256 #[cfg(target_os = "linux")]
16257 #[tokio::test]
16258 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16259 let supervisor =
16260 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16261 .with_cgroup_placement(None);
16262 let result = supervisor.spawn(ModuleSpec {
16263 module_id: "no-cgroup-placement".to_string(),
16264 program: fake_aft_stub_path(),
16265 args: Vec::new(),
16266 env: Vec::new(),
16267 reserved: false,
16268 reserved_prefixes: Vec::new(),
16269 protocol: ModuleProtocol::Subc,
16270 overlap: Default::default(),
16271 carry_containment: false,
16272 });
16273
16274 assert!(
16275 result.is_ok(),
16276 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16277 );
16278 }
16279
16280 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16281 async fn undecided_snapshot_uses_shared_restart_predicate() {
16282 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16283 .will_recover_after_connection_loss()
16284 .unwrap());
16285 assert!(
16286 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16287 .will_recover_after_connection_loss()
16288 .unwrap()
16289 );
16290 }
16291
16292 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16293 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16294 assert!(
16295 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16296 .will_recover_after_connection_loss()
16297 .unwrap()
16298 );
16299 }
16300
16301 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16302 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16303 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16304 .will_recover_after_connection_loss()
16305 .unwrap());
16306 assert!(
16307 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16308 .will_recover_after_connection_loss()
16309 .unwrap()
16310 );
16311 }
16312
16313 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16314 async fn warming_snapshot_is_limited_to_startup_phases() {
16315 for state in [
16316 ModuleState::Starting,
16317 ModuleState::Running,
16318 ModuleState::Restarting,
16319 ] {
16320 assert!(
16321 module_with_recovery_snapshot(state, true, 0)
16322 .is_warming()
16323 .unwrap(),
16324 "{state:?} should be warming"
16325 );
16326 }
16327 for state in [
16328 ModuleState::Unresponsive,
16329 ModuleState::Draining,
16330 ModuleState::Stopped,
16331 ModuleState::Failed,
16332 ModuleState::Disabled,
16333 ] {
16334 assert!(
16335 !module_with_recovery_snapshot(state, true, 0)
16336 .is_warming()
16337 .unwrap(),
16338 "{state:?} should not be warming"
16339 );
16340 }
16341 }
16342
16343 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16344 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16345 let registry = Arc::new(Registry::default());
16346 let supervisor =
16347 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
16348 let module = supervisor
16349 .spawn(ModuleSpec {
16350 module_id: "terminal-history".to_string(),
16351 program: fake_aft_stub_path(),
16352 args: Vec::new(),
16353 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16354 reserved: false,
16355 reserved_prefixes: Vec::new(),
16356 protocol: ModuleProtocol::Subc,
16357 overlap: Default::default(),
16358 carry_containment: false,
16359 })
16360 .unwrap();
16361
16362 let deadline = Instant::now() + Duration::from_secs(5);
16363 loop {
16364 let history = module.terminal_history();
16365 if history.entries.len() == 2 {
16366 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16367 assert_eq!(history.dropped, 0);
16368 assert_eq!(
16369 history
16370 .entries
16371 .iter()
16372 .map(|entry| entry.exit_code)
16373 .collect::<Vec<_>>(),
16374 vec![Some(23), Some(23)]
16375 );
16376 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16377 return;
16378 }
16379 assert!(
16380 Instant::now() < deadline,
16381 "module did not retain two terminal exits: {history:?}"
16382 );
16383 sleep(Duration::from_millis(10)).await;
16384 }
16385 }
16386
16387 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16391 async fn disable_during_crash_backoff_cancels_pending_respawn() {
16392 let backoff = Duration::from_secs(2);
16393 let supervisor = Supervisor::new_for_test(
16394 Arc::new(Registry::default()),
16395 RestartPolicy::new(10, backoff),
16396 );
16397 let module = supervisor
16398 .spawn(ModuleSpec {
16399 module_id: "disable-during-backoff".to_string(),
16400 program: fake_aft_stub_path(),
16401 args: Vec::new(),
16402 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16403 reserved: false,
16404 reserved_prefixes: Vec::new(),
16405 protocol: ModuleProtocol::Subc,
16406 overlap: Default::default(),
16407 carry_containment: false,
16408 })
16409 .unwrap();
16410
16411 let deadline = Instant::now() + Duration::from_secs(5);
16413 loop {
16414 if module.status().unwrap().state == ModuleState::Restarting {
16415 break;
16416 }
16417 assert!(
16418 Instant::now() < deadline,
16419 "module never entered the crash backoff"
16420 );
16421 sleep(Duration::from_millis(10)).await;
16422 }
16423
16424 let started = Instant::now();
16425 module.set_enabled(false).await.unwrap();
16426 let waited = started.elapsed();
16427
16428 assert!(
16429 waited < backoff / 2,
16430 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16431 );
16432 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16433
16434 sleep(backoff + Duration::from_millis(500)).await;
16436 let status = module.status().unwrap();
16437 assert_eq!(status.state, ModuleState::Disabled);
16438 assert_eq!(
16439 status.spawn_generation, 1,
16440 "module respawned after the operator disabled it"
16441 );
16442 }
16443
16444 #[cfg(unix)]
16447 fn protocol_none_sigterm_exits_clean_spec(
16448 module_id: &str,
16449 dir: &std::path::Path,
16450 ) -> (ModuleSpec, PathBuf, PathBuf) {
16451 let ready = dir.join("ready");
16452 let marker = dir.join("sigterm");
16453 let spec = ModuleSpec {
16454 module_id: module_id.to_string(),
16455 program: fake_aft_stub_path(),
16456 args: Vec::new(),
16457 env: vec![
16458 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16459 (
16460 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16461 marker.display().to_string(),
16462 ),
16463 (
16464 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16465 ready.display().to_string(),
16466 ),
16467 ],
16468 reserved: false,
16469 reserved_prefixes: Vec::new(),
16470 protocol: ModuleProtocol::None,
16471 overlap: Default::default(),
16472 carry_containment: false,
16473 };
16474 (spec, ready, marker)
16475 }
16476
16477 #[cfg(unix)]
16481 async fn wait_for_file(path: &std::path::Path) {
16482 let deadline = Instant::now() + Duration::from_secs(10);
16483 while !path.exists() {
16484 assert!(
16485 Instant::now() < deadline,
16486 "{} never appeared",
16487 path.display()
16488 );
16489 sleep(Duration::from_millis(10)).await;
16490 }
16491 }
16492
16493 #[cfg(unix)]
16497 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16498 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16499 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16500 let (spec, ready, marker) =
16501 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16502 let supervisor = Supervisor::new_for_test(
16503 Arc::new(Registry::default()),
16504 RestartPolicy::new(3, Duration::ZERO),
16505 );
16506 let module = supervisor.spawn(spec).unwrap();
16507 wait_for_file(&ready).await;
16508 let deadline = Instant::now() + Duration::from_secs(10);
16512 let first_pid = loop {
16513 let status = module.status().unwrap();
16514 if status.state == ModuleState::Running {
16515 if let Some(pid) = status.pid {
16516 break pid;
16517 }
16518 }
16519 assert!(
16520 Instant::now() < deadline,
16521 "a running module must report its pid after exec confirmation: {status:?}"
16522 );
16523 sleep(Duration::from_millis(10)).await;
16524 };
16525
16526 rustix::process::kill_process(
16527 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16528 rustix::process::Signal::TERM,
16529 )
16530 .unwrap();
16531
16532 let deadline = Instant::now() + Duration::from_secs(10);
16533 let respawned = loop {
16534 let status = module.status().unwrap();
16535 if status.state == ModuleState::Running
16536 && status.pid.is_some_and(|pid| pid != first_pid)
16537 {
16538 break status;
16539 }
16540 assert!(
16541 Instant::now() < deadline,
16542 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16543 );
16544 sleep(Duration::from_millis(10)).await;
16545 };
16546 assert_eq!(respawned.spawn_generation, 2);
16547 assert!(
16548 marker.exists(),
16549 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16550 );
16551
16552 let history = module.terminal_history();
16553 assert_eq!(history.entries.len(), 1, "{history:?}");
16554 let entry = &history.entries[0];
16555 assert_eq!(entry.exit_code, Some(0));
16556 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16557 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16558
16559 module.stop().await.unwrap();
16560 }
16561
16562 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16566 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16567 let supervisor = Supervisor::new_for_test(
16568 Arc::new(Registry::default()),
16569 RestartPolicy::new(1, Duration::ZERO),
16570 );
16571 let module = supervisor
16572 .spawn(ModuleSpec {
16573 module_id: "none-clean-exit-budget".to_string(),
16574 program: fake_aft_stub_path(),
16575 args: Vec::new(),
16576 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16577 reserved: false,
16578 reserved_prefixes: Vec::new(),
16579 protocol: ModuleProtocol::None,
16580 overlap: Default::default(),
16581 carry_containment: false,
16582 })
16583 .unwrap();
16584
16585 let deadline = Instant::now() + Duration::from_secs(10);
16588 loop {
16589 let status = module.status().unwrap();
16590 if status.state == ModuleState::Failed {
16591 break;
16592 }
16593 assert!(
16594 Instant::now() < deadline,
16595 "module never exhausted its budget: {status:?} {:?}",
16596 module.terminal_history()
16597 );
16598 sleep(Duration::from_millis(10)).await;
16599 }
16600 let history = module.terminal_history();
16601 assert_eq!(
16602 history
16603 .entries
16604 .iter()
16605 .map(|entry| (entry.exit_code, entry.disposition.clone()))
16606 .collect::<Vec<_>>(),
16607 vec![
16608 (Some(0), TerminalDisposition::Restarting),
16609 (Some(0), TerminalDisposition::Failed),
16610 ]
16611 );
16612 let detail = history.entries[1]
16613 .disposition_detail
16614 .as_deref()
16615 .expect("a budget failure names the budget");
16616 assert!(detail.contains("max_restarts=1"), "{detail}");
16617 assert_eq!(module.status().unwrap().spawn_generation, 2);
16618 }
16619
16620 #[test]
16621 fn restart_budget_failure_is_published_after_its_terminal_record() {
16622 use std::io::{Read, Write};
16623 let supervisor = Supervisor::new_for_test(
16624 Arc::new(Registry::default()),
16625 RestartPolicy::new(0, Duration::ZERO),
16626 );
16627 let runtime = supervisor.runtime_config();
16628 let spec = ModuleSpec {
16629 protocol: ModuleProtocol::None,
16630 ..windowed_crash_spec("budget-publication-order")
16631 };
16632 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16633 ModuleState::Running,
16634 true,
16635 )));
16636 lock_snapshot(&snapshot).unwrap().process_alive = true;
16637 let reaped_snapshot = snapshot.clone();
16638 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16639 let history = runtime.terminal_ring.clone();
16640 let reap = std::thread::spawn(move || {
16641 tokio::runtime::Builder::new_current_thread()
16642 .enable_all()
16643 .build()
16644 .unwrap()
16645 .block_on(on_child_exit(
16646 &spec,
16647 runtime.restart_policy,
16648 &supervisor.registry,
16649 &reaped_snapshot,
16650 &runtime.terminal_ring,
16651 &runtime.spawn_events,
16652 &runtime.child_roster,
16653 ExitReport {
16654 kind: ExitKind::Clean,
16655 code: Some(0),
16656 signal: None,
16657 at_ms: 1,
16658 },
16659 true,
16660 ))
16661 });
16662 let mut reached = [0];
16663 barrier.read_exact(&mut reached).unwrap();
16664 assert_eq!(&reached, b"R");
16665 let (before_state, before_alive) = {
16666 let state = lock_snapshot(&snapshot).unwrap();
16667 (state.state, state.process_alive)
16668 };
16669 let before_history = history.lock().unwrap().snapshot();
16670 barrier.write_all(b"X").unwrap();
16671 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16672 assert!(before_history.entries.is_empty());
16673 assert_ne!(
16674 before_state,
16675 ModuleState::Failed,
16676 "Failed was visible before its terminal record could be written"
16677 );
16678 assert!(before_alive);
16679 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16680 let history = history.lock().unwrap().snapshot();
16681 assert_eq!(history.entries.len(), 1);
16682 assert_eq!(history.entries[0].exit_code, Some(0));
16683 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16684 }
16685
16686 #[test]
16687 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16688 use std::io::{Read, Write};
16689
16690 let supervisor = Supervisor::new_for_test(
16691 Arc::new(Registry::default()),
16692 RestartPolicy::new(3, Duration::ZERO),
16693 );
16694 let mut runtime = supervisor.runtime_config();
16695 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16696 let snapshot = module.inner.snapshot.clone();
16697 lock_snapshot(&snapshot).unwrap().process_alive = true;
16698 runtime.terminal_ring = module.inner.terminal_ring.clone();
16699 let spec = windowed_crash_spec(module.module_id());
16700 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16701 let reaped_snapshot = snapshot.clone();
16702 let reap = std::thread::spawn(move || {
16703 tokio::runtime::Builder::new_current_thread()
16704 .enable_all()
16705 .build()
16706 .unwrap()
16707 .block_on(on_child_exit(
16708 &spec,
16709 runtime.restart_policy,
16710 &supervisor.registry,
16711 &reaped_snapshot,
16712 &runtime.terminal_ring,
16713 &runtime.spawn_events,
16714 &runtime.child_roster,
16715 ExitReport {
16716 kind: ExitKind::Crash,
16717 code: Some(2),
16718 signal: None,
16719 at_ms: 1,
16720 },
16721 true,
16722 ))
16723 });
16724
16725 let mut reached = [0];
16729 barrier.read_exact(&mut reached).unwrap();
16730 assert_eq!(&reached, b"R");
16731 let before_status = module.status().unwrap();
16732 let before_history = module.terminal_history();
16733 let before_recovers = module.will_recover_after_connection_loss().unwrap();
16734 barrier.write_all(b"X").unwrap();
16735 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16736
16737 assert_eq!(before_status.state, ModuleState::Running);
16738 assert!(before_status.process_alive);
16739 assert_eq!(before_status.restart_count, 3);
16740 assert!(before_history.entries.is_empty());
16741 assert!(
16742 before_recovers,
16743 "connection cleanup would report a terminal close for a scheduled crash restart"
16744 );
16745 let restarting = module.status().unwrap();
16746 assert_eq!(restarting.state, ModuleState::Restarting);
16747 assert_eq!(restarting.restart_count, 3);
16748 assert!(module.will_recover_after_connection_loss().unwrap());
16749 let history = module.terminal_history();
16750 assert_eq!(history.entries.len(), 1);
16751 assert_eq!(
16752 history.entries[0].disposition,
16753 TerminalDisposition::Restarting
16754 );
16755
16756 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16759 state.state = ModuleState::Running;
16760 state.respawn_pending = false;
16761 })
16762 .unwrap();
16763 assert!(!module.will_recover_after_connection_loss().unwrap());
16764 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16765 state.enabled = false;
16766 state.respawn_pending = true;
16767 })
16768 .unwrap();
16769 assert!(!module.will_recover_after_connection_loss().unwrap());
16770 }
16771
16772 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16773 use std::net::{TcpListener, TcpStream};
16774 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16775 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16776 let (barrier, _) = listener.accept().unwrap();
16777 for stream in [&writer, &barrier] {
16778 stream
16779 .set_read_timeout(Some(Duration::from_secs(10)))
16780 .unwrap();
16781 stream
16782 .set_write_timeout(Some(Duration::from_secs(10)))
16783 .unwrap();
16784 }
16785 *events.1.lock().unwrap() = Some(writer);
16786 barrier
16787 }
16788
16789 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16790 use std::io::{Read, Write};
16791 let barrier = events.1.lock().unwrap().take();
16792 if let Some(mut barrier) = barrier {
16793 barrier.write_all(b"R").unwrap();
16794 let mut release = [0];
16795 barrier.read_exact(&mut release).unwrap();
16796 assert_eq!(&release, b"X");
16797 }
16798 }
16799
16800 #[test]
16801 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16802 use std::io::{Read, Write};
16803
16804 let supervisor = Supervisor::new_for_test(
16805 Arc::new(Registry::default()),
16806 RestartPolicy::new(0, Duration::ZERO),
16807 );
16808 let mut runtime = supervisor.runtime_config();
16809 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16810 let snapshot = module.inner.snapshot.clone();
16811 runtime.terminal_ring = module.inner.terminal_ring.clone();
16812 let spec = ModuleSpec {
16813 module_id: module.module_id().into(),
16814 program: fake_aft_stub_path(),
16815 protocol: ModuleProtocol::None,
16816 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16817 ..windowed_crash_spec(module.module_id())
16818 };
16819 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16820 let reap = std::thread::spawn(move || {
16821 tokio::runtime::Builder::new_current_thread()
16822 .enable_all()
16823 .build()
16824 .unwrap()
16825 .block_on(async {
16826 let mut child =
16827 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16828 health_restart_child(
16829 &spec,
16830 &runtime,
16831 &supervisor.registry,
16832 &supervisor.process_liveness,
16833 &snapshot,
16834 &mut child,
16835 SupervisorHealthStatus::Failing,
16836 Some("health probe did not answer"),
16837 1,
16838 )
16839 .await
16840 .unwrap();
16841 assert!(child.is_none());
16842 });
16843 });
16844 let mut reached = [0];
16849 barrier.read_exact(&mut reached).unwrap();
16850 assert_eq!(&reached, b"R");
16851 let before_status = module.status().unwrap();
16852 let before_history = module.terminal_history();
16853 barrier.write_all(b"X").unwrap();
16854 reap.join().unwrap();
16855 assert!(before_history.entries.is_empty());
16856 assert_ne!(
16857 before_status.state,
16858 ModuleState::Failed,
16859 "Failed was visible before its terminal record could be written"
16860 );
16861 assert!(
16862 before_status.process_alive,
16863 "process death was visible before its terminal record could be written"
16864 );
16865 let failed = module.status().unwrap();
16866 assert_eq!(failed.state, ModuleState::Failed);
16867 assert!(!failed.process_alive);
16868 let history = module.terminal_history();
16869 assert_eq!(history.entries.len(), 1);
16870 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16871 assert_eq!(
16872 history.entries[0].disposition_detail.as_deref(),
16873 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
16874 );
16875 }
16876
16877 #[test]
16878 fn observed_exits_publish_process_death_after_terminal_history() {
16879 use std::io::{Read, Write};
16880 for (kind, enabled, shutdown, wait_error, expected_state) in [
16881 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
16882 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
16883 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
16884 (
16885 ExitKind::DeliberateSeverance,
16886 true,
16887 false,
16888 false,
16889 ModuleState::Restarting,
16890 ),
16891 #[cfg(unix)]
16892 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
16893 (ExitKind::Crash, true, false, true, ModuleState::Failed),
16894 ] {
16895 let supervisor = Supervisor::new_for_test(
16896 Arc::new(Registry::default()),
16897 RestartPolicy::new(1, Duration::ZERO),
16898 );
16899 let runtime = supervisor.runtime_config();
16900 #[cfg(unix)]
16901 if shutdown {
16902 runtime.child_roster.close();
16903 }
16904 #[cfg(not(unix))]
16905 let _ = shutdown;
16906 let spec = windowed_crash_spec("exit-publication-order");
16907 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16908 ModuleState::Running,
16909 enabled,
16910 )));
16911 lock_snapshot(&snapshot).unwrap().process_alive = true;
16912 let reaped_snapshot = snapshot.clone();
16913 let history = runtime.terminal_ring.clone();
16914 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16915 let reap = std::thread::spawn(move || {
16916 if wait_error {
16917 super::fail_after_wait_error(
16918 &spec.module_id,
16919 &reaped_snapshot,
16920 &runtime.terminal_ring,
16921 &runtime.spawn_events,
16922 "wait failed".into(),
16923 &runtime.failure_context(true),
16924 );
16925 } else {
16926 tokio::runtime::Builder::new_current_thread()
16927 .enable_all()
16928 .build()
16929 .unwrap()
16930 .block_on(on_child_exit(
16931 &spec,
16932 runtime.restart_policy,
16933 &supervisor.registry,
16934 &reaped_snapshot,
16935 &runtime.terminal_ring,
16936 &runtime.spawn_events,
16937 &runtime.child_roster,
16938 ExitReport {
16939 kind,
16940 code: Some(0),
16941 signal: None,
16942 at_ms: 1,
16943 },
16944 true,
16945 ));
16946 }
16947 });
16948 let mut reached = [0];
16949 barrier.read_exact(&mut reached).unwrap();
16950 assert_eq!(&reached, b"R");
16951 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
16952 let before_history = history.lock().unwrap().snapshot();
16953 barrier.write_all(b"X").unwrap();
16954 reap.join().unwrap();
16955 assert!(
16956 before_alive,
16957 "{kind:?}: death was published before recording"
16958 );
16959 assert!(before_history.entries.is_empty());
16960 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
16961 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
16962 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
16963 }
16964 }
16965
16966 #[tokio::test]
16967 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
16968 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
16969 let path = dir.join("terminals.jsonl");
16970 std::fs::create_dir(&path).unwrap();
16971 let supervisor = Supervisor::new_for_test(
16972 Arc::new(Registry::default()),
16973 RestartPolicy::new(0, Duration::ZERO),
16974 )
16975 .with_terminal_journal(path, "health-budget-journal-failure".into());
16976 let runtime = supervisor.runtime_config();
16977 let spec = ModuleSpec {
16978 program: fake_aft_stub_path(),
16979 protocol: ModuleProtocol::None,
16980 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16981 ..windowed_crash_spec("health-budget-journal-failure")
16982 };
16983 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16984 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16985 health_restart_child(
16986 &spec,
16987 &runtime,
16988 &supervisor.registry,
16989 &supervisor.process_liveness,
16990 &snapshot,
16991 &mut child,
16992 SupervisorHealthStatus::Failing,
16993 None,
16994 1,
16995 )
16996 .await
16997 .unwrap();
16998 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16999 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17000 let history = runtime
17001 .terminal_ring
17002 .lock()
17003 .unwrap()
17004 .durable_history(&spec.module_id);
17005 assert!(history.journal_write_failures > 0);
17006 assert_eq!(history.entries.len(), 1);
17007 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17008 }
17009
17010 #[test]
17011 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
17012 let supervisor = Supervisor::new_for_test(
17013 Arc::new(Registry::default()),
17014 RestartPolicy::new(0, Duration::ZERO),
17015 );
17016 let runtime = supervisor.runtime_config();
17017 let spec = ModuleSpec {
17018 program: fake_aft_stub_path(),
17019 protocol: ModuleProtocol::None,
17020 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17021 ..windowed_crash_spec("health-budget-recording-panic")
17022 };
17023 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17024 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17025 tokio::runtime::Builder::new_current_thread()
17026 .enable_all()
17027 .build()
17028 .unwrap()
17029 .block_on(async {
17030 let mut child =
17031 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17032 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17033 health_restart_child(
17034 &spec,
17035 &runtime,
17036 &supervisor.registry,
17037 &supervisor.process_liveness,
17038 &snapshot,
17039 &mut child,
17040 SupervisorHealthStatus::Failing,
17041 None,
17042 1,
17043 )
17044 .await
17045 .unwrap();
17046 });
17047 }));
17048 let panic = result.expect_err("recording must still unwind");
17049 assert_eq!(
17050 panic.downcast_ref::<String>().map(String::as_str),
17051 Some("spawn event sequence exhausted")
17052 );
17053 let state = lock_snapshot(&snapshot).unwrap();
17054 assert_eq!(state.state, ModuleState::Failed);
17055 assert!(!state.process_alive);
17056 assert!(state.last_exit.is_some());
17057 }
17058
17059 #[tokio::test]
17060 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
17061 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
17062 let path = dir.join("terminals.jsonl");
17063 std::fs::create_dir(&path).unwrap();
17064 let supervisor = Supervisor::new_for_test(
17065 Arc::new(Registry::default()),
17066 RestartPolicy::new(0, Duration::ZERO),
17067 )
17068 .with_terminal_journal(path, "budget-journal-failure".into());
17069 let runtime = supervisor.runtime_config();
17070 let spec = windowed_crash_spec("budget-journal-failure");
17071 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17072 assert!(matches!(
17073 on_child_exit(
17074 &spec,
17075 runtime.restart_policy,
17076 &supervisor.registry,
17077 &snapshot,
17078 &runtime.terminal_ring,
17079 &runtime.spawn_events,
17080 &runtime.child_roster,
17081 crash_exit_report(1),
17082 true,
17083 )
17084 .await,
17085 NextAction::Stop { .. }
17086 ));
17087 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17088 let history = runtime
17089 .terminal_ring
17090 .lock()
17091 .unwrap()
17092 .durable_history(&spec.module_id);
17093 assert!(history.journal_write_failures > 0);
17094 assert_eq!(history.entries.len(), 1);
17095 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17096 }
17097
17098 #[test]
17099 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
17100 let supervisor = Supervisor::new_for_test(
17101 Arc::new(Registry::default()),
17102 RestartPolicy::new(0, Duration::ZERO),
17103 );
17104 let runtime = supervisor.runtime_config();
17105 let spec = windowed_crash_spec("budget-recording-panic");
17106 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17107 runtime
17108 .spawn_events
17109 .emit_spawned(&spec.module_id, None, 1, 1);
17110 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17113 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17114 tokio::runtime::Builder::new_current_thread()
17115 .enable_all()
17116 .build()
17117 .unwrap()
17118 .block_on(on_child_exit(
17119 &spec,
17120 runtime.restart_policy,
17121 &supervisor.registry,
17122 &snapshot,
17123 &runtime.terminal_ring,
17124 &runtime.spawn_events,
17125 &runtime.child_roster,
17126 crash_exit_report(1),
17127 true,
17128 ))
17129 }));
17130 let panic = result.err().expect("recording must still unwind");
17131 assert_eq!(
17132 panic.downcast_ref::<String>().map(String::as_str),
17133 Some("spawn event sequence exhausted")
17134 );
17135 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17136 }
17137
17138 #[cfg(unix)]
17141 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17142 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17143 for disable in [false, true] {
17144 let label = if disable {
17145 "none-requested-disable"
17146 } else {
17147 "none-requested-stop"
17148 };
17149 let dir = cortexkit_test_support::ScratchDir::new(label);
17150 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17151 let supervisor = Supervisor::new_for_test(
17152 Arc::new(Registry::default()),
17153 RestartPolicy::new(3, Duration::ZERO),
17154 );
17155 let module = supervisor.spawn(spec).unwrap();
17156 wait_for_file(&ready).await;
17157
17158 if disable {
17159 module.set_enabled(false).await.unwrap();
17160 } else {
17161 module.stop().await.unwrap();
17162 }
17163 assert!(
17164 marker.exists(),
17165 "{label}: the child must have left through its SIGTERM handler with exit 0"
17166 );
17167
17168 sleep(Duration::from_millis(500)).await;
17171 let status = module.status().unwrap();
17172 let expected = if disable {
17173 ModuleState::Disabled
17174 } else {
17175 ModuleState::Stopped
17176 };
17177 assert_eq!(status.state, expected, "{label}");
17178 assert_eq!(
17179 status.spawn_generation, 1,
17180 "{label}: respawned after a requested stop"
17181 );
17182 let history = module.terminal_history();
17183 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17184 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17185 assert_ne!(
17186 history.entries[0].disposition,
17187 TerminalDisposition::Restarting,
17188 "{label}"
17189 );
17190 }
17191 }
17192
17193 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17196 async fn subc_wire_clean_exit_is_still_a_stop() {
17197 let supervisor = Supervisor::new_for_test(
17198 Arc::new(Registry::default()),
17199 RestartPolicy::new(3, Duration::ZERO),
17200 );
17201 let module = supervisor
17202 .spawn(ModuleSpec {
17203 module_id: "wire-clean-exit".to_string(),
17204 program: fake_aft_stub_path(),
17205 args: Vec::new(),
17206 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17207 reserved: false,
17208 reserved_prefixes: Vec::new(),
17209 protocol: ModuleProtocol::Subc,
17210 overlap: Default::default(),
17211 carry_containment: false,
17212 })
17213 .unwrap();
17214
17215 let deadline = Instant::now() + Duration::from_secs(10);
17216 while module.terminal_history().entries.is_empty() {
17217 assert!(Instant::now() < deadline, "module never exited");
17218 sleep(Duration::from_millis(10)).await;
17219 }
17220 sleep(Duration::from_millis(500)).await;
17222 let status = module.status().unwrap();
17223 assert_eq!(status.state, ModuleState::Stopped);
17224 assert_eq!(status.spawn_generation, 1);
17225 let history = module.terminal_history();
17226 assert_eq!(history.entries.len(), 1, "{history:?}");
17227 assert_eq!(history.entries[0].exit_code, Some(0));
17228 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17229 }
17230
17231 #[cfg(unix)]
17232 #[tokio::test]
17233 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17234 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17235 let record = dir.join("live-children.json");
17236 let supervisor = Supervisor::new_for_test(
17237 Arc::new(Registry::default()),
17238 RestartPolicy::new(0, Duration::ZERO),
17239 );
17240 let mut runtime = supervisor.runtime_config();
17241 runtime.child_roster.record_to(record.clone());
17242 let gate = Arc::new(super::ReloadExitRecordGate::default());
17243 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17244 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17245 let spec = ModuleSpec {
17246 module_id: "reload-exit-roster".into(),
17247 program: fake_aft_stub_path(),
17248 args: Vec::new(),
17249 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17250 reserved: false,
17251 reserved_prefixes: Vec::new(),
17252 protocol: ModuleProtocol::Subc,
17253 overlap: Default::default(),
17254 carry_containment: false,
17255 };
17256 let mut child = None;
17257 let reload = super::finish_reload_child(
17258 &spec,
17259 &runtime,
17260 &supervisor.registry,
17261 &supervisor.process_liveness,
17262 &snapshot,
17263 &mut child,
17264 );
17265 tokio::pin!(reload);
17266 tokio::select! {
17267 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17268 _ = gate.reached.notified() => {}
17269 }
17270 assert!(runtime
17271 .terminal_ring
17272 .lock()
17273 .unwrap()
17274 .snapshot()
17275 .entries
17276 .is_empty());
17277 assert_eq!(
17278 crate::live_children::read_record(&record).unwrap().len(),
17279 1,
17280 "shutdown must still wait for the reaped child until its terminal record exists"
17281 );
17282 runtime.child_roster.close();
17283 gate.resume.notify_one();
17284 assert!(reload.await.is_err());
17285 assert!(crate::live_children::read_record(&record)
17286 .unwrap()
17287 .is_empty());
17288 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17289 assert_eq!(history.entries.len(), 1);
17290 assert_eq!(
17291 history.entries[0].disposition,
17292 TerminalDisposition::DaemonShutdown
17293 );
17294 }
17295
17296 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17300 async fn every_restart_increment_path_advances_lifetime_count() {
17301 let supervisor = Supervisor::new_for_test(
17302 Arc::new(Registry::default()),
17303 RestartPolicy::new(1, Duration::ZERO),
17304 );
17305 let runtime = supervisor.runtime_config();
17306 let spec = ModuleSpec {
17307 module_id: "lifetime-increment-path".to_string(),
17308 program: PathBuf::from("/unused/lifetime-increment-path"),
17309 args: Vec::new(),
17310 env: Vec::new(),
17311 reserved: false,
17312 reserved_prefixes: Vec::new(),
17313 protocol: ModuleProtocol::Subc,
17314 overlap: Default::default(),
17315 carry_containment: false,
17316 };
17317
17318 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17319 assert!(matches!(
17320 on_child_exit(
17321 &spec,
17322 runtime.restart_policy,
17323 &supervisor.registry,
17324 &crash_snapshot,
17325 &runtime.terminal_ring,
17326 &runtime.spawn_events,
17327 &runtime.child_roster,
17328 ExitReport {
17329 kind: ExitKind::Crash,
17330 code: Some(1),
17331 signal: None,
17332 at_ms: 1,
17333 },
17334 true,
17335 )
17336 .await,
17337 NextAction::Restart { schedule: _ }
17338 ));
17339 let (crash_restarts, crash_lifetime) = {
17340 let state = lock_snapshot(&crash_snapshot).unwrap();
17341 (state.crash_restarts.len(), state.lifetime_restarts)
17342 };
17343 assert_eq!(crash_restarts, 1);
17344 assert_eq!(crash_lifetime, 1);
17345
17346 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17347 let mut health_child = None;
17348 assert!(matches!(
17349 health_restart_child(
17350 &spec,
17351 &runtime,
17352 &supervisor.registry,
17353 &supervisor.process_liveness,
17354 &health_snapshot,
17355 &mut health_child,
17356 SupervisorHealthStatus::Failing,
17357 None,
17358 2,
17359 )
17360 .await,
17361 Ok(())
17362 ));
17363 assert!(health_child.is_none());
17364 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17365 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17366 let (health_restarts, health_lifetime) = {
17367 let state = lock_snapshot(&health_snapshot).unwrap();
17368 (state.crash_restarts.len(), state.lifetime_restarts)
17369 };
17370 assert_eq!(health_restarts, 1);
17371 assert_eq!(health_lifetime, 1);
17372
17373 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17374 let mut reload_child = None;
17375 assert!(matches!(
17376 handle_reload_spawn_failure(
17377 &spec,
17378 &runtime,
17379 &supervisor.process_liveness,
17380 &reload_snapshot,
17381 &mut reload_child,
17382 "forced reload spawn failure".to_string(),
17383 )
17384 .await,
17385 Err(SuperviseError::ReloadFailed { .. })
17386 ));
17387 let (reload_restarts, reload_lifetime) = {
17388 let state = lock_snapshot(&reload_snapshot).unwrap();
17389 (state.crash_restarts.len(), state.lifetime_restarts)
17390 };
17391 assert_eq!(reload_restarts, 1);
17392 assert_eq!(reload_lifetime, 1);
17393 }
17394
17395 #[tokio::test]
17396 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17397 let supervisor = Supervisor::new_for_test(
17398 Arc::new(Registry::default()),
17399 RestartPolicy::new(3, Duration::ZERO),
17400 );
17401 let runtime = supervisor.runtime_config();
17402 let spec = ModuleSpec {
17403 module_id: "deliberately-severed".to_string(),
17404 program: PathBuf::from("/unused/deliberately-severed"),
17405 args: Vec::new(),
17406 env: Vec::new(),
17407 reserved: false,
17408 reserved_prefixes: Vec::new(),
17409 protocol: ModuleProtocol::Subc,
17410 overlap: Default::default(),
17411 carry_containment: false,
17412 };
17413 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17414 let process = ProcessIdentity {
17415 pid: 41,
17416 start_time: 101,
17417 };
17418 record_deliberate_severance(&snapshot, process).unwrap();
17419 let exit_report = apply_deliberate_severance_marker(
17420 &snapshot,
17421 Some(process),
17422 ExitReport {
17423 kind: ExitKind::Crash,
17424 code: Some(1),
17425 signal: None,
17426 at_ms: 1,
17427 },
17428 );
17429 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17430
17431 assert!(matches!(
17432 on_child_exit(
17433 &spec,
17434 runtime.restart_policy,
17435 &supervisor.registry,
17436 &snapshot,
17437 &runtime.terminal_ring,
17438 &runtime.spawn_events,
17439 &runtime.child_roster,
17440 exit_report,
17441 true,
17442 )
17443 .await,
17444 NextAction::Restart { schedule: _ }
17445 ));
17446 let state = lock_snapshot(&snapshot).unwrap();
17447 assert_eq!(state.lifetime_restarts, 1);
17448 assert_eq!(state.crash_restarts.len(), 0);
17449 }
17450
17451 #[tokio::test]
17452 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17453 let supervisor = Supervisor::new_for_test(
17454 Arc::new(Registry::default()),
17455 RestartPolicy::new(3, Duration::ZERO),
17456 );
17457 let runtime = supervisor.runtime_config();
17458 let spec = ModuleSpec {
17459 module_id: "genuine-crash".to_string(),
17460 program: PathBuf::from("/unused/genuine-crash"),
17461 args: Vec::new(),
17462 env: Vec::new(),
17463 reserved: false,
17464 reserved_prefixes: Vec::new(),
17465 protocol: ModuleProtocol::Subc,
17466 overlap: Default::default(),
17467 carry_containment: false,
17468 };
17469 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17470
17471 assert!(matches!(
17472 on_child_exit(
17473 &spec,
17474 runtime.restart_policy,
17475 &supervisor.registry,
17476 &snapshot,
17477 &runtime.terminal_ring,
17478 &runtime.spawn_events,
17479 &runtime.child_roster,
17480 ExitReport {
17481 kind: ExitKind::Crash,
17482 code: Some(1),
17483 signal: None,
17484 at_ms: 1,
17485 },
17486 true,
17487 )
17488 .await,
17489 NextAction::Restart { schedule: _ }
17490 ));
17491 let state = lock_snapshot(&snapshot).unwrap();
17492 assert_eq!(state.lifetime_restarts, 1);
17493 assert_eq!(state.crash_restarts.len(), 1);
17494 }
17495
17496 fn crash_exit_report(at_ms: u64) -> ExitReport {
17497 ExitReport {
17498 kind: ExitKind::Crash,
17499 code: Some(1),
17500 signal: None,
17501 at_ms,
17502 }
17503 }
17504
17505 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17506 ModuleSpec {
17507 module_id: module_id.to_string(),
17508 program: PathBuf::from("/unused").join(module_id),
17509 args: Vec::new(),
17510 env: Vec::new(),
17511 reserved: false,
17512 reserved_prefixes: Vec::new(),
17513 protocol: ModuleProtocol::Subc,
17514 overlap: Default::default(),
17515 carry_containment: false,
17516 }
17517 }
17518
17519 #[tokio::test]
17525 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17526 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17527 let supervisor = Supervisor::new_for_test(
17528 Arc::new(Registry::default()),
17529 RestartPolicy::new(2, Duration::ZERO),
17530 );
17531 let runtime = supervisor.runtime_config();
17532 let spec = windowed_crash_spec("crash-loop-in-window");
17533 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17534
17535 for attempt in 1..=2 {
17536 assert!(
17537 matches!(
17538 on_child_exit(
17539 &spec,
17540 runtime.restart_policy,
17541 &supervisor.registry,
17542 &snapshot,
17543 &runtime.terminal_ring,
17544 &runtime.spawn_events,
17545 &runtime.child_roster,
17546 crash_exit_report(attempt),
17547 true,
17548 )
17549 .await,
17550 NextAction::Restart { schedule: _ }
17551 ),
17552 "crash {attempt} is inside the budget and must respawn"
17553 );
17554 }
17555
17556 assert!(matches!(
17557 on_child_exit(
17558 &spec,
17559 runtime.restart_policy,
17560 &supervisor.registry,
17561 &snapshot,
17562 &runtime.terminal_ring,
17563 &runtime.spawn_events,
17564 &runtime.child_roster,
17565 crash_exit_report(3),
17566 true,
17567 )
17568 .await,
17569 NextAction::Stop { .. }
17570 ));
17571
17572 {
17573 let state = lock_snapshot(&snapshot).unwrap();
17574 assert_eq!(state.state, ModuleState::Failed);
17575 assert_eq!(state.crash_restarts.len(), 2);
17576 assert_eq!(state.lifetime_restarts, 2);
17577 }
17578
17579 let history = runtime
17580 .terminal_ring
17581 .lock()
17582 .expect("terminal ring is not poisoned")
17583 .snapshot();
17584 let last = history
17585 .entries
17586 .last()
17587 .expect("the refused crash is retained");
17588 assert_eq!(last.disposition, TerminalDisposition::Failed);
17589 assert_eq!(
17590 last.disposition_detail.as_deref(),
17591 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17592 );
17593
17594 let captured = crate::router::test_log::captured_logs(&logs);
17595 assert!(
17596 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17597 "the stop must be logged with its window: {captured}"
17598 );
17599 }
17600
17601 #[tokio::test]
17609 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17610 let supervisor = Supervisor::new_for_test(
17611 Arc::new(Registry::default()),
17612 RestartPolicy::new(2, Duration::ZERO),
17613 );
17614 let runtime = supervisor.runtime_config();
17615 let spec = windowed_crash_spec("crash-across-windows");
17616 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17617
17618 for attempt in 1..=2 {
17619 assert!(matches!(
17620 on_child_exit(
17621 &spec,
17622 runtime.restart_policy,
17623 &supervisor.registry,
17624 &snapshot,
17625 &runtime.terminal_ring,
17626 &runtime.spawn_events,
17627 &runtime.child_roster,
17628 crash_exit_report(attempt),
17629 true,
17630 )
17631 .await,
17632 NextAction::Restart { schedule: _ }
17633 ));
17634 }
17635
17636 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17639 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17640 })
17641 .unwrap();
17642
17643 assert!(
17644 matches!(
17645 on_child_exit(
17646 &spec,
17647 runtime.restart_policy,
17648 &supervisor.registry,
17649 &snapshot,
17650 &runtime.terminal_ring,
17651 &runtime.spawn_events,
17652 &runtime.child_roster,
17653 crash_exit_report(3),
17654 true,
17655 )
17656 .await,
17657 NextAction::Restart { schedule: _ }
17658 ),
17659 "a crash older than the window must not hold a budget slot"
17660 );
17661
17662 let state = lock_snapshot(&snapshot).unwrap();
17663 assert_eq!(state.state, ModuleState::Restarting);
17664 assert_eq!(
17665 state.crash_restarts.len(),
17666 2,
17667 "the aged instant is dropped and the new one takes its place"
17668 );
17669 assert_eq!(
17670 state.lifetime_restarts, 3,
17671 "the ledger counts every restart, including the ones the window forgot"
17672 );
17673 }
17674
17675 #[tokio::test]
17680 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17681 let supervisor = Supervisor::new_for_test(
17682 Arc::new(Registry::default()),
17683 RestartPolicy::new(2, Duration::ZERO),
17684 );
17685 let runtime = supervisor.runtime_config();
17686 let spec = windowed_crash_spec("operator-cleared-budget");
17687 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17688
17689 for attempt in 1..=2 {
17690 assert!(matches!(
17691 on_child_exit(
17692 &spec,
17693 runtime.restart_policy,
17694 &supervisor.registry,
17695 &snapshot,
17696 &runtime.terminal_ring,
17697 &runtime.spawn_events,
17698 &runtime.child_roster,
17699 crash_exit_report(attempt),
17700 true,
17701 )
17702 .await,
17703 NextAction::Restart { schedule: _ }
17704 ));
17705 }
17706
17707 reset_restart_count(&snapshot, &spec.module_id).unwrap();
17708 {
17709 let state = lock_snapshot(&snapshot).unwrap();
17710 assert!(
17711 state.crash_restarts.is_empty(),
17712 "an operator restart returns the full budget"
17713 );
17714 assert_eq!(
17715 state.lifetime_restarts, 2,
17716 "clearing the budget must not unmake the crashes"
17717 );
17718 }
17719
17720 assert!(
17721 matches!(
17722 on_child_exit(
17723 &spec,
17724 runtime.restart_policy,
17725 &supervisor.registry,
17726 &snapshot,
17727 &runtime.terminal_ring,
17728 &runtime.spawn_events,
17729 &runtime.child_roster,
17730 crash_exit_report(3),
17731 true,
17732 )
17733 .await,
17734 NextAction::Restart { schedule: _ }
17735 ),
17736 "the cleared budget must be spendable again"
17737 );
17738 let state = lock_snapshot(&snapshot).unwrap();
17739 assert_eq!(state.crash_restarts.len(), 1);
17740 assert_eq!(state.lifetime_restarts, 3);
17741 }
17742
17743 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17744 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17745 let severed = ProcessIdentity {
17746 pid: 41,
17747 start_time: 101,
17748 };
17749 let successor = ProcessIdentity {
17750 pid: 41,
17751 start_time: 202,
17752 };
17753 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17754 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17755 state.pid = Some(successor.pid);
17756 state.process_start_time = Some(successor.start_time);
17757 })
17758 .unwrap();
17759 assert!(!module.record_deliberate_severance(severed).unwrap());
17760
17761 let exit_report = apply_deliberate_severance_marker(
17762 &module.inner.snapshot,
17763 Some(successor),
17764 ExitReport {
17765 kind: ExitKind::Crash,
17766 code: Some(1),
17767 signal: None,
17768 at_ms: 1,
17769 },
17770 );
17771
17772 assert_eq!(exit_report.kind, ExitKind::Crash);
17773 }
17774
17775 #[tokio::test]
17776 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17777 let registry = Registry::default();
17778 let supervisor = Supervisor::new_for_test(
17779 Arc::new(Registry::default()),
17780 RestartPolicy::new(3, Duration::ZERO),
17781 );
17782 let runtime = supervisor.runtime_config();
17783 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17784 let spec = ModuleSpec {
17785 module_id: "drain-deliberate-severance".to_string(),
17786 program: fake_aft_stub_path(),
17787 args: Vec::new(),
17788 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17789 reserved: false,
17790 reserved_prefixes: Vec::new(),
17791 protocol: ModuleProtocol::Subc,
17792 overlap: Default::default(),
17793 carry_containment: false,
17794 };
17795 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17796 let process = ProcessIdentity {
17797 pid: 41,
17798 start_time: 101,
17799 };
17800 child.process_identity = Some(process);
17801 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17802 state.pid = Some(process.pid);
17803 state.process_start_time = Some(process.start_time);
17804 })
17805 .unwrap();
17806 record_deliberate_severance(&snapshot, process).unwrap();
17807
17808 drain_child_to_state(
17809 &spec.module_id,
17810 spec.protocol,
17811 StopNotice::SentOverConnection,
17814 ®istry,
17815 None,
17816 &snapshot,
17817 &runtime.terminal_ring,
17818 &runtime.spawn_events,
17819 child,
17820 Duration::from_secs(1),
17821 ModuleState::Stopped,
17822 Some(false),
17823 &super::FailureContext::operator(),
17824 )
17825 .await
17826 .unwrap();
17827
17828 let state = lock_snapshot(&snapshot).unwrap();
17829 assert_eq!(
17830 state.last_exit.as_ref().map(|exit| exit.kind),
17831 Some(ExitKind::DeliberateSeverance)
17832 );
17833 assert_eq!(state.lifetime_restarts, 1);
17834 assert_eq!(state.crash_restarts.len(), 0);
17835 drop(state);
17836 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17837 assert_eq!(
17838 history.entries[0].exit_kind,
17839 subc_control::TerminalExitKind::DeliberateSeverance
17840 );
17841 }
17842
17843 #[tokio::test]
17844 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17845 let registry = Registry::default();
17846 let supervisor = Supervisor::new_for_test(
17847 Arc::new(Registry::default()),
17848 RestartPolicy::new(3, Duration::ZERO),
17849 );
17850 let runtime = supervisor.runtime_config();
17851 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17852 let spec = ModuleSpec {
17853 module_id: "ordinary-drain".to_string(),
17854 program: fake_aft_stub_path(),
17855 args: Vec::new(),
17856 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17857 reserved: false,
17858 reserved_prefixes: Vec::new(),
17859 protocol: ModuleProtocol::Subc,
17860 overlap: Default::default(),
17861 carry_containment: false,
17862 };
17863 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17864
17865 drain_child_to_state(
17866 &spec.module_id,
17867 spec.protocol,
17868 StopNotice::SentOverConnection,
17871 ®istry,
17872 None,
17873 &snapshot,
17874 &runtime.terminal_ring,
17875 &runtime.spawn_events,
17876 child,
17877 Duration::from_secs(1),
17878 ModuleState::Stopped,
17879 Some(false),
17880 &super::FailureContext::operator(),
17881 )
17882 .await
17883 .unwrap();
17884
17885 let state = lock_snapshot(&snapshot).unwrap();
17886 assert_eq!(
17887 state.last_exit.as_ref().map(|exit| exit.kind),
17888 Some(ExitKind::Crash)
17889 );
17890 assert_eq!(state.lifetime_restarts, 0);
17891 assert_eq!(state.crash_restarts.len(), 0);
17892 }
17893
17894 #[test]
17895 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
17896 assert!(!include_str!("server.rs")
17902 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
17903 }
17904
17905 #[test]
17912 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
17913 assert!(drained_after_quiescence_wait(&Ok(true)));
17914 assert!(!drained_after_quiescence_wait(&Ok(false)));
17915 assert!(!drained_after_quiescence_wait(&Err(
17916 SuperviseError::StatePoisoned { module_id: None }
17917 )));
17918 }
17919
17920 #[test]
17929 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
17930 let ring = Arc::new(Mutex::new(TerminalRing::new(
17931 TerminalRingConfig::default(),
17932 0,
17933 )));
17934 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
17935
17936 let snapshot = ring.lock().unwrap().snapshot();
17937 assert_eq!(snapshot.entries.len(), 1);
17938 let entry = &snapshot.entries[0];
17939 assert_eq!(entry.exit_code, None);
17940 assert_eq!(entry.exit_signal, None);
17941 assert_eq!(entry.disposition, TerminalDisposition::Failed);
17942 }
17943
17944 #[test]
17947 fn spawn_generation_of_a_failed_launch_is_never_reused() {
17948 let feed = super::SpawnEventFeed::default();
17949 feed.configure_incarnation("launch-numbering".to_string());
17950 let (incarnation, failed) = feed.allocate_generation("relaunched");
17951 assert_eq!(incarnation, "launch-numbering");
17952 let (_, next) = feed.allocate_generation("relaunched");
17953 assert!(next > failed);
17954 assert_eq!(feed.emit_spawned("relaunched", Some(next), 7, 1), next);
17955 assert!(feed.emit_spawned("relaunched", None, 8, 2) > next);
17957 assert_eq!(
17958 feed.allocate_generation("other").1,
17959 1,
17960 "numbering is per module"
17961 );
17962 }
17963
17964 #[test]
17965 fn wait_error_exit_path_preserves_spawn_event_density() {
17966 let feed = super::SpawnEventFeed::default();
17967 feed.configure_incarnation("wait-error-density".to_string());
17968 feed.emit_spawned("wait-error", None, 41, 1);
17969 let ring = Arc::new(Mutex::new(TerminalRing::new(
17970 TerminalRingConfig::default(),
17971 0,
17972 )));
17973
17974 record_wait_error_terminal("wait-error", &ring, &feed);
17975 feed.emit_spawned("after-wait-error", None, 42, 2);
17976
17977 let state = feed.0.lock().unwrap();
17978 let sequences = state
17979 .events
17980 .iter()
17981 .map(|event| event.cursor.seq)
17982 .collect::<Vec<_>>();
17983 assert_eq!(sequences, vec![1, 2, 3]);
17984 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
17985 assert_eq!(state.events[1].exit_code, None);
17986 assert_eq!(state.events[1].exit_signal, None);
17987 }
17988
17989 #[test]
17993 fn wait_error_exit_report_is_classified_as_a_crash() {
17994 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
17995 }
17996}
17997
17998#[cfg(all(test, unix))]
17999mod restart_ordering_tests {
18000 use super::*;
18001 use std::io::Write;
18002 use tokio::io::AsyncReadExt;
18003
18004 #[test]
18005 #[ignore = "child process fixture"]
18006 fn serving_child() {
18007 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
18008 return;
18009 };
18010 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
18011 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
18012 for stream in listener.incoming() {
18013 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
18014 }
18015 }
18016
18017 async fn restart_before_publication(requests: usize) {
18018 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
18019 let address_file = home.path().join("address");
18020 let spec = ModuleSpec {
18021 module_id: "restart-serving-child".into(),
18022 program: std::env::current_exe().unwrap(),
18023 args: vec![
18024 "--exact".into(),
18025 "supervise::restart_ordering_tests::serving_child".into(),
18026 "--ignored".into(),
18027 "--nocapture".into(),
18028 ],
18029 env: vec![(
18030 "RESTART_TEST_ADDRESS_FILE".into(),
18031 address_file.display().to_string(),
18032 )],
18033 reserved: false,
18034 reserved_prefixes: vec![],
18035 protocol: ModuleProtocol::None,
18036 overlap: Default::default(),
18037 carry_containment: false,
18038 };
18039 let mut supervisor =
18040 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18041 let (spawned_tx, spawned_rx) = oneshot::channel();
18042 let spawned_tx = Mutex::new(Some(spawned_tx));
18043 let (release_tx, release_rx) = std::sync::mpsc::channel();
18044 let release_rx = Mutex::new(release_rx);
18045 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
18048 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
18049 release_rx
18050 .lock()
18051 .unwrap()
18052 .recv_timeout(Duration::from_secs(20))
18053 .unwrap();
18054 })));
18055 let registered = supervisor
18056 .register_configured_with_health(
18057 spec.clone(),
18058 true,
18059 HealthConfig::default(),
18060 None,
18061 RestartPolicy::new(3, Duration::ZERO),
18062 DEFAULT_REGISTER_TIMEOUT,
18063 )
18064 .unwrap();
18065 let module = registered.module.clone();
18066 let startup = tokio::task::spawn_blocking(move || {
18067 supervisor.start_registered_configured(registered, None, true)
18068 });
18069 spawned_rx.await.unwrap();
18070 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18071 let address = loop {
18072 if let Ok(address) = std::fs::read_to_string(&address_file) {
18073 if !address.is_empty() {
18074 break address;
18075 }
18076 }
18077 tokio::time::sleep(Duration::from_millis(1)).await;
18078 };
18079 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
18080 let mut reply = String::new();
18081 stream.read_to_string(&mut reply).await.unwrap();
18082 reply.trim().parse::<u32>().unwrap()
18083 })
18084 .await
18085 .expect("child must serve before generation publication");
18086 assert_eq!(
18087 lock_snapshot(&module.inner.snapshot)
18088 .unwrap()
18089 .spawn_generation,
18090 0
18091 );
18092 let mut replies = Vec::new();
18093 for _ in 0..requests {
18094 let mut restart = Box::pin(module.restart(Some(0)));
18095 std::future::poll_fn(|cx| {
18098 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18099 std::task::Poll::Ready(())
18100 })
18101 .await;
18102 replies.push(restart);
18103 }
18104 release_tx.send(()).unwrap();
18105 startup.await.unwrap().unwrap();
18106 for reply in replies {
18107 reply.await.unwrap();
18108 }
18109 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
18112 loop {
18113 if lock_snapshot(&module.inner.snapshot)
18114 .unwrap()
18115 .spawn_generation
18116 > 1
18117 {
18118 break;
18119 }
18120 tokio::time::sleep(Duration::from_millis(1)).await;
18121 }
18122 })
18123 .await;
18124 module.update_spec_for_test(spec).await.unwrap();
18128 let (generation, pid) = {
18129 let state = lock_snapshot(&module.inner.snapshot).unwrap();
18130 (state.spawn_generation, state.pid)
18131 };
18132 module.stop().await.unwrap();
18133 replacement.expect("restart must publish a replacement for the already-serving child");
18134 assert_ne!(
18135 pid,
18136 Some(old_pid),
18137 "a serving child predating the restart must be replaced"
18138 );
18139 assert_eq!(generation, 2, "queued restarts must share one replacement");
18140 }
18141
18142 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18143 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18144 restart_before_publication(1).await;
18145 }
18146
18147 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18148 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18149 restart_before_publication(2).await;
18150 }
18151
18152 #[tokio::test]
18153 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18154 let supervisor =
18155 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18156 let registered = supervisor
18157 .register_configured_with_health(
18158 ModuleSpec {
18159 module_id: "queued-before-replacement".into(),
18160 program: "/bin/sleep".into(),
18165 args: vec!["600".into()],
18166 env: vec![],
18167 reserved: false,
18168 reserved_prefixes: vec![],
18169 protocol: ModuleProtocol::None,
18170 overlap: Default::default(),
18171 carry_containment: false,
18172 },
18173 true,
18174 HealthConfig::default(),
18175 None,
18176 RestartPolicy::default(),
18177 DEFAULT_REGISTER_TIMEOUT,
18178 )
18179 .unwrap();
18180 let RegisteredConfiguredModule {
18181 mut spec,
18182 mut runtime,
18183 module,
18184 mut commands,
18185 } = registered;
18186 let snapshot = &module.inner.snapshot;
18187 let mut restart = Box::pin(module.restart(Some(0)));
18188 std::future::poll_fn(|cx| {
18189 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18190 std::task::Poll::Ready(())
18191 })
18192 .await;
18193 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18196 let replacement_pid = child.as_ref().unwrap().id();
18197 let command = commands.recv().await.unwrap();
18198 assert!(
18199 handle_supervisor_command(
18200 command,
18201 &mut spec,
18202 &mut runtime,
18203 &supervisor.registry,
18204 &supervisor.process_liveness,
18205 snapshot,
18206 &mut child,
18207 &mut commands,
18208 &mut VecDeque::new(),
18209 )
18210 .await
18211 );
18212 restart.await.unwrap();
18213 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18214 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18215 if let Some(mut child) = child {
18216 child.start_kill().unwrap();
18217 child.wait().await.unwrap();
18218 }
18219 assert_eq!(
18220 kept_pid, replacement_pid,
18221 "a newer replacement must survive"
18222 );
18223 assert!(
18224 !pending,
18225 "a satisfied restart must not schedule another spawn"
18226 );
18227 }
18228}
18229
18230#[cfg(all(test, unix))]
18231mod health_event_tests {
18232 use super::*;
18233 use cortexkit_test_support::ScratchDir;
18234 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18235
18236 struct Actor {
18237 module: SupervisedModule,
18238 registry: Arc<Registry>,
18239 forwarding: Arc<ForwardingTable>,
18240 spec: ModuleSpec,
18241 health: HealthConfig,
18242 rx: mpsc::Receiver<crate::router::OutboundFrame>,
18243 roster: ChildRoster,
18244 _home: ScratchDir,
18245 }
18246
18247 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18248 let deadline = std::time::Instant::now() + Duration::from_secs(10);
18252 let tick = Instant::now();
18253 loop {
18254 if let Some(value) = observe() {
18255 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18256 return value;
18257 }
18258 assert!(
18259 std::time::Instant::now() < deadline,
18260 "timed out waiting for {reason}"
18261 );
18262 tokio::task::yield_now().await;
18263 }
18264 }
18265
18266 impl Actor {
18267 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18268 Self::start_with_policy(
18269 protocol,
18270 cadence,
18271 DEFAULT_REGISTER_TIMEOUT,
18272 RestartPolicy::new(3, Duration::ZERO),
18273 )
18274 .await
18275 }
18276
18277 async fn start_with_policy(
18278 protocol: ModuleProtocol,
18279 cadence: Duration,
18280 register_timeout: Duration,
18281 policy: RestartPolicy,
18282 ) -> Self {
18283 let home = ScratchDir::new("health-events");
18284 let registry = Arc::new(Registry::default());
18285 let forwarding = Arc::new(ForwardingTable::default());
18286 let health = HealthConfig {
18287 cadence,
18288 deadline: Duration::from_secs(3600),
18289 ..HealthConfig::default()
18290 };
18291 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18292 .with_forwarding(forwarding.clone())
18293 .with_health_config(health.clone());
18294 let spec = ModuleSpec {
18295 module_id: "event-health-child".into(),
18296 program: PathBuf::from("/bin/sleep"),
18304 args: vec!["600".into()],
18307 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18308 .into_iter()
18309 .map(|key| (key.into(), home.path().display().to_string()))
18310 .collect(),
18311 reserved: false,
18312 reserved_prefixes: vec![],
18313 protocol,
18314 overlap: Default::default(),
18315 carry_containment: false,
18316 };
18317 let module = supervisor
18318 .supervise_configured_requested(
18319 spec.clone(),
18320 true,
18321 health.clone(),
18322 Some(0),
18323 policy,
18324 register_timeout,
18325 None,
18326 )
18327 .unwrap();
18328 let (_, rx) = mpsc::channel(8);
18329 let actor = Self {
18330 module,
18331 registry,
18332 forwarding,
18333 spec,
18334 health,
18335 rx,
18336 roster: supervisor.child_roster.clone(),
18337 _home: home,
18338 };
18339 actor
18340 .wait_for_select(|checkpoint| checkpoint.generation > 0)
18341 .await;
18342 actor
18343 }
18344
18345 async fn expire_registration_wait(&self, remaining: Duration) {
18346 tokio::time::advance(remaining).await;
18347 wait_for("registration timeout TERM grace", || {
18348 lock_snapshot(&self.module.inner.snapshot)
18349 .unwrap()
18350 .actor_select
18351 .is_none()
18352 .then_some(())
18353 })
18354 .await;
18355 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18356 }
18357
18358 fn turns(&self) -> u64 {
18359 lock_snapshot(&self.module.inner.snapshot)
18360 .unwrap()
18361 .actor_turns
18362 }
18363
18364 async fn wait_for_select(
18365 &self,
18366 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18367 ) -> ActorSelectCheckpoint {
18368 wait_for("actor select checkpoint", || {
18369 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18370 state.actor_select.filter(|checkpoint| {
18371 checkpoint.generation == state.spawn_generation
18372 && state.state == ModuleState::Running
18373 && state.process_alive
18374 && state.reported_pid().is_some()
18375 && expected(checkpoint)
18376 })
18377 })
18378 .await
18379 }
18380
18381 fn hello(&mut self, connection: u64, health: bool) {
18382 let manifest =
18383 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18384 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18385 .build();
18386 let connection = ConnectionId::new(connection);
18387 let (tx, rx) = mpsc::channel(8);
18388 self.rx = rx;
18389 self.forwarding
18390 .register_module_connection(
18391 connection,
18392 self.spec.module_id.clone(),
18393 subc_protocol::PROTOCOL_VERSION,
18394 Concurrency::ModuleManaged,
18395 FrameSink::new(tx),
18396 )
18397 .unwrap();
18398 self.registry
18399 .register_with_control_ops(
18400 manifest,
18401 subc_protocol::PROTOCOL_VERSION,
18402 connection,
18403 if health {
18404 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18405 } else {
18406 vec![]
18407 },
18408 )
18409 .unwrap();
18410 }
18411
18412 async fn probe(&mut self) -> crate::router::OutboundFrame {
18413 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18414 Ok(frame) => Some(frame),
18415 Err(mpsc::error::TryRecvError::Empty) => None,
18416 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18417 })
18418 .await;
18419 let body: Value = serde_json::from_slice(&frame.body).unwrap();
18420 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18421 frame
18422 }
18423
18424 fn no_probe(&mut self) {
18425 assert!(matches!(
18426 self.rx.try_recv(),
18427 Err(mpsc::error::TryRecvError::Empty)
18428 ));
18429 }
18430
18431 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18432 self.forwarding
18433 .complete_module_control_rpc(
18434 ConnectionId::new(connection),
18435 frame.header.corr,
18436 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18437 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18438 status: HealthStatus::Ok,
18439 detail: None,
18440 metrics: None,
18441 }),
18442 )
18443 .unwrap();
18444 }
18445 }
18446
18447 #[tokio::test(start_paused = true)]
18448 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18449 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18450 let initial = actor.module.status().unwrap();
18451 tokio::time::advance(Duration::from_secs(119)).await;
18452 tokio::task::yield_now().await;
18453 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18454 assert!(actor.module.terminal_history().entries.is_empty());
18455 actor.expire_registration_wait(Duration::from_secs(1)).await;
18456 actor
18457 .wait_for_select(|at| at.generation > initial.spawn_generation)
18458 .await;
18459 let status = actor.module.status().unwrap();
18460 assert_ne!(status.pid, initial.pid);
18461 assert!(!cortexkit_test_support::process_alive(
18462 initial.pid.unwrap() as i32
18463 ));
18464 assert_eq!(status.restart_count, 1);
18465 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18466 let entries = actor.module.terminal_history().entries;
18467 assert_eq!(entries.len(), 1);
18468 assert_eq!(
18469 entries[0].disposition_detail.as_deref(),
18470 Some("registration_timeout")
18471 );
18472 actor.module.stop().await.unwrap();
18473 }
18474
18475 #[tokio::test(start_paused = true)]
18476 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18477 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18478 let pid = actor.module.status().unwrap().pid;
18479 tokio::time::advance(Duration::from_secs(119)).await;
18480 actor.hello(1, false);
18481 let registered = actor
18482 .wait_for_select(|at| at.registered_connection.is_some())
18483 .await;
18484 assert_eq!(
18485 registered.wake_after, None,
18486 "HELLO must cancel the deadline"
18487 );
18488 tokio::time::advance(Duration::from_secs(300)).await;
18489 tokio::task::yield_now().await;
18490 assert_eq!(actor.module.status().unwrap().pid, pid);
18491 assert!(actor.module.terminal_history().entries.is_empty());
18492 actor
18494 .registry
18495 .deregister_connection(ConnectionId::new(1))
18496 .unwrap();
18497 actor
18498 .wait_for_select(|at| at.registered_connection.is_none())
18499 .await;
18500 tokio::time::advance(Duration::from_secs(300)).await;
18501 tokio::task::yield_now().await;
18502 assert_eq!(actor.module.status().unwrap().pid, pid);
18503 assert!(actor.module.terminal_history().entries.is_empty());
18504 actor.module.stop().await.unwrap();
18505
18506 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18507 let pid = transient.module.status().unwrap().pid;
18508 tokio::time::advance(Duration::from_secs(119)).await;
18509 let turns = transient.turns();
18510 transient.hello(1, false);
18513 transient
18514 .registry
18515 .deregister_connection(ConnectionId::new(1))
18516 .unwrap();
18517 let parked = transient.wait_for_select(|at| at.turn > turns).await;
18518 assert_eq!(
18519 parked.wake_after, None,
18520 "transient HELLO must cancel the deadline"
18521 );
18522 tokio::time::advance(Duration::from_secs(300)).await;
18523 tokio::task::yield_now().await;
18524 assert_eq!(transient.module.status().unwrap().pid, pid);
18525 assert!(transient.module.terminal_history().entries.is_empty());
18526 transient.module.stop().await.unwrap();
18527 }
18528
18529 #[tokio::test(start_paused = true)]
18530 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18531 let actor = Actor::start_with_policy(
18532 ModuleProtocol::Subc,
18533 Duration::from_secs(30),
18534 Duration::from_secs(10),
18535 RestartPolicy::new(1, Duration::ZERO),
18536 )
18537 .await;
18538 let generation = actor.module.status().unwrap().spawn_generation;
18539 actor
18540 .expire_registration_wait(Duration::from_secs(10))
18541 .await;
18542 actor.wait_for_select(|at| at.generation > generation).await;
18543 actor
18544 .expire_registration_wait(Duration::from_secs(10))
18545 .await;
18546 let retry = wait_for("failed retry schedule", || {
18547 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18548 (state.state == ModuleState::Failed)
18549 .then_some(state.failed_retry.pending)
18550 .flatten()
18551 })
18552 .await;
18553 assert_eq!(retry.delay, Duration::from_secs(600));
18554 assert_eq!(retry.attempt, 1);
18555 let entries = actor.module.terminal_history().entries;
18556 assert_eq!(entries.len(), 2);
18557 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18558 assert!(entries[1]
18559 .disposition_detail
18560 .as_deref()
18561 .unwrap()
18562 .contains("registration_timeout"));
18563 tokio::time::advance(Duration::from_secs(599)).await;
18564 tokio::task::yield_now().await;
18565 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18566 tokio::time::advance(Duration::from_secs(1)).await;
18567 actor
18568 .wait_for_select(|at| at.generation > generation + 1)
18569 .await;
18570 actor
18572 .expire_registration_wait(Duration::from_secs(10))
18573 .await;
18574 wait_for("slow retry registration timeout", || {
18575 (actor.module.terminal_history().entries.len() == 3).then_some(())
18576 })
18577 .await;
18578 actor.module.stop().await.unwrap();
18579 }
18580
18581 #[tokio::test(start_paused = true)]
18582 async fn registration_timeout_honours_ten_second_override() {
18583 let actor = Actor::start_with_policy(
18584 ModuleProtocol::Subc,
18585 Duration::from_secs(30),
18586 Duration::from_secs(10),
18587 RestartPolicy::new(3, Duration::ZERO),
18588 )
18589 .await;
18590 let generation = actor.module.status().unwrap().spawn_generation;
18591 tokio::time::advance(Duration::from_secs(9)).await;
18592 tokio::task::yield_now().await;
18593 assert!(actor.module.terminal_history().entries.is_empty());
18594 actor.expire_registration_wait(Duration::from_secs(1)).await;
18595 actor.wait_for_select(|at| at.generation > generation).await;
18596 assert_eq!(actor.module.terminal_history().entries.len(), 1);
18597 actor.module.stop().await.unwrap();
18598 }
18599
18600 #[tokio::test]
18601 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18602 use std::os::unix::process::{CommandExt, ExitStatusExt};
18603
18604 struct Helper(std::process::Child);
18605 impl Drop for Helper {
18606 fn drop(&mut self) {
18607 if matches!(self.0.try_wait(), Ok(None)) {
18610 let _ = self.0.kill();
18611 let _ = self.0.wait();
18612 }
18613 }
18614 }
18615
18616 let home = ScratchDir::new("registration-timeout-helper");
18617 let supervisor = Supervisor::new_for_test(
18618 Arc::new(Registry::default()),
18619 RestartPolicy::new(0, Duration::ZERO),
18620 );
18621 let spec = ModuleSpec {
18622 module_id: "registration-timeout-group".into(),
18623 program: "/bin/sleep".into(),
18624 args: vec!["120".into()],
18625 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18626 .into_iter()
18627 .map(|key| (key.into(), home.path().display().to_string()))
18628 .collect(),
18629 reserved: false,
18630 reserved_prefixes: vec![],
18631 protocol: ModuleProtocol::Subc,
18632 overlap: Default::default(),
18633 carry_containment: false,
18634 };
18635 let module = supervisor
18636 .supervise_configured_requested(
18637 spec,
18638 true,
18639 HealthConfig::default(),
18640 None,
18641 RestartPolicy::new(0, Duration::ZERO),
18642 Duration::from_secs(10),
18643 None,
18644 )
18645 .unwrap();
18646 let leader = timeout(Duration::from_secs(5), async {
18647 loop {
18648 if let Some(pid) = module.status().unwrap().pid {
18649 break pid;
18650 }
18651 sleep(Duration::from_millis(5)).await;
18652 }
18653 })
18654 .await
18655 .expect("supervised group leader must be reported before the deadline");
18656 assert!(leader > 1);
18657 let ready = home.join("helper.ready");
18658 let mut helper = Helper(
18661 std::process::Command::new("/bin/sh")
18662 .args([
18663 "-c",
18664 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18665 "helper",
18666 ])
18667 .arg(&ready)
18668 .process_group(i32::try_from(leader).unwrap())
18669 .stdout(std::process::Stdio::null())
18670 .stderr(std::process::Stdio::null())
18671 .spawn()
18672 .unwrap(),
18673 );
18674 timeout(Duration::from_secs(5), async {
18675 while !ready.exists() {
18676 assert!(helper.0.try_wait().unwrap().is_none());
18677 sleep(Duration::from_millis(5)).await;
18678 }
18679 })
18680 .await
18681 .expect("helper must install TERM ignore before the registration deadline");
18682 let helper_pid =
18683 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18684 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18685 sleep(Duration::from_millis(20)).await;
18686 assert!(
18687 helper.0.try_wait().unwrap().is_none(),
18688 "helper must ignore TERM"
18689 );
18690
18691 timeout(Duration::from_secs(20), async {
18692 while module.state().unwrap() != ModuleState::Failed {
18693 sleep(Duration::from_millis(5)).await;
18694 }
18695 })
18696 .await
18697 .expect("registration timeout must stop and reap the group leader");
18698 let helper_exit = timeout(Duration::from_secs(3), async {
18699 loop {
18700 if let Some(status) = helper.0.try_wait().unwrap() {
18701 break status;
18702 }
18703 sleep(Duration::from_millis(5)).await;
18704 }
18705 })
18706 .await
18707 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18708 assert_eq!(helper_exit.signal(), Some(9));
18709 assert!(!cortexkit_test_support::process_alive(
18710 i32::try_from(leader).unwrap()
18711 ));
18712 assert_eq!(
18713 module.status().unwrap().last_exit.unwrap().kind,
18714 ExitKind::Crash
18715 );
18716 assert!(module.terminal_history().entries[0]
18717 .disposition_detail
18718 .as_deref()
18719 .unwrap()
18720 .contains("registration_timeout"));
18721 module.stop().await.unwrap();
18722 }
18723
18724 #[tokio::test(start_paused = true)]
18725 async fn registration_timeout_does_not_count_an_earlier_exit() {
18726 let actor = Actor::start_with_policy(
18727 ModuleProtocol::Subc,
18728 Duration::from_secs(30),
18729 DEFAULT_REGISTER_TIMEOUT,
18730 RestartPolicy::new(0, Duration::ZERO),
18731 )
18732 .await;
18733 let pid = actor.module.status().unwrap().pid.unwrap();
18734 rustix::process::kill_process(
18735 rustix::process::Pid::from_raw(pid as i32).unwrap(),
18736 rustix::process::Signal::TERM,
18737 )
18738 .unwrap();
18739 wait_for("early exit", || {
18740 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18741 })
18742 .await;
18743 tokio::time::advance(Duration::from_secs(121)).await;
18744 tokio::task::yield_now().await;
18745 let entries = actor.module.terminal_history().entries;
18746 assert_eq!(entries.len(), 1);
18747 assert!(!entries[0]
18748 .disposition_detail
18749 .as_deref()
18750 .unwrap()
18751 .contains("registration_timeout"));
18752 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18753 actor.module.stop().await.unwrap();
18754 }
18755
18756 #[tokio::test(start_paused = true)]
18757 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18758 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18759 let pid = actor.module.status().unwrap().pid;
18760 tokio::time::advance(Duration::from_secs(119)).await;
18761 actor.roster.close();
18762 let turns = actor.turns();
18763 tokio::time::advance(Duration::from_secs(300)).await;
18764 let parked = actor.wait_for_select(|at| at.turn > turns).await;
18765 assert_eq!(parked.wake_after, None);
18766 assert_eq!(actor.module.status().unwrap().pid, pid);
18767 assert!(actor.module.terminal_history().entries.is_empty());
18768 actor.module.stop().await.unwrap();
18769 let status = actor.module.status().unwrap();
18770 assert_eq!(status.restart_count, 0);
18771 assert_eq!(
18772 actor.module.terminal_history().entries[0].disposition,
18773 TerminalDisposition::DaemonShutdown
18774 );
18775 }
18776
18777 #[tokio::test(start_paused = true)]
18778 async fn no_health_children_park_for_sixty_seconds() {
18779 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18780 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18781 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18782 unadvertised.hello(1, false);
18783 for _ in 0..6000 {
18788 tokio::time::advance(Duration::from_millis(10)).await;
18789 tokio::task::yield_now().await;
18790 }
18791 for actor in [&none, &unregistered, &unadvertised] {
18792 assert!(
18793 actor.turns() <= 5,
18794 "no-health actor woke {} times",
18795 actor.turns()
18796 );
18797 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18798 }
18799 }
18800
18801 #[tokio::test(start_paused = true)]
18802 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18803 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18807 let tick = Instant::now();
18808 actor.hello(1, true);
18809 actor.probe().await;
18810 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18811 }
18812
18813 #[tokio::test(start_paused = true)]
18814 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18815 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18816 let hello_tick = Instant::now();
18817 actor.hello(1, true);
18818 let armed = actor
18819 .wait_for_select(|checkpoint| {
18820 checkpoint.registered_connection == Some(ConnectionId::new(1))
18821 && checkpoint.next_probe_at.is_some()
18822 })
18823 .await;
18824 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18825 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18826 let first_deadline = hello_tick + delay;
18827 assert_eq!(armed.next_probe_at, Some(first_deadline));
18828 tokio::time::advance(Duration::from_secs(29)).await;
18829 let turns = actor.turns();
18830 actor
18831 .registry
18832 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18833 .unwrap();
18834 let updated = actor
18835 .wait_for_select(|checkpoint| checkpoint.turn > turns)
18836 .await;
18837 assert_eq!(
18838 updated.next_probe_at,
18839 Some(first_deadline),
18840 "catalog update must not reset cadence"
18841 );
18842 actor.no_probe();
18843 tokio::time::advance(first_deadline - Instant::now()).await;
18844 let frame = actor.probe().await;
18845 actor.answer(1, frame);
18846 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18847 let next_deadline = Instant::now() + next;
18848 let rearmed = actor
18849 .wait_for_select(|checkpoint| {
18850 checkpoint
18851 .next_probe_at
18852 .is_some_and(|deadline| deadline > first_deadline)
18853 })
18854 .await;
18855 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18856 tokio::time::advance(next - Duration::from_nanos(1)).await;
18857 actor.no_probe();
18858 tokio::time::advance(Duration::from_nanos(1)).await;
18859 actor.probe().await;
18860 }
18861
18862 #[tokio::test(start_paused = true)]
18863 async fn disconnect_disarms_and_reconnect_rearms_health() {
18864 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18865 actor.hello(1, true);
18866 actor
18867 .wait_for_select(|checkpoint| {
18868 checkpoint.registered_connection == Some(ConnectionId::new(1))
18869 && checkpoint.next_probe_at.is_some()
18870 })
18871 .await;
18872 let turns = actor.turns();
18873 let tick = Instant::now();
18874 actor
18875 .registry
18876 .deregister_connection(ConnectionId::new(1))
18877 .unwrap();
18878 let disarmed = actor
18879 .wait_for_select(|checkpoint| {
18880 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
18881 })
18882 .await;
18883 assert_eq!(disarmed.next_probe_at, None);
18884 assert_eq!(disarmed.wake_after, None);
18885 assert_eq!(
18886 Instant::now(),
18887 tick,
18888 "disconnect must disarm in the notification tick"
18889 );
18890 tokio::time::advance(Duration::from_secs(60)).await;
18891 actor.no_probe();
18892 let reconnect_tick = Instant::now();
18893 actor.hello(2, true);
18894 let rearmed = actor
18895 .wait_for_select(|checkpoint| {
18896 checkpoint.registered_connection == Some(ConnectionId::new(2))
18897 && checkpoint.next_probe_at.is_some()
18898 })
18899 .await;
18900 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18901 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
18902 tokio::time::advance(delay).await;
18903 actor.probe().await;
18904 }
18905
18906 #[tokio::test(start_paused = true)]
18907 async fn reregistration_without_health_disarms_the_old_schedule() {
18908 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18909 actor.hello(1, true);
18910 actor
18911 .wait_for_select(|checkpoint| {
18912 checkpoint.registered_connection == Some(ConnectionId::new(1))
18913 && checkpoint.next_probe_at.is_some()
18914 })
18915 .await;
18916 actor
18917 .registry
18918 .deregister_connection(ConnectionId::new(1))
18919 .unwrap();
18920 actor.hello(2, false);
18921 let disarmed = actor
18922 .wait_for_select(|checkpoint| {
18923 checkpoint.registered_connection == Some(ConnectionId::new(2))
18924 })
18925 .await;
18926 assert_eq!(disarmed.next_probe_at, None);
18927 assert_eq!(disarmed.wake_after, None);
18928 tokio::time::advance(Duration::from_secs(60)).await;
18929 actor.no_probe();
18930 }
18931
18932 #[tokio::test(start_paused = true)]
18933 async fn restart_waits_for_its_own_hello_before_rearming_health() {
18934 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18935 actor.hello(1, true);
18936 let armed = actor
18937 .wait_for_select(|checkpoint| {
18938 checkpoint.registered_connection == Some(ConnectionId::new(1))
18939 && checkpoint.next_probe_at.is_some()
18940 })
18941 .await;
18942 actor
18946 .registry
18947 .deregister_connection(ConnectionId::new(1))
18948 .unwrap();
18949 actor.module.restart(Some(0)).await.unwrap();
18950 let parked = actor
18951 .wait_for_select(|checkpoint| {
18952 checkpoint.generation > armed.generation
18953 && checkpoint.registered_connection.is_none()
18954 && checkpoint.next_probe_at.is_none()
18955 })
18956 .await;
18957 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
18958 actor.rx.close();
18959 while actor.rx.try_recv().is_ok() {}
18962 let turns = actor.turns();
18963 tokio::time::advance(Duration::from_secs(60)).await;
18964 let after = actor.turns();
18965 eprintln!(
18966 "restart park turns: before={turns} after={after} delta={}",
18967 after - turns
18968 );
18969 assert_eq!(after, turns, "replacement must park before HELLO");
18970 let hello_tick = Instant::now();
18971 actor.hello(2, true);
18972 let rearmed = actor
18973 .wait_for_select(|checkpoint| {
18974 checkpoint.registered_connection == Some(ConnectionId::new(2))
18975 && checkpoint.next_probe_at.is_some()
18976 })
18977 .await;
18978 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18979 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
18980 tokio::time::advance(delay).await;
18981 actor.probe().await;
18982 }
18983
18984 #[tokio::test(start_paused = true)]
18985 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
18986 use tokio::{
18987 io::{AsyncReadExt, AsyncWriteExt},
18988 net::TcpListener,
18989 };
18990 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18991 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
18992 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
18993 let (started_tx, mut started_rx) = mpsc::channel(8);
18994 let peer = tokio::spawn(async move {
18995 loop {
18996 let (mut socket, _) = listener.accept().await.unwrap();
18997 let mut request = [0; 1024];
18998 assert!(socket.read(&mut request).await.unwrap() > 0);
18999 started_tx.send(Instant::now()).await.unwrap();
19000 socket
19001 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19002 .await
19003 .unwrap();
19004 }
19005 });
19006 actor
19007 .module
19008 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19009 .await
19010 .unwrap();
19011 let first_tick = Instant::now();
19012 let first = actor
19013 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
19014 .await;
19015 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
19016 assert!(started_rx.try_recv().is_err());
19017
19018 tokio::time::advance(
19021 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
19022 )
19023 .await;
19024 actor.module.restart(Some(0)).await.unwrap();
19025 let spawn_tick = Instant::now();
19026 let replacement = actor
19027 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
19028 .await;
19029 let deadline = replacement.next_probe_at.unwrap();
19030 assert!(
19031 deadline >= spawn_tick + actor.health.cadence,
19032 "a replacement must not inherit its predecessor's probe deadline"
19033 );
19034 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
19035 assert!(
19036 started_rx.try_recv().is_err(),
19037 "no HTTP probe during startup grace"
19038 );
19039 tokio::time::advance(deadline - Instant::now()).await;
19040 let landed = wait_for("first replacement HTTP probe", || {
19041 started_rx.try_recv().ok()
19042 })
19043 .await;
19044 assert!(landed >= spawn_tick + actor.health.cadence);
19045 wait_for("healthy replacement", || {
19046 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19047 .then_some(())
19048 })
19049 .await;
19050 peer.abort();
19051 }
19052
19053 #[tokio::test(start_paused = true)]
19054 async fn rescan_adds_and_removes_http_health_without_polling() {
19055 use tokio::{
19056 io::{AsyncReadExt, AsyncWriteExt},
19057 net::TcpListener,
19058 };
19059 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19060 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19061 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
19062 actor.health.cadence = Duration::from_millis(1);
19064 let (started_tx, mut started_rx) = mpsc::channel(8);
19065 let peer = tokio::spawn(async move {
19066 loop {
19067 let (mut socket, _) = listener.accept().await.unwrap();
19068 let mut request = [0; 1024];
19069 assert!(socket.read(&mut request).await.unwrap() > 0);
19070 started_tx.send(()).await.unwrap();
19071 socket
19072 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19073 .await
19074 .unwrap();
19075 }
19076 });
19077 let tick = Instant::now();
19078 let turns = actor.turns();
19079 actor
19080 .module
19081 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19082 .await
19083 .unwrap();
19084 let armed = actor
19085 .wait_for_select(|checkpoint| {
19086 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
19087 })
19088 .await;
19089 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
19090 assert_eq!(Instant::now(), tick);
19091 tokio::time::advance(Duration::from_millis(1)).await;
19092 let mut started = false;
19093 wait_for("successful HTTP probe", || {
19094 started |= started_rx.try_recv().is_ok();
19095 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19096 .then_some(())
19097 })
19098 .await;
19099 assert!(started, "added HTTP check must start at its first deadline");
19100 assert_eq!(
19101 actor.module.status().unwrap().health.status,
19102 SupervisorHealthStatus::Ok
19103 );
19104 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
19105 actor.health.http = None;
19106 let turns = actor.turns();
19107 actor
19108 .module
19109 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19110 .await
19111 .unwrap();
19112 let parked = actor
19113 .wait_for_select(|checkpoint| {
19114 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
19115 })
19116 .await;
19117 assert_eq!(parked.wake_after, None);
19118 let turns = actor.turns();
19119 tokio::time::advance(Duration::from_secs(60)).await;
19120 assert_eq!(
19121 actor.turns(),
19122 turns,
19123 "removed HTTP check must park the actor"
19124 );
19125 assert!(
19126 started_rx.try_recv().is_err(),
19127 "removed HTTP check must stay disarmed"
19128 );
19129 assert_eq!(
19130 actor.module.status().unwrap().health,
19131 ModuleHealthStatus::default()
19132 );
19133 peer.abort();
19134 }
19135}
19136
19137#[cfg(test)]
19138mod drain_busy_reprobe_tests {
19139 use super::*;
19143 use std::collections::BTreeMap;
19144 use std::sync::Mutex as StdMutex;
19145 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19146
19147 const MODULE_ID: &str = "drain-reprobe-module";
19148 const CONNECTION: u64 = 7;
19149 const GAUGE: &str = "runs_in_flight";
19150 const CADENCE: Duration = Duration::from_secs(30);
19153 const CEILING: Duration = Duration::from_secs(30);
19154
19155 #[derive(Clone, Copy)]
19157 enum Answer {
19158 BusyUntil { until: Instant, latency: Duration },
19161 OmitsGauge,
19163 Silent,
19165 }
19166
19167 struct Drain {
19168 forwarding: Arc<ForwardingTable>,
19169 runtime: SupervisorRuntimeConfig,
19170 endpoint: crate::ModuleEndpointId,
19171 probes: Arc<StdMutex<Vec<Instant>>>,
19173 responder: tokio::task::JoinHandle<()>,
19174 }
19175
19176 impl Drain {
19177 fn start(answer: Answer) -> Self {
19178 let registry = Arc::new(Registry::default());
19179 let forwarding = Arc::new(ForwardingTable::default());
19180 let health = HealthConfig {
19181 cadence: CADENCE,
19182 ..HealthConfig::default()
19183 };
19184 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19185 .with_forwarding(forwarding.clone())
19186 .with_health_config(health)
19187 .runtime_config();
19188 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19189 let endpoint = forwarding
19190 .register_module_connection(
19191 ConnectionId::new(CONNECTION),
19192 MODULE_ID.to_string(),
19193 subc_protocol::PROTOCOL_VERSION,
19194 Concurrency::ModuleManaged,
19195 FrameSink::new(tx),
19196 )
19197 .unwrap();
19198 let probes = Arc::new(StdMutex::new(Vec::new()));
19199 let responder = tokio::spawn({
19200 let forwarding = forwarding.clone();
19201 let probes = probes.clone();
19202 async move {
19203 while let Some(frame) = rx.recv().await {
19204 let body: Value = serde_json::from_slice(&frame.body).unwrap();
19205 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19206 probes.lock().unwrap().push(Instant::now());
19207 let metrics = match answer {
19208 Answer::BusyUntil { until, latency } => {
19209 sleep(latency).await;
19210 let busy = u64::from(Instant::now() < until);
19211 serde_json::json!({ GAUGE: busy })
19212 }
19213 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19214 Answer::Silent => continue,
19215 };
19216 let _ = forwarding.complete_module_control_rpc(
19219 ConnectionId::new(CONNECTION),
19220 frame.header.corr,
19221 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19222 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19223 status: HealthStatus::Ok,
19224 detail: None,
19225 metrics: Some(metrics),
19226 }),
19227 );
19228 }
19229 }
19230 });
19231 Self {
19232 forwarding,
19233 runtime,
19234 endpoint,
19235 probes,
19236 responder,
19237 }
19238 }
19239
19240 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19243 let (drained, elapsed, _) = self.run_traced(ceiling).await;
19244 (drained, elapsed)
19245 }
19246
19247 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19249 let started = Instant::now();
19250 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19251 let drained = wait_for_forwarding_quiescence(
19252 &self.forwarding,
19253 MODULE_ID,
19254 &self.runtime,
19255 self.endpoint,
19256 started + ceiling,
19257 &[GAUGE.to_string()],
19258 DrainScope::Active,
19259 &mut trace,
19260 )
19261 .await
19262 .unwrap();
19263 (drained, started.elapsed(), trace)
19264 }
19265
19266 fn probes(&self) -> Vec<Instant> {
19267 self.probes.lock().unwrap().clone()
19268 }
19269 }
19270
19271 impl Drop for Drain {
19272 fn drop(&mut self) {
19273 self.responder.abort();
19274 }
19275 }
19276
19277 #[tokio::test(start_paused = true)]
19278 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19279 let drain = Drain::start(Answer::BusyUntil {
19280 until: Instant::now() + Duration::from_secs(1),
19281 latency: Duration::ZERO,
19282 });
19283 let (drained, elapsed) = drain.run(CEILING).await;
19284 assert!(
19285 drained,
19286 "gauges that read zero during the drain must let it finish drained, \
19287 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19288 );
19289 assert!(
19290 elapsed < Duration::from_secs(3),
19291 "drain took {elapsed:?} after the gauges went quiet at 1s"
19292 );
19293 let probes = drain.probes();
19294 assert!(
19295 probes.len() >= 2,
19296 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19297 );
19298 }
19299
19300 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19305 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19306 let drain = Drain::start(Answer::BusyUntil {
19307 until: Instant::now() + Duration::from_millis(600),
19308 latency: Duration::ZERO,
19309 });
19310 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19311 assert!(drained, "the gauge fell at 600ms: {trace}");
19312 assert!(
19313 elapsed < Duration::from_secs(3),
19314 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19315 );
19316 let probes = drain.probes();
19317 assert!(
19318 probes.len() >= 2 && trace.probes as usize == probes.len(),
19319 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19320 );
19321 }
19322
19323 #[tokio::test(start_paused = true)]
19324 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19325 let started = Instant::now();
19326 let drain = Drain::start(Answer::BusyUntil {
19327 until: started + Duration::from_secs(3600),
19328 latency: Duration::ZERO,
19329 });
19330 let (drained, elapsed) = drain.run(CEILING).await;
19331 assert!(!drained, "a module busy throughout must not drain");
19332 assert!(
19333 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19334 "a busy module is held until the ceiling and no longer: {elapsed:?}"
19335 );
19336 let probes = drain.probes();
19339 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19340 assert!(
19341 probes.len() as u128 <= max_probes,
19342 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19343 probes.len()
19344 );
19345 for pair in probes.windows(2) {
19346 assert!(
19347 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19348 "consecutive probes {:?} apart",
19349 pair[1] - pair[0]
19350 );
19351 }
19352 }
19353
19354 #[tokio::test(start_paused = true)]
19355 async fn a_slow_probe_is_never_overlapped_by_the_next() {
19356 let latency = Duration::from_secs(1);
19359 let drain = Drain::start(Answer::BusyUntil {
19360 until: Instant::now() + Duration::from_secs(3600),
19361 latency,
19362 });
19363 let (drained, _) = drain.run(Duration::from_secs(5)).await;
19364 assert!(!drained);
19365 let probes = drain.probes();
19366 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19367 for pair in probes.windows(2) {
19368 assert!(
19369 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19370 "a probe started {:?} after the previous one, before it could finish",
19371 pair[1] - pair[0]
19372 );
19373 }
19374 }
19375
19376 #[tokio::test(start_paused = true)]
19377 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19378 let drain = Drain::start(Answer::Silent);
19379 let ceiling = Duration::from_secs(2);
19380 let (drained, elapsed) = drain.run(ceiling).await;
19381 assert!(!drained, "an unanswered probe counts as busy");
19382 assert!(
19383 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19384 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19385 );
19386 assert_eq!(
19387 drain.probes().len(),
19388 1,
19389 "the unanswered probe holds the only slot until the ceiling"
19390 );
19391 }
19392
19393 #[derive(Clone, Default)]
19395 struct LogCapture {
19396 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19397 }
19398
19399 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19400 fn on_event(
19401 &self,
19402 event: &tracing::Event<'_>,
19403 _context: tracing_subscriber::layer::Context<'_, S>,
19404 ) {
19405 struct Fields(BTreeMap<String, String>);
19406 impl tracing::field::Visit for Fields {
19407 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19408 self.0
19409 .insert(field.name().to_string(), format!("{value:?}"));
19410 }
19411 }
19412 let mut fields = Fields(BTreeMap::new());
19413 event.record(&mut fields);
19414 self.events.lock().unwrap().push(fields.0);
19415 }
19416 }
19417
19418 impl LogCapture {
19419 fn install(&self) -> tracing::subscriber::DefaultGuard {
19420 use tracing_subscriber::layer::SubscriberExt;
19421 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19422 }
19423
19424 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19425 self.events
19426 .lock()
19427 .unwrap()
19428 .iter()
19429 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19430 .cloned()
19431 .collect()
19432 }
19433 }
19434
19435 #[tokio::test(start_paused = true)]
19440 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19441 let capture = LogCapture::default();
19442 let _guard = capture.install();
19443 let drain = Drain::start(Answer::OmitsGauge);
19444 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19445 assert!(!drained, "a gauge never reported cannot read quiet");
19446 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19447
19448 let missing = capture.with_message("lacks a declared busy gauge");
19449 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19450 assert_eq!(
19451 missing[0].get("missing_gauges").map(String::as_str),
19452 Some(GAUGE)
19453 );
19454
19455 warn_drain_timed_out(
19456 MODULE_ID,
19457 Duration::from_secs(2),
19458 RouteCloseReason::Restart,
19459 &Default::default(),
19460 &trace,
19461 );
19462 let timeouts = capture.with_message("route drain timed out");
19463 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19464 let gauges = &timeouts[0]["busy_gauges"];
19465 assert!(
19466 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19467 "the timeout line must name the gauge and why it read busy: {gauges}"
19468 );
19469 assert!(
19470 gauges.ends_with("; no value ever read"),
19471 "no value was ever read for the gauge: {gauges}"
19472 );
19473 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19474 }
19475
19476 #[tokio::test(start_paused = true)]
19480 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19481 let busy = Drain::start(Answer::BusyUntil {
19482 until: Instant::now() + Duration::from_secs(3600),
19483 latency: Duration::ZERO,
19484 });
19485 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19486 let rendered = trace.to_string();
19487 assert!(
19488 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19489 && rendered.contains("; last value non_zero(1) @"),
19490 "{rendered}"
19491 );
19492
19493 let silent = Drain::start(Answer::Silent);
19494 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19495 assert_eq!(
19498 trace.to_string(),
19499 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19500 );
19501 assert_eq!(trace.probes, 1);
19502 }
19503
19504 #[test]
19506 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19507 let gauges = [GAUGE.to_string()];
19508 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19509 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19510 assert_eq!(
19511 busy_gauge_observation(&verdicts),
19512 BusyGaugeObservation::Busy
19513 );
19514 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19515 assert_eq!(
19516 busy_gauge_observation(&zero),
19517 BusyGaugeObservation::Quiescent
19518 );
19519 let none = gauge_verdicts(None, &gauges);
19520 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19521 }
19522}
19523
19524#[cfg(test)]
19525mod health_evidence_tests {
19526 use super::{HealthProbeError, HealthProbeEvidence};
19527 use std::collections::HashSet;
19528
19529 #[test]
19537 fn only_a_dead_lane_is_proof_of_death() {
19538 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19539 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19543 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19544 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19545 }
19546
19547 #[test]
19553 fn every_evidence_class_has_a_distinct_label() {
19554 let labels = [
19555 HealthProbeError::lane_dead("").label(),
19556 HealthProbeError::no_answer("").label(),
19557 HealthProbeError::bad_answer("").label(),
19558 HealthProbeError::misconfigured("").label(),
19559 ];
19560 let unique: HashSet<_> = labels.iter().collect();
19561 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19562 }
19563
19564 #[test]
19570 fn classification_preserves_the_original_message() {
19571 let err = HealthProbeError::no_answer("module did not answer within 5s");
19572 assert_eq!(err.to_string(), "module did not answer within 5s");
19573 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19574 }
19575}
19576
19577#[cfg(test)]
19578mod health_tombstone_tests {
19579 use std::{path::PathBuf, sync::Arc, time::Duration};
19580
19581 use subc_protocol::{
19582 manifest::Concurrency,
19583 session::{HealthStatus, ModuleControlResponse},
19584 };
19585 use tokio::sync::mpsc;
19586
19587 use super::{
19588 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19589 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19590 };
19591 use crate::{
19592 control::ControlHandler,
19593 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19594 registry::{ConnectionId, Registry},
19595 router::FrameSink,
19596 };
19597
19598 struct ProbeHarness {
19599 spec: ModuleSpec,
19600 runtime: SupervisorRuntimeConfig,
19601 forwarding: Arc<ForwardingTable>,
19602 module_connection: ConnectionId,
19603 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19604 handler: ControlHandler,
19605 module: super::SupervisedModule,
19606 }
19607
19608 fn probe_harness() -> ProbeHarness {
19609 let registry = Arc::new(Registry::default());
19610 let forwarding = Arc::new(ForwardingTable::default());
19611 let supervisor_handle = super::SupervisorHandle::new();
19612 let health = HealthConfig {
19613 http: None,
19614 cadence: Duration::from_secs(30),
19615 deadline: Duration::from_secs(5),
19616 failure_threshold: 3,
19617 on_degraded: HealthAction::Report,
19618 on_failing: HealthAction::Report,
19619 critical: false,
19620 };
19621 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
19622 .with_forwarding(Arc::clone(&forwarding))
19623 .with_handle(supervisor_handle.clone())
19624 .with_health_config(health);
19625 let spec = ModuleSpec {
19626 module_id: "late-health-module".to_string(),
19627 program: PathBuf::from("disabled-module"),
19628 args: Vec::new(),
19629 env: Vec::new(),
19630 reserved: false,
19631 reserved_prefixes: Vec::new(),
19632 protocol: ModuleProtocol::Subc,
19633 overlap: Default::default(),
19634 carry_containment: false,
19635 };
19636 let module = supervisor
19637 .supervise_configured(spec.clone(), false)
19638 .unwrap();
19639 let runtime = supervisor.runtime_config();
19640 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19641 .with_supervisor(supervisor_handle);
19642 let module_connection = ConnectionId::new(700);
19643 let (module_tx, module_rx) = mpsc::channel(8);
19644 forwarding
19645 .register_module_connection(
19646 module_connection,
19647 spec.module_id.clone(),
19648 subc_protocol::PROTOCOL_VERSION,
19649 Concurrency::ModuleManaged,
19650 FrameSink::new(module_tx),
19651 )
19652 .unwrap();
19653
19654 ProbeHarness {
19655 spec,
19656 runtime,
19657 forwarding,
19658 module_connection,
19659 module_rx,
19660 handler,
19661 module,
19662 }
19663 }
19664
19665 async fn finish_after(
19666 harness: &mut ProbeHarness,
19667 stall: Duration,
19668 ) -> ModuleControlRpcCompletion {
19669 assert!(stall > harness.runtime.health.deadline);
19670 let deadline = harness.runtime.health.deadline;
19671 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19672 let answer = async {
19673 let frame = harness.module_rx.recv().await.expect("health.check frame");
19674 tokio::time::advance(deadline).await;
19675 tokio::task::yield_now().await;
19676 tokio::time::advance(stall - deadline).await;
19677 harness
19678 .forwarding
19679 .complete_module_control_rpc(
19680 harness.module_connection,
19681 frame.header.corr,
19682 Some("health.check"),
19683 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19684 status: HealthStatus::Ok,
19685 detail: None,
19686 metrics: None,
19687 }),
19688 )
19689 .unwrap()
19690 };
19691 let (probe_result, completion) = tokio::join!(probe, answer);
19692 let err = probe_result.expect_err("probe must miss its deadline");
19693 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19694 completion
19695 }
19696
19697 async fn time_out_without_answer(harness: &mut ProbeHarness) {
19698 let deadline = harness.runtime.health.deadline;
19699 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19700 let exhaust_deadline = async {
19701 let _frame = harness.module_rx.recv().await.expect("health.check frame");
19702 tokio::time::advance(deadline).await;
19703 tokio::task::yield_now().await;
19704 };
19705 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19706 let err = probe_result.expect_err("probe must miss its deadline");
19707 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19708 }
19709
19710 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19711 let registry = Arc::clone(&harness.module.inner.registry);
19712 let snapshot = Arc::clone(&harness.module.inner.snapshot);
19713 let process_liveness = super::SupervisorProcessLiveness::default();
19714 let mut child = None;
19715 let cycle = super::run_health_probe_cycle(
19716 &harness.spec,
19717 &harness.runtime,
19718 ®istry,
19719 &process_liveness,
19720 &snapshot,
19721 &mut child,
19722 );
19723 let peer = async {
19724 let frame = harness.module_rx.recv().await.expect("health.check frame");
19725 if answer {
19726 harness
19727 .forwarding
19728 .complete_module_control_rpc(
19729 harness.module_connection,
19730 frame.header.corr,
19731 Some("health.check"),
19732 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19733 status: HealthStatus::Ok,
19734 detail: None,
19735 metrics: Some(serde_json::json!({"ready": true})),
19736 }),
19737 )
19738 .unwrap();
19739 } else {
19740 tokio::time::advance(harness.runtime.health.deadline).await;
19741 tokio::task::yield_now().await;
19742 }
19743 };
19744 tokio::join!(cycle, peer);
19745 }
19746
19747 #[tokio::test(start_paused = true)]
19748 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19749 let mut harness = probe_harness();
19750 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19754 monitor.abort();
19755 let _ = monitor.await;
19756 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19757 state.enabled = true;
19758 state.state = super::ModuleState::Running;
19759 state.process_alive = true;
19760 })
19761 .unwrap();
19762
19763 run_probe_cycle(&mut harness, true).await;
19764 assert_eq!(
19765 harness.module.status().unwrap().health.status,
19766 super::SupervisorHealthStatus::Ok
19767 );
19768
19769 for failures in 1..harness.runtime.health.failure_threshold {
19770 run_probe_cycle(&mut harness, false).await;
19771 let status = harness.module.status().unwrap();
19772 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19773 assert_eq!(status.health.consecutive_failures, failures);
19774 assert!(status.health.last_probe_ms.is_some());
19775 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19776 assert!(status.health.metrics.is_none());
19777 assert_eq!(status.state, super::ModuleState::Running);
19778 assert!(status.process_alive);
19779 assert_eq!(status.restart_count, 0);
19780 assert_eq!(status.lifetime_restarts, 0);
19781 assert!(status.health.last_action.is_none());
19782 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19783 }
19784
19785 run_probe_cycle(&mut harness, true).await;
19786 let recovered = harness.module.status().unwrap();
19787 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19788 assert_eq!(recovered.health.consecutive_failures, 0);
19789 assert!(recovered.health.detail.is_none());
19790 assert_eq!(
19791 recovered.health.metrics,
19792 Some(serde_json::json!({"ready": true}))
19793 );
19794 assert_eq!(recovered.lifetime_restarts, 0);
19795
19796 for failures in 1..=harness.runtime.health.failure_threshold {
19797 run_probe_cycle(&mut harness, false).await;
19798 let status = harness.module.status().unwrap();
19799 assert_eq!(status.health.consecutive_failures, failures);
19800 if failures < harness.runtime.health.failure_threshold {
19801 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19802 assert_eq!(status.state, super::ModuleState::Running);
19803 assert_eq!(status.lifetime_restarts, 0);
19804 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19805 } else {
19806 assert_eq!(
19807 status.health.status,
19808 super::SupervisorHealthStatus::Unresponsive
19809 );
19810 assert_eq!(status.state, super::ModuleState::Restarting);
19811 assert_eq!(status.restart_count, 1);
19812 assert_eq!(status.lifetime_restarts, 1);
19813 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19814 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19815 }
19816 }
19817 }
19818
19819 #[tokio::test(start_paused = true)]
19820 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19821 let mut harness = probe_harness();
19822
19823 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19824 let first_latency = match &first {
19825 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19826 other => panic!("late answer was not retained: {other:?}"),
19827 };
19828 assert!(harness.handler.observe_module_control_completion(first));
19829
19830 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19831 let second_latency = match &second {
19832 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19833 other => panic!("late answer was not retained: {other:?}"),
19834 };
19835 assert!(harness.handler.observe_module_control_completion(second));
19836
19837 assert_eq!(first_latency, Duration::from_secs(8));
19838 assert_eq!(
19839 second_latency - first_latency,
19840 Duration::from_secs(3),
19841 "latency must grow linearly with the additional stall"
19842 );
19843 let health = harness.module.status().unwrap().health;
19844 assert_eq!(health.late_answer_count, 2);
19845 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19846 }
19847
19848 #[tokio::test(start_paused = true)]
19856 async fn late_answer_clears_the_consecutive_failure_streak() {
19857 let mut harness = probe_harness();
19858
19859 time_out_without_answer(&mut harness).await;
19861 harness
19862 .module
19863 .record_health_probe_failure_for_test("[no-answer] test miss")
19864 .unwrap();
19865 assert_eq!(
19866 harness.module.status().unwrap().health.consecutive_failures,
19867 1,
19868 "precondition: the miss must be on the streak before the late answer"
19869 );
19870
19871 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
19873 assert!(matches!(
19874 late,
19875 ModuleControlRpcCompletion::LateHealthAnswer { .. }
19876 ));
19877 assert!(harness.handler.observe_module_control_completion(late));
19878
19879 let health = harness.module.status().unwrap().health;
19880 assert_eq!(
19881 health.consecutive_failures, 0,
19882 "a late answer is an answer: the streak must reset"
19883 );
19884 assert_eq!(health.late_answer_count, 1);
19885 }
19886
19887 #[tokio::test(start_paused = true)]
19888 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
19889 let mut harness = probe_harness();
19890
19891 for _ in 0..20 {
19892 time_out_without_answer(&mut harness).await;
19893 assert_eq!(
19894 harness.forwarding.health_probe_tombstone_count().unwrap(),
19895 1
19896 );
19897 }
19898 }
19899}
19900
19901#[cfg(test)]
19902mod child_env_tests {
19903 #[cfg(windows)]
19904 use super::{
19905 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
19906 };
19907 #[cfg(windows)]
19908 use std::sync::{Arc, OnceLock};
19909 #[cfg(windows)]
19910 #[test]
19911 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
19912 let spec = spec(Vec::new());
19913 let mut command = Command::new("unused");
19914 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
19915 .unwrap()
19916 .unwrap();
19917 let name = command
19918 .as_std()
19919 .get_envs()
19920 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
19921 .and_then(|(_, value)| value)
19922 .unwrap();
19923 assert_eq!(name, OsStr::new(handoff.name()));
19924 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
19925 assert!(command.as_std().get_envs().any(|(key, value)| key
19926 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
19927 && value == Some(OsStr::new("env"))));
19928 assert!(command
19929 .as_std()
19930 .get_envs()
19931 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
19932 }
19933
19934 #[cfg(windows)]
19935 #[test]
19936 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
19937 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
19938 snapshot.process_alive = true;
19939 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
19940 assert_eq!(
19941 observed_launch_nonce_source(&snapshot).as_deref(),
19942 Some("env")
19943 );
19944 let consumed = Arc::new(OnceLock::new());
19945 snapshot.pipe_consumed = Some(consumed.clone());
19946 assert_eq!(
19947 observed_launch_nonce_source(&snapshot).as_deref(),
19948 Some("env")
19949 );
19950 consumed.set(()).unwrap();
19951 assert_eq!(
19952 observed_launch_nonce_source(&snapshot).as_deref(),
19953 Some("pipe")
19954 );
19955 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
19956 assert_eq!(
19957 observed_launch_nonce_source(&snapshot).as_deref(),
19958 Some("env")
19959 );
19960 clear_current_process_facts(&mut snapshot);
19961 assert_eq!(observed_launch_nonce_source(&snapshot), None);
19962 }
19963 use super::{
19964 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
19965 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
19966 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
19967 };
19968 use std::{ffi::OsStr, path::PathBuf};
19969 use tokio::process::Command;
19970
19971 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
19972 ModuleSpec {
19973 module_id: "env-plan".to_string(),
19974 program: PathBuf::from("/nonexistent"),
19975 args: Vec::new(),
19976 env,
19977 reserved: false,
19978 reserved_prefixes: Vec::new(),
19979 protocol: ModuleProtocol::Subc,
19980 overlap: Default::default(),
19981 carry_containment: false,
19982 }
19983 }
19984
19985 #[test]
19986 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
19987 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
19988 let digest = LaunchSpecDigest::from_spec(&base);
19989 let mut reordered = base.clone();
19990 reordered.env.reverse();
19991 reordered.module_id = "different-id".into();
19992 reordered.reserved = true;
19993 reordered.overlap = Default::default();
19994 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
19995 let mut delimiter_collision = base.clone();
19996 delimiter_collision.env[0] = ("A=B".into(), "C".into());
19997 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
19998 let mut changed = base.clone();
19999 changed.protocol = ModuleProtocol::None;
20000 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20001 changed = base.clone();
20002 changed.args.push("--flag".into());
20003 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20004 changed = base.clone();
20005 changed.program = PathBuf::from("/other");
20006 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20007 changed = base.clone();
20008 changed.env[1].1 = "other".into();
20009 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20010 assert!(!format!("{digest:?}").contains("B=C"));
20011 assert!(!format!("{digest:?}").contains("[u8"));
20012 }
20013
20014 #[cfg(unix)]
20015 #[test]
20016 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
20017 use std::os::unix::ffi::OsStringExt;
20018 let mut left = spec(Vec::new());
20019 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
20020 let mut right = left.clone();
20021 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
20022 assert_eq!(
20023 left.program.to_string_lossy(),
20024 right.program.to_string_lossy()
20025 );
20026 assert_ne!(
20027 LaunchSpecDigest::from_spec(&left),
20028 LaunchSpecDigest::from_spec(&right)
20029 );
20030 }
20031
20032 #[test]
20046 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
20047 let mut command = Command::new("/nonexistent");
20048 apply_child_env(&mut command, &spec(Vec::new()));
20049 let removed = command
20050 .as_std()
20051 .get_envs()
20052 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
20053 assert!(
20054 removed,
20055 "ambient CK_LOG must be explicitly removed for an unconfigured module"
20056 );
20057
20058 let mut configured = Command::new("/nonexistent");
20059 apply_child_env(
20060 &mut configured,
20061 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
20062 );
20063 let effective = configured
20064 .as_std()
20065 .get_envs()
20066 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
20067 .last()
20068 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20069 assert_eq!(
20070 effective,
20071 Some(Some("debug".to_string())),
20072 "a module's configured CK_LOG must survive the ambient removal"
20073 );
20074 }
20075
20076 #[test]
20085 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
20086 let connection_file = std::path::Path::new("/run/subc-connection.json");
20087 let handle = SupervisorHandle::new();
20088
20089 let mut none_spec = spec(Vec::new());
20090 none_spec.protocol = ModuleProtocol::None;
20091 let mut none = Command::new("/nonexistent");
20092 let none_handoff =
20093 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
20094 .expect("protocol-none spawn args apply");
20095 assert!(
20096 none_handoff.is_none(),
20097 "protocol:none spawn must not receive a nonce descriptor"
20098 );
20099 #[cfg(windows)]
20100 assert!(!none.as_std().get_envs().any(|(key, value)| key
20101 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20102 && value.is_some()));
20103 assert!(
20104 !none.as_std().get_envs().any(|(key, value)| key
20105 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
20106 && value.is_some()),
20107 "protocol:none spawn must not name a nonce descriptor"
20108 );
20109 let none_args: Vec<String> = none
20110 .as_std()
20111 .get_args()
20112 .map(|a| a.to_string_lossy().into_owned())
20113 .collect();
20114 assert!(
20115 !none_args.iter().any(|a| a == SUBC_ARG),
20116 "protocol:none argv must not carry --subc; got {none_args:?}"
20117 );
20118 let none_has_nonce = none
20119 .as_std()
20120 .get_envs()
20121 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
20122 assert!(
20123 !none_has_nonce,
20124 "protocol:none spawn must not receive a launch nonce"
20125 );
20126 let none_has_module_id = none
20127 .as_std()
20128 .get_envs()
20129 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20130 assert!(
20131 none_has_module_id,
20132 "SUBC_MODULE_ID is inert and stays on every path"
20133 );
20134 assert!(
20135 handle.spawn_nonce(&none_spec.module_id).is_none(),
20136 "no nonce record for a process that will never present one"
20137 );
20138
20139 let wire_spec = spec(Vec::new());
20141 let mut wire = Command::new("/nonexistent");
20142 let wire_handoff =
20143 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20144 .expect("subc-wire spawn args apply");
20145 let wire_fd_env = wire
20146 .as_std()
20147 .get_envs()
20148 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20149 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20150 #[cfg(unix)]
20151 assert_eq!(
20152 wire_fd_env,
20153 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20154 "a subc-wire spawn names the pipe it will receive at descriptor 3"
20155 );
20156 #[cfg(windows)]
20157 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20158 #[cfg(not(any(unix, windows)))]
20159 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20160 let wire_args: Vec<String> = wire
20161 .as_std()
20162 .get_args()
20163 .map(|a| a.to_string_lossy().into_owned())
20164 .collect();
20165 assert_eq!(
20166 wire_args,
20167 vec![
20168 SUBC_ARG.to_string(),
20169 connection_file.to_string_lossy().into_owned()
20170 ],
20171 "a subc-wire spawn still carries --subc <path>"
20172 );
20173 assert_eq!(
20174 wire.as_std()
20175 .get_envs()
20176 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20177 !cfg!(unix),
20178 "only Windows supplies the environment nonce"
20179 );
20180 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20181 }
20182
20183 #[test]
20193 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20194 let role = |command: &Command| {
20195 command
20196 .as_std()
20197 .get_envs()
20198 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20199 .last()
20200 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20201 };
20202 let forged = spec(vec![(
20203 SUBC_SPAWN_ROLE_ENV.to_string(),
20204 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20205 )]);
20206
20207 let mut plain = Command::new("/nonexistent");
20208 apply_child_env(&mut plain, &forged);
20209 apply_spawn_role(&mut plain, SpawnRole::Plain);
20210 assert_eq!(
20211 role(&plain),
20212 Some(None),
20213 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20214 );
20215
20216 let mut candidate = Command::new("/nonexistent");
20217 apply_child_env(&mut candidate, &spec(Vec::new()));
20218 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20219 assert_eq!(
20220 role(&candidate),
20221 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20222 );
20223 }
20224
20225 #[test]
20231 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20232 let mut command = Command::new("/nonexistent");
20233 apply_child_env(
20234 &mut command,
20235 &spec(vec![
20236 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20237 ("KEPT".to_string(), "yes".to_string()),
20238 ]),
20239 );
20240 let keys: Vec<String> = command
20241 .as_std()
20242 .get_envs()
20243 .filter(|(_, value)| value.is_some())
20244 .map(|(key, _)| key.to_string_lossy().into_owned())
20245 .collect();
20246 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20247 assert!(
20248 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20249 "daemon-private capture key leaked to the child: {keys:?}"
20250 );
20251 }
20252}
20253
20254#[cfg(test)]
20255mod jitter_tests {
20256 use super::jittered_health_delay;
20257 use std::{collections::HashSet, time::Duration};
20258
20259 const FLEET: [&str; 14] = [
20268 "aft",
20269 "alfonso-core",
20270 "magic-context",
20271 "broca",
20272 "thalamus",
20273 "quota",
20274 "engram",
20275 "plexus",
20276 "cerebellum",
20277 "astrocyte",
20278 "synapse",
20279 "subc-mcp",
20280 "cortexkit-credentials",
20281 "subc-federation",
20282 ];
20283
20284 #[test]
20292 fn probe_delays_disperse_across_the_fleet() {
20293 let cadence = Duration::from_secs(30);
20294 let delays: HashSet<Duration> = FLEET
20295 .iter()
20296 .map(|id| jittered_health_delay(id, 0, cadence))
20297 .collect();
20298 assert_eq!(
20299 delays.len(),
20300 FLEET.len(),
20301 "every supervised module must land on its own probe offset"
20302 );
20303 }
20304
20305 #[test]
20311 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20312 let cadence = Duration::from_secs(30);
20313 let span = cadence / 10;
20314 for id in FLEET {
20315 for probe_index in 0..8 {
20316 let delay = jittered_health_delay(id, probe_index, cadence);
20317 assert!(
20318 delay >= cadence,
20319 "{id}#{probe_index}: jitter must not shorten the cadence"
20320 );
20321 assert!(
20322 delay < cadence + span,
20323 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20324 );
20325 }
20326 }
20327 }
20328
20329 #[test]
20335 fn a_module_offset_is_stable_across_restarts() {
20336 let cadence = Duration::from_secs(30);
20337 for id in FLEET {
20338 assert_eq!(
20339 jittered_health_delay(id, 0, cadence),
20340 jittered_health_delay(id, 0, cadence),
20341 "{id}: the same module and probe index must produce the same offset"
20342 );
20343 }
20344 }
20345
20346 #[test]
20348 fn zero_cadence_yields_zero_delay() {
20349 assert_eq!(
20350 jittered_health_delay("aft", 0, Duration::ZERO),
20351 Duration::ZERO
20352 );
20353 }
20354}
20355
20356#[cfg(all(test, target_os = "linux"))]
20357mod cgroup_placement_tests {
20358 use super::{
20359 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20360 SuperviseError, SupervisedChild,
20361 };
20362 use crate::stderr_tail::{StderrRing, StderrTailConfig};
20363 use cortexkit_test_support::ScratchDir;
20364 use std::{
20365 fs, io,
20366 path::{Path, PathBuf},
20367 sync::{Arc, Mutex},
20368 };
20369 use tokio::process::Command;
20370
20371 #[tokio::test]
20372 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20373 use super::*;
20374 let dir = ScratchDir::new("unique-spawn-cgroups");
20375 let root = PathBuf::from(format!(
20376 "/sys/fs/cgroup/subc-unique-test-{}-{}",
20377 std::process::id(),
20378 unix_ms_now()
20379 ));
20380 if let Err(error) = fs::create_dir(&root) {
20381 assert!(
20382 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20383 "required cgroup test cannot execute: {error}"
20384 );
20385 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20386 return;
20387 }
20388 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20389 let group_count = || {
20390 fs::read_dir(root.join("subc-modules"))
20391 .unwrap()
20392 .map(|entry| entry.unwrap().file_type().unwrap())
20393 .filter(|kind| kind.is_dir())
20394 .count()
20395 };
20396 let supervisor =
20397 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20398 .with_cgroup_placement(Some(placement.clone()));
20399 let runtime = supervisor.runtime_config();
20400 let mut spec = ModuleSpec {
20401 module_id: "unique-spawn".into(),
20402 program: PathBuf::from("/bin/sleep"),
20403 args: vec!["60".into()],
20404 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20405 .into_iter()
20406 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20407 .collect(),
20408 reserved: false,
20409 reserved_prefixes: vec![],
20410 protocol: ModuleProtocol::None,
20411 overlap: Default::default(),
20412 carry_containment: false,
20413 };
20414 let spawn = |spec: &ModuleSpec| {
20415 spawn_child(
20416 spec,
20417 None,
20418 None,
20419 &runtime.stderr_ring,
20420 None,
20421 &runtime.child_roster,
20422 runtime.spawn_delivery(),
20423 Some(&placement),
20424 runtime.register_timeout,
20425 )
20426 .unwrap()
20427 };
20428 let mut live = spawn(&spec);
20429 for _ in 0..3 {
20430 let next = spawn(&spec);
20432 assert_ne!(live.module_id, next.module_id);
20433 live.start_kill().unwrap();
20434 live.wait().await.unwrap();
20435 live = next;
20436 assert!(
20437 live.child.try_wait().unwrap().is_none(),
20438 "retiring the old slot must not kill the replacement"
20439 );
20440 assert_eq!(
20441 group_count(),
20442 1,
20443 "only the live spawn's cgroup should remain"
20444 );
20445 }
20446 supervisor.begin_daemon_shutdown();
20447 let reap = tokio::spawn(async move {
20448 live.wait().await.unwrap();
20449 });
20450 supervisor
20451 .end_children_for_daemon_shutdown(false, std::future::pending())
20452 .await;
20453 reap.await.unwrap();
20454 assert_eq!(group_count(), 0);
20455 spec.program = PathBuf::from("/bin/true");
20457 spec.args.clear();
20458 let fresh_roster = ChildRoster::default();
20459 let mut short = spawn_child(
20460 &spec,
20461 None,
20462 None,
20463 &runtime.stderr_ring,
20464 None,
20465 &fresh_roster,
20466 None,
20467 Some(&placement),
20468 runtime.register_timeout,
20469 )
20470 .unwrap();
20471 short.wait().await.unwrap();
20472 assert_eq!(group_count(), 0);
20473 spec.module_id = "_".repeat(255);
20474 let mut long_id = spawn_child(
20475 &spec,
20476 None,
20477 None,
20478 &runtime.stderr_ring,
20479 None,
20480 &fresh_roster,
20481 None,
20482 Some(&placement),
20483 runtime.register_timeout,
20484 )
20485 .unwrap();
20486 long_id.wait().await.unwrap();
20487 assert_eq!(
20488 group_count(),
20489 0,
20490 "valid long module IDs must not exceed cgroup NAME_MAX"
20491 );
20492 fs::remove_dir(root.join("subc-modules")).unwrap();
20493 fs::remove_dir(root).unwrap();
20494 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20495 }
20496
20497 #[test]
20498 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20499 let path = Path::new("/definitely-missing-subc-cgroup");
20500 let mut command = Command::new("true");
20501 let error = apply_cgroup_placement(
20502 &mut command,
20503 &ModuleSpec {
20504 module_id: "broken-cgroup".to_string(),
20505 program: PathBuf::from("true"),
20506 args: Vec::new(),
20507 env: Vec::new(),
20508 reserved: false,
20509 reserved_prefixes: Vec::new(),
20510 protocol: ModuleProtocol::Subc,
20511 overlap: Default::default(),
20512 carry_containment: false,
20513 },
20514 path,
20515 )
20516 .expect_err("a parent cgroup open failure must reject the supervised spawn");
20517 let reason = error.to_string();
20518
20519 assert!(
20520 matches!(error, SuperviseError::Cgroup { .. }),
20521 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20522 );
20523 assert!(
20524 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20525 "parent cgroup open failure must name cgroup.procs: {reason}"
20526 );
20527 }
20528
20529 #[tokio::test]
20530 async fn reaping_a_child_removes_its_empty_module_cgroup() {
20531 let root = ScratchDir::new("supervisor-reap-cgroup");
20532 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20533 let placement = subc_cgroup::prepare_at(&root)
20534 .expect("prepare scratch cgroup root")
20535 .expect("scratch root has a cgroup.procs marker");
20536 let module_id = "reaped-module";
20537 let module = placement
20538 .module_path(module_id)
20539 .expect("create scratch module cgroup");
20540 let child = Command::new("true")
20541 .env("XDG_DATA_HOME", root.path())
20542 .env("XDG_RUNTIME_DIR", root.path())
20543 .env("XDG_CONFIG_HOME", root.path())
20544 .spawn()
20545 .expect("spawn short-lived child");
20546 let pid = child.id().expect("spawned child has pid");
20547 let mut child = SupervisedChild {
20548 registration_channel: None,
20549 registration_spawn: None,
20550 live_program_binding: None,
20551 registered_config_connection: None,
20552 admission_base: None,
20553 registration_declaration: None,
20554 _program_binding: None,
20555 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20556 child: crate::supervise::containment::ProcessChild::plain(child, pid),
20557 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20558 protocol: ModuleProtocol::Subc,
20559 module_id: module_id.to_string(),
20560 cgroup_placement: Some(placement),
20561 stdout_pump: None,
20562 stderr_pump: None,
20563 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20564 spawned_at: tokio::time::Instant::now(),
20565 config_delivery: None,
20566 registration_deadline: None,
20567 registration_watch: None,
20568 spawned_at_ms: 0,
20569 spawned_from: PathBuf::from("true"),
20570 launch_spec_digest: LaunchSpecDigest([0; 32]),
20571 spawned_file_identity: None,
20572 process_start_time: None,
20573 process_identity: None,
20574 pid,
20575 roster_guard: None,
20576 #[cfg(target_os = "macos")]
20577 privacy_exec: None,
20578 spawn_failure: None,
20579 admission_kill_sent: false,
20580 };
20581
20582 child.wait().await.expect("reap short-lived child");
20583
20584 assert!(
20585 !module.exists(),
20586 "waiting for a non-carried child must remove its empty cgroup before returning"
20587 );
20588 child.finish_exit(false).await.unwrap();
20589 }
20590
20591 #[test]
20592 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20593 let root = ScratchDir::new("supervisor-non-empty-cgroup");
20594 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20595 let placement = subc_cgroup::prepare_at(&root)
20596 .expect("prepare scratch cgroup root")
20597 .expect("scratch root has a cgroup.procs marker");
20598 let module = placement
20599 .module_path("surviving-module")
20600 .expect("create scratch module cgroup");
20601 fs::write(module.join("surviving-process"), b"still present")
20602 .expect("make scratch cgroup non-empty");
20603 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20604
20605 remove_module_cgroup(&placement, "surviving-module");
20606
20607 let logs = crate::router::test_log::captured_logs(&logs);
20608 assert!(
20609 module.exists(),
20610 "failed removal must leave the cgroup intact"
20611 );
20612 assert!(
20613 logs.contains("could not remove module cgroup after process exit; continuing teardown")
20614 && logs.contains("surviving-module"),
20615 "best-effort removal must report the failure without returning it: {logs}"
20616 );
20617 }
20618
20619 #[test]
20620 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20621 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20622 let reason = SuperviseError::Spawn {
20623 program: PathBuf::from("/bin/true"),
20624 source: io::Error::from_raw_os_error(13),
20625 cgroup_path: Some(cgroup_path.clone()),
20626 }
20627 .to_string();
20628
20629 assert!(
20630 reason.contains(&cgroup_path.display().to_string()),
20631 "a pre_exec spawn failure must name the cgroup path: {reason}"
20632 );
20633 }
20634}
20635
20636#[cfg(test)]
20637mod spawn_subscriber_lag_tests {
20638 use super::*;
20639
20640 #[tokio::test]
20645 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20646 let feed = SpawnEventFeed::default();
20647 feed.configure_incarnation("lag-incarnation".to_string());
20648 let (tx, mut rx) = mpsc::channel(1);
20651 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20652 .expect("subscribe");
20653 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20654 for index in 0..emitted {
20655 feed.emit_spawned(&format!("lag-module-{index}"), None, 1000, 0);
20656 tokio::task::yield_now().await;
20659 }
20660 assert_eq!(
20661 feed.subscriber_count(),
20662 0,
20663 "the lagged subscriber must be removed"
20664 );
20665
20666 let mut data = Vec::new();
20667 let mut last = None;
20668 loop {
20669 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20670 .await
20671 .expect("the forwarder must finish once the subscriber is dropped");
20672 let Some(outbound) = next else { break };
20673 let frame = outbound.frame;
20674 if frame.header.ty == FrameType::StreamData {
20675 assert!(last.is_none(), "no data may follow the terminal frame");
20676 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20677 data.push(event.cursor.seq);
20678 } else {
20679 assert!(last.is_none(), "exactly one terminal frame");
20680 last = Some(frame);
20681 }
20682 }
20683 assert!(!data.is_empty(), "queued frames drain before the terminal");
20684 for pair in data.windows(2) {
20685 assert_eq!(
20686 pair[1],
20687 pair[0] + 1,
20688 "queued frames arrive dense and in order"
20689 );
20690 }
20691 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20692 assert_eq!(terminal.header.ty, FrameType::Error);
20693 assert_eq!(terminal.header.corr, 7);
20694 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20695 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20696 let detail = body.detail.expect("lagged error carries detail");
20697 assert_eq!(
20698 detail["first_undelivered_cursor"]["seq"],
20699 data.last().unwrap() + 1,
20700 "the named cursor is the first event the subscriber did not receive"
20701 );
20702 assert_eq!(
20703 detail["first_undelivered_cursor"]["daemon_incarnation"],
20704 "lag-incarnation"
20705 );
20706 }
20707}
20708
20709#[cfg(test)]
20710mod terminal_history_read_concurrency_tests {
20711 use super::*;
20712 use crate::terminal_journal::read_pause;
20713 use cortexkit_test_support::ScratchDir;
20714 use std::sync::mpsc as std_mpsc;
20715
20716 fn journaled_ring(
20717 journal: &Arc<crate::terminal_journal::TerminalJournal>,
20718 ) -> Arc<Mutex<TerminalRing>> {
20719 Arc::new(Mutex::new(
20720 TerminalRing::new(TerminalRingConfig::default(), 1)
20721 .with_journal(Some(Arc::clone(journal))),
20722 ))
20723 }
20724
20725 fn crash(at_ms: u64) -> ExitReport {
20726 ExitReport {
20727 kind: ExitKind::Crash,
20728 code: Some(1),
20729 signal: None,
20730 at_ms,
20731 }
20732 }
20733
20734 fn record_within(
20737 module_id: &'static str,
20738 ring: &Arc<Mutex<TerminalRing>>,
20739 at_ms: u64,
20740 bound: Duration,
20741 ) -> bool {
20742 let ring = Arc::clone(ring);
20743 let (done, done_rx) = std_mpsc::channel();
20744 std::thread::spawn(move || {
20745 record_terminal(
20746 module_id,
20747 &ring,
20748 &SpawnEventFeed::default(),
20749 &crash(at_ms),
20750 TerminalDisposition::Restarting,
20751 );
20752 let _ = done.send(());
20753 });
20754 done_rx.recv_timeout(bound).is_ok()
20755 }
20756
20757 #[test]
20762 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20763 let dir = ScratchDir::new("terminal-history-concurrent-read");
20764 let path = dir.join("terminals.jsonl");
20765 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20766 path.clone(),
20767 "daemon".into(),
20768 ));
20769 let reader_ring = journaled_ring(&journal);
20770 let other_ring = journaled_ring(&journal);
20771 assert!(record_within(
20772 "reader-module",
20773 &reader_ring,
20774 10,
20775 Duration::from_secs(5)
20776 ));
20777
20778 let (started, release) = read_pause::install(&path);
20779 let reading = {
20780 let ring = Arc::clone(&reader_ring);
20781 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20782 };
20783 started
20784 .recv_timeout(Duration::from_secs(5))
20785 .expect("the history read reached its pause");
20786
20787 let bound = Duration::from_secs(1);
20788 assert!(
20789 record_within("other-module", &other_ring, 20, bound),
20790 "another module's exit waited on a history read (journal writer held)"
20791 );
20792 assert!(
20793 record_within("reader-module", &reader_ring, 30, bound),
20794 "the read module's own exit waited on its history read (ring held)"
20795 );
20796
20797 drop(release);
20798 let paused = reading.join().unwrap();
20799 assert_eq!(
20800 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20801 vec![10],
20802 "an exit recorded after the read began lands in neither half of it"
20803 );
20804 assert_eq!(paused.journal_skipped_lines, 0);
20805 assert_eq!(paused.journal_read_errors, 0);
20806
20807 let after = durable_terminal_history_of(&reader_ring, "reader-module");
20808 assert_eq!(
20809 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20810 vec![10, 30],
20811 "the next read merges ring and journal with no duplicate"
20812 );
20813 assert_eq!(after.journal_skipped_lines, 0);
20814 }
20815}
20816
20817#[cfg(test)]
20822mod stderr_settle_tests {
20823 use std::{
20824 future::Future,
20825 io,
20826 pin::Pin,
20827 sync::{Arc, Mutex},
20828 task::{Context, Poll},
20829 time::Duration,
20830 };
20831
20832 use tokio::{
20833 io::{AsyncRead, ReadBuf},
20834 sync::oneshot,
20835 time::Instant,
20836 };
20837
20838 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20839 use crate::stderr_tail::{
20840 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20841 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20842 };
20843
20844 const BOUND: Duration = Duration::from_millis(250);
20845
20846 struct HeldReader {
20850 before: Option<Vec<u8>>,
20851 gate: Option<oneshot::Receiver<()>>,
20852 after: io::Cursor<Vec<u8>>,
20853 }
20854
20855 impl AsyncRead for HeldReader {
20856 fn poll_read(
20857 mut self: Pin<&mut Self>,
20858 cx: &mut Context<'_>,
20859 buf: &mut ReadBuf<'_>,
20860 ) -> Poll<io::Result<()>> {
20861 if let Some(bytes) = self.before.take() {
20862 buf.put_slice(&bytes);
20863 return Poll::Ready(Ok(()));
20864 }
20865 if let Some(gate) = self.gate.as_mut() {
20866 match Pin::new(gate).poll(cx) {
20867 Poll::Pending => return Poll::Pending,
20868 Poll::Ready(_) => self.gate = None,
20869 }
20870 }
20871 Pin::new(&mut self.after).poll_read(cx, buf)
20872 }
20873 }
20874
20875 struct DiscardSink;
20876
20877 impl OutputSink for DiscardSink {
20878 fn write_line(&mut self, _line: &[u8]) {}
20879 }
20880
20881 fn line(text: &str) -> TailEntry {
20882 TailEntry::Line {
20883 text: text.to_string(),
20884 truncated: false,
20885 at_ms: None,
20886 }
20887 }
20888
20889 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
20890 ring.lock().unwrap()
20891 }
20892
20893 fn held_pump(
20897 ring: &Arc<Mutex<StderrRing>>,
20898 before: &str,
20899 after: &str,
20900 ) -> (StderrPump, oneshot::Sender<()>) {
20901 let generation = lock(ring).begin_process();
20902 let (release, gate) = oneshot::channel();
20903 let reader = HeldReader {
20904 before: Some(before.as_bytes().to_vec()),
20905 gate: Some(gate),
20906 after: io::Cursor::new(after.as_bytes().to_vec()),
20907 };
20908 let task = tokio::spawn(pump_stderr_to(
20909 reader,
20910 Arc::clone(ring),
20911 generation,
20912 DiscardSink,
20913 ));
20914 (StderrPump { task, generation }, release)
20915 }
20916
20917 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
20918 for _ in 0..1000 {
20919 if done(&lock(ring)) {
20920 return;
20921 }
20922 tokio::time::sleep(Duration::from_millis(1)).await;
20923 }
20924 panic!(
20925 "ring never reached the expected state: {:?}",
20926 lock(ring).snapshot(None, None)
20927 );
20928 }
20929
20930 #[tokio::test(start_paused = true)]
20931 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
20932 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20933 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
20934
20935 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
20936 let before_release = lock(&ring).snapshot(None, None);
20937 assert!(
20938 matches!(before_release.capture, CaptureState::Incomplete { .. }),
20939 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
20940 );
20941
20942 let next = lock(&ring).begin_process();
20945 lock(&ring).push_line_from(next, "next process booting");
20946 release.send(()).unwrap();
20947 wait_until(&ring, |ring| {
20948 ring.snapshot(None, None).capture == CaptureState::Captured
20949 })
20950 .await;
20951
20952 assert_eq!(
20953 untimed(lock(&ring).snapshot(None, None).entries),
20954 vec![
20955 line("booting"),
20956 line("config error: missing storage"),
20957 TailEntry::ProcessStart,
20958 line("next process booting"),
20959 ],
20960 "the crash's last line must survive a slow reader and stay in the crashed process's section"
20961 );
20962 }
20963
20964 #[tokio::test(start_paused = true)]
20965 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
20966 ) {
20967 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20968 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
20971
20972 let started = Instant::now();
20973 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
20974 assert_eq!(
20975 started.elapsed(),
20976 BOUND,
20977 "the restart must wait exactly the bound for a pipe that stays open, no longer"
20978 );
20979
20980 let next = lock(&ring).begin_process();
20981 lock(&ring).push_line_from(next, "next process booting");
20982 tokio::time::sleep(Duration::from_secs(60)).await;
20983
20984 let snapshot = lock(&ring).snapshot(None, None);
20985 match &snapshot.capture {
20986 CaptureState::Incomplete { reason } => assert!(
20987 reason.contains("had not reached EOF") && reason.contains("250ms"),
20988 "the reason must say what is missing and after how long: {reason}"
20989 ),
20990 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
20991 }
20992 assert_eq!(
20993 untimed(snapshot.entries),
20994 vec![
20995 line("parent exiting"),
20996 TailEntry::ProcessStart,
20997 line("next process booting"),
20998 ]
20999 );
21000 }
21001
21002 #[tokio::test(start_paused = true)]
21003 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
21004 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21005 let (pump, release) = held_pump(&ring, "one\n", "two\n");
21006 release.send(()).unwrap();
21007
21008 settle_stderr_pump("clean", &ring, pump, BOUND).await;
21009
21010 let snapshot = lock(&ring).snapshot(None, None);
21011 assert_eq!(snapshot.capture, CaptureState::Captured);
21012 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
21013 }
21014
21015 #[tokio::test(start_paused = true)]
21016 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
21017 const BURST: usize = 150;
21018 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
21019 let path = capture.join("two-pipe.stderr.log");
21020 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
21021 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21022 let generation = lock(&ring).begin_process();
21023 let stdout = (0..BURST)
21024 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
21025 .collect::<String>();
21026 let stderr = (0..BURST)
21027 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
21028 .collect::<String>();
21029 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
21034 let (release, gate) = oneshot::channel();
21035 let reader = HeldReader {
21036 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
21037 gate: Some(gate),
21038 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
21039 };
21040 let (finished, completion) = oneshot::channel();
21041 let stdout_sink = sink.clone();
21042 let pump = tokio::spawn(async move {
21043 pump_stdout_to(reader, stdout_sink).await;
21044 finished.send(()).unwrap();
21045 });
21046 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
21047
21048 let started = Instant::now();
21049 settle_stdout_pump("two-pipe", pump, BOUND).await;
21050 assert_eq!(
21051 started.elapsed(),
21052 BOUND,
21053 "a late reader must not hold up restart"
21054 );
21055 assert_eq!(
21056 std::fs::read_to_string(&path).unwrap().lines().count(),
21057 BURST + 1
21058 );
21059
21060 release
21061 .send(())
21062 .expect("stdout reader must survive the exit drain bound");
21063 tokio::time::timeout(Duration::from_secs(1), completion)
21064 .await
21065 .expect("released stdout reader must reach EOF")
21066 .expect("stdout capture must finish at EOF, not be cancelled");
21067 let contents = std::fs::read_to_string(&path).unwrap();
21068 let mut lines = contents
21069 .lines()
21070 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
21071 .collect::<Vec<_>>();
21072 let mut expected = (0..BURST)
21073 .flat_map(|i| {
21074 [
21075 format!("out-{i:04}-{}", "o".repeat(64)),
21076 format!("err-{i:04}-{}", "e".repeat(64)),
21077 ]
21078 })
21079 .collect::<Vec<_>>();
21080 lines.sort_unstable();
21081 expected.sort_unstable();
21082 assert_eq!(
21083 lines, expected,
21084 "both pipes must retain every complete line exactly once"
21085 );
21086 }
21087}
21088
21089#[cfg(all(test, windows))]
21103mod job_containment_tests {
21104 use super::*;
21105 use cortexkit_test_support::ScratchDir;
21106 use std::{
21107 path::{Path, PathBuf},
21108 sync::{Arc, Mutex},
21109 time::{Duration, Instant},
21110 };
21111
21112 fn stub_path() -> PathBuf {
21118 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
21119 assert!(
21120 path.exists(),
21121 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
21122 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21123 path.display()
21124 );
21125 path
21126 }
21127
21128 fn read_grandchild_pid(path: &Path) -> u32 {
21130 let deadline = Instant::now() + Duration::from_secs(10);
21131 loop {
21132 if let Ok(contents) = std::fs::read_to_string(path) {
21133 if let Ok(pid) = contents.trim().parse() {
21134 return pid;
21135 }
21136 }
21137 assert!(
21138 Instant::now() < deadline,
21139 "the stub never recorded a grandchild pid at {}",
21140 path.display()
21141 );
21142 std::thread::sleep(Duration::from_millis(10));
21143 }
21144 }
21145
21146 struct Fixture {
21149 _dir: ScratchDir,
21150 _roster: ChildRoster,
21151 module_id: String,
21152 grandchild: u32,
21153 child: Option<SupervisedChild>,
21154 registry: Arc<Registry>,
21155 snapshot: Arc<Mutex<SupervisorSnapshot>>,
21156 terminal_ring: Arc<Mutex<TerminalRing>>,
21157 spawn_events: SpawnEventFeed,
21158 }
21159
21160 fn fixture(label: &str, module_id: &str) -> Fixture {
21161 fixture_with_carry(label, module_id, false)
21162 }
21163
21164 fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21165 let dir = ScratchDir::new(label);
21166 let pid_file = dir.join("grandchild.pid");
21167 let supervisor = Supervisor::new_for_test(
21168 Arc::new(Registry::default()),
21169 RestartPolicy::new(3, Duration::ZERO),
21170 );
21171 let runtime = supervisor.runtime_config();
21172 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21173 let spec = ModuleSpec {
21174 module_id: module_id.to_string(),
21175 program: stub_path(),
21176 args: Vec::new(),
21180 env: vec![
21181 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21182 (
21183 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21184 pid_file.display().to_string(),
21185 ),
21186 ],
21187 reserved: false,
21188 reserved_prefixes: Vec::new(),
21189 protocol: ModuleProtocol::Subc,
21190 overlap: Default::default(),
21191 carry_containment,
21192 };
21193 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21194 .expect("spawn the supervised fixture");
21195 let grandchild = read_grandchild_pid(&pid_file);
21196 Fixture {
21197 _dir: dir,
21198 _roster: runtime.child_roster.clone(),
21199 module_id: module_id.to_string(),
21200 grandchild,
21201 child: Some(child),
21202 registry: Arc::new(Registry::default()),
21203 snapshot,
21204 terminal_ring: Arc::clone(&runtime.terminal_ring),
21205 spawn_events: SpawnEventFeed::default(),
21206 }
21207 }
21208
21209 async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21210 let label = if carry {
21211 "admission-kill-carried-tree"
21212 } else {
21213 "admission-kill-plain-tree"
21214 };
21215 let mut fixture = fixture_with_carry(label, label, carry);
21216 let descendant = subc_os::Process::open(fixture.grandchild)
21217 .unwrap()
21218 .expect("the stub descendant must be alive before admission kill");
21219 assert!(descendant.observe().is_some());
21220 let child = fixture.child.as_mut().unwrap();
21221 assert_eq!(child.child.container().is_some(), carry);
21222 assert_eq!(child.job.is_some(), !carry);
21223
21224 child.start_admission_kill().unwrap();
21225 assert!(child.admission_kill_sent);
21226 let status = timeout(Duration::from_secs(10), child.wait())
21227 .await
21228 .unwrap()
21229 .unwrap();
21230 assert_eq!(
21231 status.code(),
21232 Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21233 "containment teardown must preserve the leader's distinctive status"
21234 );
21235 assert!(
21238 descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21239 "admission kill left a descendant running before exit finalization"
21240 );
21241 assert_eq!(child.child.container().is_some(), carry);
21242 assert_eq!(child.job.is_some(), !carry);
21243 child.finish_exit(false).await.unwrap();
21244 }
21245
21246 #[tokio::test]
21247 async fn admission_kill_reaps_grandchild_with_plain_job() {
21248 assert_admission_kill_reaps_grandchild(false).await;
21249 }
21250
21251 #[tokio::test]
21252 async fn admission_kill_reaps_grandchild_with_carried_job() {
21253 assert_admission_kill_reaps_grandchild(true).await;
21254 }
21255
21256 #[tokio::test]
21257 async fn admission_kill_retires_registration_before_wait() {
21258 let mut fixture = fixture("admission-registration", "admission-registration");
21259 let handle = SupervisorHandle::new();
21260 let spawn =
21261 RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21262 spawn.ready();
21263 let child = fixture.child.as_mut().unwrap();
21264 child.registration_spawn = Some(spawn.clone());
21265 child.start_admission_kill().unwrap();
21266 assert!(
21267 !spawn.launch_ready(),
21268 "admission kill must retire before its caller can await the child"
21269 );
21270 timeout(Duration::from_secs(10), child.wait())
21271 .await
21272 .unwrap()
21273 .unwrap();
21274 child.finish_exit(false).await.unwrap();
21275 }
21276
21277 impl Fixture {
21278 async fn drain(&mut self) {
21280 let child = self
21281 .child
21282 .take()
21283 .expect("the fixture child is still present");
21284 drain_child_to_state(
21285 &self.module_id,
21286 ModuleProtocol::Subc,
21287 StopNotice::NotSent,
21290 &self.registry,
21291 None,
21292 &self.snapshot,
21293 &self.terminal_ring,
21294 &self.spawn_events,
21295 child,
21296 Duration::from_millis(500),
21297 ModuleState::Stopped,
21298 Some(false),
21299 &FailureContext::operator(),
21300 )
21301 .await
21302 .expect("drain the supervised fixture");
21303 }
21304 }
21305
21306 #[tokio::test]
21312 async fn teardown_reaps_the_grandchild() {
21313 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21314 let grandchild = fixture.grandchild;
21315
21316 assert!(
21317 subc_jobobject::process_exists(grandchild),
21318 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21319 );
21320
21321 fixture.drain().await;
21322
21323 assert!(
21324 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21325 "grandchild {grandchild} outlived module teardown: the tree was not contained"
21326 );
21327 }
21328
21329 #[test]
21342 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21343 let dir = ScratchDir::new("teardown-uncontained");
21344 let pid_file = dir.join("grandchild.pid");
21345 let mut child = std::process::Command::new(stub_path())
21346 .env("FAKE_AFT_NEVER_CONNECT", "1")
21347 .env(
21348 "FAKE_AFT_GRANDCHILD_PID_FILE",
21349 pid_file.display().to_string(),
21350 )
21351 .stdin(std::process::Stdio::null())
21352 .stdout(std::process::Stdio::null())
21353 .stderr(std::process::Stdio::null())
21354 .spawn()
21355 .expect("spawn the uncontained fixture");
21356 let grandchild = read_grandchild_pid(&pid_file);
21357
21358 child.kill().expect("kill the direct child");
21360 let _ = child.wait();
21361
21362 assert!(
21363 subc_jobobject::process_exists(grandchild),
21364 "grandchild {grandchild} died with the direct child, so this control no longer \
21365 distinguishes contained from uncontained teardown and the regression test is \
21366 passing vacuously"
21367 );
21368
21369 kill_tree(grandchild);
21372 }
21373
21374 #[tokio::test]
21383 async fn dropping_containment_reaps_the_grandchild() {
21384 let mut fixture = fixture("drop-containment", "tree-drop");
21385 let grandchild = fixture.grandchild;
21386
21387 assert!(subc_jobobject::process_exists(grandchild));
21388
21389 fixture.child.as_mut().expect("child present").job = None;
21391
21392 assert!(
21393 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21394 "grandchild {grandchild} survived the containment handle closing, so a daemon \
21395 crash would leave the tree behind"
21396 );
21397 }
21398
21399 fn kill_tree(pid: u32) {
21401 let _ = std::process::Command::new("taskkill.exe")
21402 .args(["/PID", &pid.to_string(), "/T", "/F"])
21403 .stdin(std::process::Stdio::null())
21404 .stdout(std::process::Stdio::null())
21405 .stderr(std::process::Stdio::null())
21406 .status();
21407 assert!(
21408 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21409 "could not clean up grandchild {pid}"
21410 );
21411 }
21412}
21413
21414#[cfg(test)]
21415mod privacy_trampoline_configuration_tests {
21416 #[tokio::test]
21417 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21418 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21419 #[cfg(target_os = "macos")]
21420 {
21421 let supervisor = super::Supervisor::new(
21422 std::sync::Arc::new(crate::Registry::default()),
21423 super::RestartPolicy::default(),
21424 );
21425 let error = supervisor.spawn(spec()).unwrap_err();
21426 assert!(
21427 error
21428 .to_string()
21429 .contains("no privacy trampoline configured"),
21430 "{error}"
21431 );
21432 }
21433 }
21434
21435 #[tokio::test]
21436 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21437 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21438 #[cfg(target_os = "macos")]
21439 {
21440 let supervisor = super::Supervisor::new(
21441 std::sync::Arc::new(crate::Registry::default()),
21442 super::RestartPolicy::default(),
21443 )
21444 .with_privacy_trampoline(std::env::current_exe().unwrap());
21445 let error = supervisor.spawn(spec()).unwrap_err();
21446 assert!(
21447 error
21448 .to_string()
21449 .contains("binary does not implement the privacy trampoline protocol"),
21450 "{error}"
21451 );
21452 }
21453 }
21454
21455 #[cfg(target_os = "macos")]
21456 fn spec() -> super::ModuleSpec {
21457 super::ModuleSpec {
21458 module_id: "privacy-configuration".into(),
21459 program: "/bin/sleep".into(),
21460 args: vec!["30".into()],
21461 env: vec![],
21462 reserved: false,
21463 reserved_prefixes: vec![],
21464 protocol: subc_control::ModuleProtocol::None,
21465 overlap: super::ModuleOverlap::Exclusive,
21466 carry_containment: false,
21467 }
21468 }
21469}
21470
21471#[cfg(test)]
21472mod privacy_exec_boundary_tests {
21473 #[cfg(target_os = "macos")]
21474 use super::*;
21475 #[cfg(target_os = "macos")]
21476 use std::{
21477 io::{Read, Write},
21478 net::{TcpListener, TcpStream},
21479 };
21480
21481 #[cfg(target_os = "macos")]
21484 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21485 if let Some((_, path)) = spec
21486 .env
21487 .iter()
21488 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21489 {
21490 let mut barrier = TcpStream::connect(path).unwrap();
21491 barrier
21492 .set_read_timeout(Some(Duration::from_secs(30)))
21493 .unwrap();
21494 barrier.write_all(&pid.to_ne_bytes()).unwrap();
21495 let mut release = [0];
21496 barrier.read_exact(&mut release).unwrap();
21497 assert_eq!(&release, b"X");
21498 }
21499 }
21500
21501 #[cfg(target_os = "macos")]
21502 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21503 ModuleSpec {
21504 module_id: "privacy-boundary".into(),
21505 program: program.into(),
21506 args: args.iter().map(|arg| (*arg).into()).collect(),
21507 env: vec![],
21508 reserved: false,
21509 reserved_prefixes: vec![],
21510 protocol: ModuleProtocol::None,
21511 overlap: ModuleOverlap::Exclusive,
21512 carry_containment: false,
21513 }
21514 }
21515
21516 #[cfg(target_os = "macos")]
21517 async fn accept(listener: TcpListener) -> TcpStream {
21518 let listener = tokio::net::TcpListener::from_std({
21520 listener.set_nonblocking(true).unwrap();
21521 listener
21522 })
21523 .unwrap();
21524 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21525 .await
21526 .unwrap()
21527 .unwrap();
21528 let stream = stream.into_std().unwrap();
21529 stream.set_nonblocking(false).unwrap();
21530 stream
21531 .set_read_timeout(Some(Duration::from_secs(30)))
21532 .unwrap();
21533 stream
21534 }
21535
21536 #[tokio::test]
21537 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21538 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21539 #[cfg(target_os = "macos")]
21540 {
21541 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21542 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21545 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21546 let record = root.join("live-children.json");
21547 let supervisor =
21548 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21549 .with_live_children_record(&record);
21550 let runtime = supervisor.runtime_config();
21551 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21552 let mut spec = spec("/bin/sleep", &["30"]);
21553 spec.env = vec![
21554 (
21555 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21556 exec_listener.local_addr().unwrap().to_string(),
21557 ),
21558 (
21559 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21560 sample_listener.local_addr().unwrap().to_string(),
21561 ),
21562 ];
21563 let spawn = tokio::task::spawn_blocking(move || {
21564 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21565 });
21566 let mut sample = accept(sample_listener).await;
21567 let mut pid = [0; 4];
21568 sample.read_exact(&mut pid).unwrap();
21569 let pid = u32::from_ne_bytes(pid);
21570 let mut exec = accept(exec_listener).await;
21571 let mut ready = [0];
21572 exec.read_exact(&mut ready).unwrap();
21573 assert_eq!(&ready, b"R");
21574 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21577 assert_eq!(
21578 observe_spawned_image(pid).unwrap().executable,
21579 Some(trampoline)
21580 );
21581 sample.write_all(b"X").unwrap();
21582 let mut child = spawn.await.unwrap();
21583 let early = crate::live_children::read_record(&record).unwrap();
21584 assert_eq!(early.len(), 1);
21585 assert_eq!(early[0].pid, pid);
21586 assert_eq!(
21587 early[0].executable, None,
21588 "unconfirmed trampoline image entered the roster"
21589 );
21590 assert!(child.report_ready.get().is_none());
21591 child.privacy_exec.as_mut().unwrap().deadline =
21594 tokio::time::Instant::now() + Duration::from_secs(30);
21595 exec.write_all(b"X").unwrap();
21596 child.confirm_privacy_exec().await;
21597 assert_eq!(child.spawn_failure, None);
21598 assert!(child.report_ready.get().is_some());
21599 let confirmed = crate::live_children::read_record(&record).unwrap();
21600 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21601 assert_ne!(module, trampoline);
21602 assert_eq!(confirmed[0].executable, Some(module.into()));
21603 child.start_kill().unwrap();
21604 child.wait().await.unwrap();
21605 child.release_roster();
21606 }
21607 }
21608
21609 #[tokio::test]
21610 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21611 async fn macos_unstamped_reexec_records_the_observed_native_image() {
21612 #[cfg(target_os = "macos")]
21613 {
21614 let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21615 let record = root.join("live-children.json");
21616 let supervisor =
21617 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21618 .with_live_children_record(&record);
21619 let runtime = supervisor.runtime_config();
21620 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21621 let fixture = crate::config_gate::program::native_fixture_for_test();
21622 let mut spec = spec(
21623 "/bin/sh",
21624 &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21625 );
21626 spec.env
21627 .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21628 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21629 let native_image = subc_os::file_identity(&fixture).unwrap();
21630 let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21633 while observe_spawned_image(child.pid).and_then(|image| image.executable)
21634 != Some(native_image)
21635 {
21636 tokio::time::sleep(Duration::from_millis(1)).await;
21637 }
21638 })
21639 .await;
21640 child.confirm_privacy_exec().await;
21641 let failure = child.spawn_failure.clone();
21642 let ready = child.report_ready.get().is_some();
21643 let confirmed = crate::live_children::read_record(&record).unwrap();
21644 child.start_kill().unwrap();
21645 tokio::time::timeout(Duration::from_secs(5), child.wait())
21646 .await
21647 .unwrap()
21648 .unwrap();
21649 child.release_roster();
21650
21651 reexec.expect("the shell must re-exec the native fixture");
21652 assert_eq!(
21653 failure, None,
21654 "an unstamped program may re-exec before observation"
21655 );
21656 assert!(ready);
21657 assert_eq!(confirmed.len(), 1);
21658 assert_eq!(confirmed[0].executable, Some(native_image.into()));
21659 }
21660 }
21661
21662 #[tokio::test]
21663 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21664 async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21665 #[cfg(target_os = "macos")]
21666 {
21667 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21668 let supervisor =
21669 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21670 let runtime = supervisor.runtime_config();
21671 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21672 let fixture = crate::config_gate::program::native_fixture_for_test();
21673 let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21674 spec.env.push((
21675 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21676 listener.local_addr().unwrap().to_string(),
21677 ));
21678 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21679 let mut exec = accept(listener).await;
21680 let mut ready = [0];
21681 exec.read_exact(&mut ready).unwrap();
21682 assert_eq!(&ready, b"R");
21683 assert_eq!(
21684 observe_spawned_image(child.pid).unwrap().executable,
21685 subc_os::file_identity(&test_privacy_trampoline())
21686 );
21687 let (reader, writer) = std::io::pipe().unwrap();
21690 drop(writer);
21691 child.privacy_exec.as_mut().unwrap().reader =
21692 tokio::io::unix::AsyncFd::new(reader).unwrap();
21693 child.confirm_privacy_exec().await;
21694 let failure = child.spawn_failure.clone();
21695 let ready = child.report_ready.get().is_some();
21696 child.start_kill().unwrap();
21697 tokio::time::timeout(Duration::from_secs(5), child.wait())
21698 .await
21699 .unwrap()
21700 .unwrap();
21701 child.release_roster();
21702 assert_eq!(
21703 failure.as_deref(),
21704 Some("privacy identity trampoline did not exec the module")
21705 );
21706 assert!(!ready);
21707 }
21708 }
21709
21710 #[tokio::test]
21711 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21712 async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21713 #[cfg(target_os = "macos")]
21714 {
21715 let registry = Arc::new(Registry::default());
21716 let policy = RestartPolicy::new(0, Duration::ZERO);
21717 let fixture = crate::config_gate::program::native_fixture_for_test();
21718 let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21719 let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21720 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy)
21721 .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21722 let runtime = supervisor.runtime_config();
21723 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21724 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21727 child.confirm_privacy_exec().await;
21728 assert_eq!(child.spawn_failure, None);
21729 assert!(child.report_ready.get().is_some());
21730 let pid = child.pid;
21731 tokio::task::spawn_blocking(move || {
21732 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21733 })
21734 .await
21735 .unwrap()
21736 .unwrap();
21737 let status = child.wait().await.unwrap();
21738 assert_eq!(status.code(), Some(121));
21739 assert!(child.privacy_exec.is_none());
21740 let report = classify_reaped_child_exit(&snapshot, &child, &status);
21741 on_child_exit(
21742 &spec,
21743 policy,
21744 ®istry,
21745 &snapshot,
21746 &runtime.terminal_ring,
21747 &runtime.spawn_events,
21748 &runtime.child_roster,
21749 report,
21750 true,
21751 )
21752 .await;
21753 let state = lock_snapshot(&snapshot).unwrap();
21754 assert_eq!(state.state, ModuleState::Failed);
21755 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21756 assert_eq!(state.reported_pid(), None);
21757 drop(state);
21758 let history = runtime.terminal_ring.lock().unwrap().snapshot();
21759 assert_eq!(history.entries.len(), 1);
21760 let terminal = &history.entries[0];
21761 assert_eq!(terminal.exit_code, Some(121));
21762 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21763 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21764 assert_eq!(
21765 terminal.disposition_detail.as_deref(),
21766 Some(policy.budget_exhausted_detail().as_str()),
21767 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21768 );
21769 assert_eq!(child.spawn_failure, None);
21770 child.release_roster();
21771 }
21772 }
21773}
21774
21775#[cfg(all(test, target_os = "macos"))]
21776#[path = "supervise/suspended_spawn_tests.rs"]
21777mod suspended_spawn_tests;
21778
21779#[cfg(all(test, unix))]
21783mod launch_nonce_descriptor_tests {
21784 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21785 use crate::stderr_tail::{StderrRing, StderrTailConfig};
21786 use cortexkit_test_support::ScratchDir;
21787 use std::{
21788 path::PathBuf,
21789 sync::{Arc, Mutex},
21790 time::{Duration, Instant},
21791 };
21792
21793 async fn probe(role: super::SpawnRole) {
21794 let scratch = ScratchDir::new("launch-nonce-descriptor");
21795 let fd_copy = scratch.join("from-descriptor");
21796 let env_copy = scratch.join("environment");
21797 let script = format!(
21798 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21799 fd = fd_copy.display(), env = env_copy.display(),
21800 );
21801 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21802 let spec = ModuleSpec {
21803 module_id: "nonce-descriptor-probe".to_string(),
21804 program: PathBuf::from("/bin/sh"),
21805 args: vec!["-c".to_string(), script],
21806 env: vec![
21807 xdg("XDG_DATA_HOME"),
21808 xdg("XDG_RUNTIME_DIR"),
21809 xdg("XDG_CONFIG_HOME"),
21810 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21811 ],
21812 reserved: true,
21813 reserved_prefixes: Vec::new(),
21814 protocol: ModuleProtocol::Subc,
21815 overlap: Default::default(),
21816 carry_containment: false,
21817 };
21818 let handle = SupervisorHandle::new();
21819 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21820 let roster = ChildRoster::default();
21821 #[cfg(target_os = "macos")]
21822 {
21823 let path = super::test_privacy_trampoline();
21824 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21825 }
21826 let child = super::spawn_child_in_slot(
21827 &spec,
21828 None,
21829 Some(&handle),
21830 &ring,
21831 None,
21832 &roster,
21833 None,
21834 #[cfg(target_os = "linux")]
21835 None,
21836 role,
21837 matches!(role, super::SpawnRole::SwapCandidate),
21838 super::DEFAULT_REGISTER_TIMEOUT,
21839 )
21840 .expect("spawn probe");
21841 let deadline = Instant::now() + Duration::from_secs(10);
21842 while !(fd_copy.exists() && env_copy.exists()) {
21843 assert!(Instant::now() < deadline, "probe never wrote its copies");
21844 tokio::time::sleep(Duration::from_millis(20)).await;
21845 }
21846 let nonce = std::fs::read_to_string(fd_copy).unwrap();
21847 assert!(!nonce.is_empty());
21848 let environment = std::fs::read_to_string(env_copy).unwrap();
21849 assert!(environment
21850 .lines()
21851 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
21852 let copy = environment
21853 .lines()
21854 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
21855 assert_eq!(
21856 copy, None,
21857 "Unix children must never receive the environment nonce"
21858 );
21859 if matches!(role, super::SpawnRole::Plain) {
21860 assert_eq!(
21861 handle.spawn_nonce(&spec.module_id).as_deref(),
21862 Some(nonce.as_str())
21863 );
21864 }
21865 drop(child);
21866 }
21867
21868 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21869 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
21870 probe(super::SpawnRole::Plain).await;
21871 }
21872
21873 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21874 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
21875 probe(super::SpawnRole::SwapCandidate).await;
21876 }
21877}
21878
21879#[cfg(all(test, target_os = "linux"))]
21880#[path = "supervise/cgroup_reap_tests.rs"]
21881mod cgroup_reap_tests;
21882
21883#[cfg(all(test, target_os = "linux"))]
21884mod cgroup_containment_tests {
21885 use super::*;
21886 use cortexkit_test_support::ScratchDir;
21887
21888 #[tokio::test]
21889 async fn linux_subtree_memory_covers_module_children() {
21890 let dir = ScratchDir::new("subtree-memory");
21891 let root = PathBuf::from(format!(
21892 "/sys/fs/cgroup/subc-memory-test-{}-{}",
21893 std::process::id(),
21894 unix_ms_now()
21895 ));
21896 if let Err(error) = std::fs::create_dir(&root) {
21897 assert!(
21898 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21899 "required cgroup test cannot execute: {error}"
21900 );
21901 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
21902 return;
21903 }
21904 let placement = subc_cgroup::prepare_at(&root)
21905 .unwrap()
21906 .expect("delegated cgroup");
21907 let supervisor = Supervisor::new_for_test(
21908 Arc::new(Registry::default()),
21909 RestartPolicy::new(3, Duration::ZERO),
21910 )
21911 .with_cgroup_placement(Some(placement));
21912 let ready = dir.join("ready");
21913 let module = supervisor
21914 .spawn(ModuleSpec {
21915 module_id: "memory-tree".into(),
21916 program: PathBuf::from("/bin/sh"),
21917 args: vec![
21918 "-c".into(),
21919 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
21920 "fixture".into(),
21921 ready.display().to_string(),
21922 ],
21923 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
21924 .into_iter()
21925 .map(|key| (key.into(), dir.display().to_string()))
21926 .collect(),
21927 reserved: false,
21928 reserved_prefixes: Vec::new(),
21929 protocol: ModuleProtocol::None,
21930 overlap: Default::default(),
21931 carry_containment: false,
21932 })
21933 .unwrap();
21934 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
21935 while !ready.exists() {
21936 assert!(
21937 tokio::time::Instant::now() < deadline,
21938 "children never started"
21939 );
21940 tokio::time::sleep(Duration::from_millis(10)).await;
21941 }
21942 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
21943 let reading = &readings[0];
21944 assert!(
21945 reading.process_count.is_some_and(|count| count >= 4),
21946 "{reading:?}"
21947 );
21948 assert!(
21949 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
21950 "{reading:?}"
21951 );
21952 module.stop().await.unwrap();
21953 drop(module);
21954 std::fs::remove_dir(root.join("subc-modules")).unwrap();
21955 std::fs::remove_dir(&root).unwrap();
21956 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
21957 }
21958
21959 fn running(pid: u32) -> bool {
21960 std::fs::read_to_string(format!("/proc/{pid}/stat"))
21962 .ok()
21963 .and_then(|stat| {
21964 stat.rsplit_once(") ")
21965 .map(|(_, rest)| rest.starts_with('Z'))
21966 })
21967 .is_some_and(|zombie| !zombie)
21968 }
21969
21970 #[tokio::test]
21971 async fn linux_teardown_reaps_the_grandchild() {
21972 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
21973 }
21974
21975 #[tokio::test]
21976 async fn linux_shutdown_straggler_reaps_the_grandchild() {
21977 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
21978 }
21979
21980 async fn teardown_tree(test_name: &str, shutdown: bool) {
21981 let dir = ScratchDir::new(test_name);
21982 let root = PathBuf::from(format!(
21983 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
21984 std::process::id(),
21985 unix_ms_now()
21986 ));
21987 if let Err(error) = std::fs::create_dir(&root) {
21988 assert!(
21989 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21990 "required cgroup test cannot execute: {error}"
21991 );
21992 eprintln!(
21993 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
21994 root.display()
21995 );
21996 return;
21997 }
21998 let placement = subc_cgroup::prepare_at(&root)
21999 .expect("prepare isolated kernel cgroup")
22000 .expect("isolated cgroup is delegated");
22001 let module_id = "tree-teardown";
22002 let module = placement
22003 .module_path(module_id)
22004 .expect("create isolated module cgroup");
22005 if !module.join("cgroup.kill").exists() {
22006 std::fs::remove_dir(&module).unwrap();
22007 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22008 std::fs::remove_dir(&root).unwrap();
22009 assert!(
22010 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22011 "required cgroup.kill interface unavailable"
22012 );
22013 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
22014 return;
22015 }
22016 let supervisor = Supervisor::new_for_test(
22017 Arc::new(Registry::default()),
22018 RestartPolicy::new(3, Duration::ZERO),
22019 )
22020 .with_cgroup_placement(Some(placement));
22021 let mut runtime = supervisor.runtime_config();
22022 runtime.child_roster = runtime
22023 .child_roster
22024 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
22025 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
22026 let pid_file = dir.join("grandchild.pid");
22027 let spec = ModuleSpec {
22028 module_id: module_id.to_string(),
22029 program: PathBuf::from("/bin/sh"),
22030 args: vec![
22031 "-c".into(),
22032 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
22033 "fixture".into(),
22034 pid_file.display().to_string(),
22035 ],
22036 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22037 .into_iter()
22038 .map(|key| (key.to_string(), dir.display().to_string()))
22039 .collect(),
22040 reserved: false,
22041 reserved_prefixes: Vec::new(),
22042 protocol: ModuleProtocol::None,
22043 overlap: Default::default(),
22044 carry_containment: false,
22045 };
22046 let child =
22047 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
22048 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22049 let grandchild: u32 = loop {
22050 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
22051 if let Ok(pid) = contents.trim().parse() {
22052 break pid;
22053 }
22054 }
22055 assert!(
22056 tokio::time::Instant::now() < deadline,
22057 "grandchild pid was not recorded"
22058 );
22059 tokio::time::sleep(Duration::from_millis(10)).await;
22060 };
22061 assert!(
22062 running(grandchild),
22063 "grandchild must be alive before teardown"
22064 );
22065 if shutdown {
22066 let mut child = child;
22067 crate::child_roster::end_children_for_daemon_shutdown(
22068 &runtime.child_roster,
22069 false,
22070 std::future::pending(),
22071 )
22072 .await;
22073 child.wait().await.expect("reap shutdown straggler");
22074 } else {
22075 drain_child_to_state(
22076 module_id,
22077 ModuleProtocol::None,
22078 StopNotice::NotSent,
22079 &Registry::default(),
22080 None,
22081 &snapshot,
22082 &runtime.terminal_ring,
22083 &SpawnEventFeed::default(),
22084 child,
22085 Duration::from_millis(100),
22086 ModuleState::Stopped,
22087 Some(false),
22088 &FailureContext::operator(),
22089 )
22090 .await
22091 .expect("real supervisor teardown");
22092 }
22093 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
22094 while running(grandchild) && tokio::time::Instant::now() < deadline {
22095 tokio::time::sleep(Duration::from_millis(10)).await;
22096 }
22097 let survived = running(grandchild);
22098 if survived {
22100 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
22101 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
22102 tokio::time::sleep(Duration::from_millis(100)).await;
22103 }
22104 if module.exists() {
22105 std::fs::remove_dir(&module).expect("remove empty module cgroup");
22106 }
22107 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22108 std::fs::remove_dir(&root).unwrap();
22109 assert!(
22110 !survived,
22111 "grandchild {grandchild} outlived module teardown"
22112 );
22113 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
22114 }
22115}
22116
22117#[cfg(all(test, unix))]
22120mod record_write_stall_tests {
22121 use super::*;
22122 use std::sync::atomic::{AtomicBool, Ordering};
22123 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22124
22125 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22128 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22132
22133 fn sleeping_spec() -> ModuleSpec {
22145 ModuleSpec {
22146 module_id: "record-write-stall".into(),
22147 program: "/bin/sleep".into(),
22148 args: vec!["600".into()],
22149 env: vec![],
22150 reserved: false,
22151 reserved_prefixes: vec![],
22152 protocol: ModuleProtocol::None,
22153 overlap: Default::default(),
22154 carry_containment: false,
22155 }
22156 }
22157
22158 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22159 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22160 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22161 let supervisor =
22162 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22163 .with_live_children_record(home.join("live-children.json"));
22164 let module = supervisor.spawn(sleeping_spec()).unwrap();
22165 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22170 loop {
22171 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22172 .unwrap()
22173 .reported_pid()
22174 {
22175 return pid;
22176 }
22177 tokio::time::sleep(Duration::from_millis(10)).await;
22178 }
22179 })
22180 .await
22181 .expect("the first process was admitted as the module");
22182
22183 let key = vec![0x42; 32];
22186 let daemon_id = [0x24; 16];
22187 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22188 let port = listener.local_addr().unwrap().port();
22189 let server = tokio::spawn(crate::server::serve_listener(
22190 listener,
22191 Arc::new(crate::router::Router::with_default_self_handler()),
22192 crate::server::ServerAuth::with_limits(
22193 key.clone(),
22194 daemon_id,
22195 "test",
22196 crate::server::DEFAULT_AUTH_DEADLINE,
22197 4,
22198 ),
22199 ));
22200
22201 let (held_tx, held_rx) = std::sync::mpsc::channel();
22204 let armed = AtomicBool::new(true);
22205 supervisor
22206 .child_roster
22207 .set_record_write_hook_for_test(move |live| {
22208 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22209 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22210 let _ = held_tx.send(());
22211 std::thread::sleep(HELD_RECORD_WRITE);
22212 }
22213 });
22214
22215 let module_process = std::thread::spawn(move || {
22218 held_rx
22219 .recv_timeout(Duration::from_secs(30))
22220 .expect("the restart reached the replacement's record write");
22221 let conn = ConnectionInfo {
22222 schema: SCHEMA_VERSION,
22223 wire_version: None,
22224 endpoints: vec![Endpoint {
22225 host: "127.0.0.1".to_owned(),
22226 port,
22227 }],
22228 key,
22229 daemon_id,
22230 pid: std::process::id(),
22231 daemon_ver: "test".to_owned(),
22232 };
22233 tokio::runtime::Builder::new_current_thread()
22234 .enable_all()
22235 .build()
22236 .unwrap()
22237 .block_on(async {
22238 let started = Instant::now();
22239 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22240 .await
22241 .unwrap();
22242 let result =
22243 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22244 (result, started.elapsed())
22245 })
22246 });
22247
22248 module.restart(Some(0)).await.unwrap();
22249 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22250 .await
22251 .unwrap();
22252 assert!(
22253 auth.is_ok(),
22254 "the daemon must answer a module's handshake while a restart waits on the \
22255 live-children record write; got {auth:?} after {elapsed:?}"
22256 );
22257
22258 tokio::time::timeout(Duration::from_secs(20), async {
22259 while lock_snapshot(&module.inner.snapshot)
22260 .unwrap()
22261 .reported_pid()
22262 .is_none_or(|pid| pid == old_pid)
22263 {
22264 tokio::time::sleep(Duration::from_millis(10)).await;
22265 }
22266 })
22267 .await
22268 .expect("the restart published its replacement");
22269 module.stop().await.unwrap();
22270 server.abort();
22271 }
22272}
22273
22274#[cfg(all(test, unix))]
22275mod retained_status_tests {
22276 use super::*;
22277
22278 #[tokio::test]
22279 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22280 use crate::live_children::{ContainerIdentity, RetainedReason};
22281 for (reason, carried, expected) in [
22282 (
22283 RetainedReason::ContainerSurvivors(vec![4242]),
22284 true,
22285 Some("waiting: container survivors (pids [4242])".to_owned()),
22286 ),
22287 (
22288 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22289 true,
22290 Some(
22291 "waiting: container survivors could not be verified: member inspection denied"
22292 .to_owned(),
22293 ),
22294 ),
22295 (RetainedReason::Survived, false, None),
22296 ] {
22297 let supervisor =
22298 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22299 let spec = ModuleSpec {
22300 module_id: "retained-status-fixture".into(),
22301 program: PathBuf::from("/not-spawned"),
22302 args: Vec::new(),
22303 env: Vec::new(),
22304 reserved: false,
22305 reserved_prefixes: Vec::new(),
22306 protocol: ModuleProtocol::None,
22307 overlap: ModuleOverlap::Exclusive,
22308 carry_containment: carried,
22309 };
22310 let module = supervisor
22311 .supervise_configured(spec.clone(), false)
22312 .unwrap();
22313 let guard = supervisor
22317 .child_roster
22318 .admit(
22319 spec.module_id,
22320 0,
22321 ModuleProtocol::None,
22322 None,
22323 crate::child_roster::RecordedIdentity {
22324 container: carried.then_some(ContainerIdentity {
22325 pgid: Some(0),
22326 created_start_time: Some(0),
22327 cgroup_path: None,
22328 ..ContainerIdentity::test_metadata()
22329 }),
22330 ..crate::child_roster::RecordedIdentity::default()
22331 },
22332 )
22333 .unwrap();
22334 guard.retain(reason);
22335 let status = module.status().unwrap();
22336 assert_eq!(status.waiting_for_previous_detail, expected.clone());
22337 assert_eq!(status.health.detail, expected);
22338 assert!(!status.process_alive);
22339 }
22340 }
22341}
22342
22343fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22344 child.try_wait().is_ok_and(|status| status.is_none())
22345}