1use std::{
2 collections::BTreeMap,
3 env,
4 error::Error,
5 ffi::OsString,
6 fmt, fs, io,
7 net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8 path::{Path, PathBuf},
9 process,
10 time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17 authenticate_client, connection_file, generate_daemon_id, generate_key,
18 write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
19 LocalEndpoint, SCHEMA_VERSION,
20};
21use tokio::{
22 net::{TcpListener, TcpStream},
23 task::{JoinError, JoinHandle},
24 time::{sleep, timeout},
25};
26use tracing::{error, info, warn};
27
28use crate::{
29 daemon_config::{self, ConfiguredModule, DaemonConfigError},
30 local_listener::{self, BoundLocalListener, LocalListenerConfig},
31 server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
32 supervise::HealthConfig,
33 ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
34 ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
35 SupervisorProcessLiveness,
36};
37use std::sync::Arc;
38
39#[cfg(windows)]
40mod windows_connection_copy;
41
42pub const DEFAULT_SUBC_PORT: u16 = 8757;
43pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
44use subc_transport::CONNECTION_FILE_NAME;
45const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
46const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
47const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
48const START_LOCK_RETRIES: usize = 40;
49const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum ConnectionFileSource {
53 XdgRuntimeDir,
54 TempDirFallback,
55 Explicit,
56}
57
58impl ConnectionFileSource {
59 fn reason(self) -> &'static str {
60 match self {
61 Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
62 Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
63 Self::Explicit => "configured path",
64 }
65 }
66}
67
68impl fmt::Display for ConnectionFileSource {
69 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
70 formatter.write_str(match self {
71 Self::XdgRuntimeDir => "xdg_runtime_dir",
72 Self::TempDirFallback => "temp_dir_fallback",
73 Self::Explicit => "explicit",
74 })
75 }
76}
77
78#[derive(Debug, Clone, Default)]
82struct AdmissionFactsConfig {
83 carrier_module_id: Option<String>,
84 targets: Option<Vec<String>>,
85}
86
87#[derive(Debug, Clone, Default, PartialEq, Eq)]
96pub enum CgroupPlacementConfig {
97 #[default]
98 Disabled,
99 Current,
100 Root(PathBuf),
101}
102
103#[derive(Debug, Clone)]
104pub struct BootstrapConfig {
105 #[cfg(feature = "test-support")]
106 shutdown_intent_window: Duration,
107 pub operator_queue_wait: Duration,
108 pub operator_timeout: Duration,
109 pub operator_stuck_grace: Duration,
110 #[cfg(feature = "test-support")]
111 pub operator_script: Option<PathBuf>,
112 #[cfg(feature = "test-support")]
113 pub operator_events: Option<PathBuf>,
114 pub connection_file_path: PathBuf,
115 pub port: u16,
116 pub daemon_ver: String,
117 configured_modules: Vec<ConfiguredModule>,
118 storage_config: Option<daemon_config::StorageConfig>,
119 admission_facts: AdmissionFactsConfig,
120 scope_authority_owners: Vec<String>,
122 federated_scope_owner: String,
123 privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
124 activity_config: daemon_config::ActivityConfig,
125 daemon_config_path: Option<PathBuf>,
126 config_store_root: Option<PathBuf>,
129 defer_file_load: bool,
130 #[cfg(test)]
133 file_derived_test_probe: Option<Vec<String>>,
134 #[cfg(all(test, feature = "test-support"))]
135 test_store_factory:
136 Option<fn(crate::config_store::StoreRoot) -> crate::config_store::ConfigStore>,
137 #[cfg(all(test, feature = "test-support"))]
138 test_control: Option<fn(ControlHandler) -> ControlHandler>,
139 #[cfg(all(test, feature = "test-support"))]
140 test_listener_bind: Option<fn(&mut BootstrapConfig)>,
141 configured_port: Option<u16>,
142 route_bind_relay_default_ms: Option<u64>,
146 reserved_capabilities: BTreeMap<String, String>,
147 watchdog_config: DaemonSelfWatchdogConfig,
148 connection_file_source: ConnectionFileSource,
149 #[cfg(all(test, windows))]
150 profile_override: Option<PathBuf>,
151 cgroup_placement: CgroupPlacementConfig,
154 capture_logs_dir: Option<PathBuf>,
158 terminal_journal_path: Option<PathBuf>,
166 machine_id_path: Option<PathBuf>,
172
173 local_listener: LocalListenerConfig,
180 live_children_path: Option<PathBuf>,
188 privacy_trampoline: Option<PathBuf>,
189}
190
191impl BootstrapConfig {
192 pub fn with_config_store_root(mut self, root: PathBuf) -> Result<Self, BootstrapError> {
193 crate::config_store::StoreRoot::from_service_definition(root.clone())
194 .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?;
195 let parent = root.parent().ok_or_else(|| {
196 BootstrapError::ConfigStore("config store root needs a parent directory".into())
197 })?;
198 std::fs::read_dir(parent).map_err(|e| {
199 BootstrapError::ConfigStore(format!("config store parent is unreadable: {e}"))
200 })?;
201 self.config_store_root = Some(root);
202 Ok(self)
203 }
204 #[cfg(feature = "test-support")]
206 pub fn with_shutdown_intent_window(mut self, window: Duration) -> Self {
207 self.shutdown_intent_window = window;
208 self
209 }
210
211 pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
212 Self {
213 #[cfg(feature = "test-support")]
214 shutdown_intent_window: Duration::from_secs(120),
215 connection_file_path: connection_file_path.into(),
216 operator_queue_wait: Duration::from_secs(150),
217 operator_timeout: Duration::from_secs(120),
218 operator_stuck_grace: Duration::from_secs(10),
219 #[cfg(feature = "test-support")]
220 operator_script: None,
221 #[cfg(feature = "test-support")]
222 operator_events: None,
223 port,
224 daemon_ver: DAEMON_VERSION.to_owned(),
225 configured_modules: Vec::new(),
226 storage_config: None,
227 admission_facts: AdmissionFactsConfig::default(),
228 scope_authority_owners: daemon_config::default_scope_authority_owners(),
229 federated_scope_owner: daemon_config::default_federated_scope_owner(),
230 privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
231 activity_config: daemon_config::ActivityConfig::default(),
232 daemon_config_path: None,
233 config_store_root: None,
234 defer_file_load: false,
235 #[cfg(test)]
236 file_derived_test_probe: None,
237 #[cfg(all(test, feature = "test-support"))]
238 test_store_factory: None,
239 #[cfg(all(test, feature = "test-support"))]
240 test_control: None,
241 #[cfg(all(test, feature = "test-support"))]
242 test_listener_bind: None,
243 configured_port: None,
244 route_bind_relay_default_ms: None,
245 reserved_capabilities: BTreeMap::new(),
246 watchdog_config: DaemonSelfWatchdogConfig::default(),
247 connection_file_source: ConnectionFileSource::Explicit,
248 #[cfg(all(test, windows))]
249 profile_override: None,
250 cgroup_placement: CgroupPlacementConfig::default(),
251 capture_logs_dir: None,
252 terminal_journal_path: None,
253 machine_id_path: None,
254 local_listener: LocalListenerConfig { socket_dir: None },
255 live_children_path: None,
256 privacy_trampoline: None,
257 }
258 }
259
260 pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
263 self.local_listener = LocalListenerConfig::in_socket_dir(dir);
264 self
265 }
266
267 pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
269 self.local_listener = config;
270 self
271 }
272
273 pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
277 self.live_children_path = Some(path.into());
278 self
279 }
280
281 pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
284 self.privacy_trampoline = Some(path.into());
285 self
286 }
287
288 pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
292 self.machine_id_path = Some(path.into());
293 self
294 }
295
296 pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
298 self.cgroup_placement = placement;
299 self
300 }
301
302 pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
306 self.capture_logs_dir = Some(dir.into());
307 self
308 }
309
310 pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
313 self.terminal_journal_path = Some(path.into());
314 self
315 }
316
317 pub fn from_env() -> Result<Self, BootstrapError> {
318 Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
319 }
320
321 pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
330 let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
331 let machine_id_path =
332 crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
333 let (connection_file, source) =
336 connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
337 let mut config = Self::new(connection_file, DEFAULT_SUBC_PORT)
338 .with_connection_file_source(source)
339 .with_daemon_config_source(daemon_config::default_config_path(), None)
340 .with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
341 .with_capture_logs_dir(run_dir.join("logs"))
342 .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
343 .with_live_children_record(crate::live_children::record_path(&run_dir))
344 .with_machine_id_path(machine_id_path);
345 config.defer_file_load = true;
346 #[cfg(feature = "test-support")]
347 let config = {
348 let mut config = config;
349 config.operator_script =
350 std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
351 config.operator_events =
352 std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
353 for (name, value) in [
354 (
355 "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
356 &mut config.operator_queue_wait,
357 ),
358 (
359 "SUBC_TEST_OPERATOR_TIMEOUT_MS",
360 &mut config.operator_timeout,
361 ),
362 (
363 "SUBC_TEST_OPERATOR_STUCK_MS",
364 &mut config.operator_stuck_grace,
365 ),
366 ] {
367 if let Ok(ms) = std::env::var(name) {
368 if let Ok(ms) = ms.parse::<u64>() {
369 *value = Duration::from_millis(ms);
370 }
371 }
372 }
373 config
374 };
375 Ok(config)
376 }
377
378 pub fn from_env_with_daemon_config_path(
383 daemon_config_path: impl AsRef<Path>,
384 ) -> Result<Self, BootstrapError> {
385 let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
386 let (daemon_config, activity_config) =
387 daemon_config::load_with_activity(&daemon_config_path)
388 .map_err(BootstrapError::DaemonConfig)?
389 .map(|(config, activity)| (Some(config), activity))
390 .unwrap_or_default();
391 let config_port = daemon_config.as_ref().and_then(|config| config.port);
392 let storage_config = daemon_config
393 .as_ref()
394 .and_then(|config| config.storage.clone());
395 let admission_facts_carrier_module_id = daemon_config
396 .as_ref()
397 .and_then(|config| config.admission_facts_carrier_module_id.clone());
398 let admission_facts_targets = daemon_config
399 .as_ref()
400 .and_then(|config| config.admission_facts_targets.clone());
401 let scope_authority_owners = daemon_config
402 .as_ref()
403 .map(|config| config.scope_authority_owners.clone())
404 .unwrap_or_else(daemon_config::default_scope_authority_owners);
405 let federated_scope_owner = daemon_config
406 .as_ref()
407 .map(|config| config.federated_scope_owner.clone())
408 .unwrap_or_else(daemon_config::default_federated_scope_owner);
409 let privileged_operator_prompt = daemon_config
410 .as_ref()
411 .map(|config| config.privileged_operator_prompt)
412 .unwrap_or_default();
413 let route_bind_relay_default_ms = daemon_config
414 .as_ref()
415 .and_then(|config| config.route_bind_relay_timeout_ms);
416 let reserved_capabilities = daemon_config
417 .as_ref()
418 .map(|config| config.reserved_capabilities.clone())
419 .unwrap_or_default();
420 let configured_modules = daemon_config
421 .map(|config| config.modules)
422 .unwrap_or_default();
423
424 let port = match env::var(SUBC_PORT_ENV) {
425 Ok(raw) if !raw.trim().is_empty() => {
426 let port = raw
427 .parse::<u16>()
428 .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
429 if let Some(config_port) = config_port {
430 info!(
431 env = SUBC_PORT_ENV,
432 env_port = port,
433 config_port,
434 "SUBC_PORT overrides daemon config port"
435 );
436 }
437 port
438 }
439 Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
440 };
441
442 let (connection_file_path, connection_file_source) =
443 connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
444 let config = Self::new(connection_file_path, port)
445 .with_configured_modules(configured_modules)
446 .with_storage_config(storage_config)
447 .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
448 .with_scope_authority_owners(scope_authority_owners)
449 .with_federated_scope_owner(federated_scope_owner)
450 .with_privileged_operator_prompt(privileged_operator_prompt)
451 .with_activity_config(activity_config)
452 .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
453 .with_reserved_capabilities(reserved_capabilities)
454 .with_daemon_config_source(daemon_config_path, config_port)
455 .with_connection_file_source(connection_file_source);
456 #[cfg(test)]
457 let config = {
458 let mut config = config;
459 config.file_derived_test_probe = Some(config.scope_authority_owners.clone());
460 config
461 };
462 Ok(config)
463 }
464
465 pub fn with_daemon_config_path(
470 mut self,
471 daemon_config_path: impl AsRef<Path>,
472 ) -> Result<Self, BootstrapError> {
473 let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
474 if self.config_store_root.is_some() {
475 self.defer_file_load = true;
476 return Ok(self.with_daemon_config_source(daemon_config_path, None));
477 }
478 let (daemon_config, activity_config) =
479 daemon_config::load_with_activity(&daemon_config_path)
480 .map_err(BootstrapError::DaemonConfig)?
481 .map(|(config, activity)| (Some(config), activity))
482 .unwrap_or_default();
483 let configured_port = daemon_config.as_ref().and_then(|config| config.port);
484 Ok(self
485 .with_parsed_daemon_config(daemon_config, activity_config)
486 .with_daemon_config_source(daemon_config_path, configured_port))
487 }
488
489 fn with_parsed_daemon_config(
490 self,
491 daemon_config: Option<daemon_config::DaemonConfig>,
492 activity_config: daemon_config::ActivityConfig,
493 ) -> Self {
494 let storage_config = daemon_config
495 .as_ref()
496 .and_then(|config| config.storage.clone());
497 let admission_facts_carrier_module_id = daemon_config
498 .as_ref()
499 .and_then(|config| config.admission_facts_carrier_module_id.clone());
500 let admission_facts_targets = daemon_config
501 .as_ref()
502 .and_then(|config| config.admission_facts_targets.clone());
503 let scope_authority_owners = daemon_config
504 .as_ref()
505 .map(|config| config.scope_authority_owners.clone())
506 .unwrap_or_else(daemon_config::default_scope_authority_owners);
507 let federated_scope_owner = daemon_config
508 .as_ref()
509 .map(|config| config.federated_scope_owner.clone())
510 .unwrap_or_else(daemon_config::default_federated_scope_owner);
511 let privileged_operator_prompt = daemon_config
512 .as_ref()
513 .map(|config| config.privileged_operator_prompt)
514 .unwrap_or_default();
515 let route_bind_relay_default_ms = daemon_config
516 .as_ref()
517 .and_then(|config| config.route_bind_relay_timeout_ms);
518 let reserved_capabilities = daemon_config
519 .as_ref()
520 .map(|config| config.reserved_capabilities.clone())
521 .unwrap_or_default();
522 let configured_modules = daemon_config
523 .map(|config| config.modules)
524 .unwrap_or_default();
525 self.with_configured_modules(configured_modules)
526 .with_storage_config(storage_config)
527 .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
528 .with_scope_authority_owners(scope_authority_owners)
529 .with_federated_scope_owner(federated_scope_owner)
530 .with_privileged_operator_prompt(privileged_operator_prompt)
531 .with_activity_config(activity_config)
532 .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
533 .with_reserved_capabilities(reserved_capabilities)
534 }
535
536 pub fn with_configured_modules(
537 mut self,
538 modules: impl IntoIterator<Item = ConfiguredModule>,
539 ) -> Self {
540 self.configured_modules = modules.into_iter().collect();
541 self.configured_modules
542 .sort_by(|left, right| left.module_id.cmp(&right.module_id));
543 self
544 }
545
546 pub fn with_storage_config(
547 mut self,
548 storage_config: Option<daemon_config::StorageConfig>,
549 ) -> Self {
550 self.storage_config = storage_config;
551 self
552 }
553
554 pub fn with_admission_facts_config(
555 mut self,
556 carrier_module_id: Option<String>,
557 targets: Option<Vec<String>>,
558 ) -> Self {
559 self.admission_facts = AdmissionFactsConfig {
560 carrier_module_id,
561 targets,
562 };
563 self
564 }
565
566 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
569 self.scope_authority_owners = owners;
570 self
571 }
572
573 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
574 self.federated_scope_owner = owner;
575 self
576 }
577
578 pub fn with_privileged_operator_prompt(
580 mut self,
581 mode: daemon_config::PrivilegedOperatorPrompt,
582 ) -> Self {
583 self.privileged_operator_prompt = mode;
584 self
585 }
586
587 pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
588 self.activity_config = config;
589 self
590 }
591
592 pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
597 self.route_bind_relay_default_ms = ms;
598 self
599 }
600
601 pub fn with_reserved_capabilities(
602 mut self,
603 reserved_capabilities: BTreeMap<String, String>,
604 ) -> Self {
605 self.reserved_capabilities = reserved_capabilities;
606 self
607 }
608
609 fn with_daemon_config_source(
610 mut self,
611 daemon_config_path: PathBuf,
612 configured_port: Option<u16>,
613 ) -> Self {
614 self.daemon_config_path = Some(daemon_config_path);
615 self.configured_port = configured_port;
616 self
617 }
618
619 fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
620 self.connection_file_source = source;
621 self
622 }
623
624 pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
625 self.watchdog_config = watchdog_config;
626 self
627 }
628}
629
630#[allow(clippy::large_enum_variant)]
634#[derive(Debug)]
635pub enum Outcome {
636 AlreadyRunning,
638 Bound(BoundDaemon),
641}
642
643#[derive(Debug)]
644pub struct BoundDaemon {
645 pub listeners: Vec<TcpListener>,
646 pub local_listener: Option<BoundLocalListener>,
650 pub local_endpoint_unavailable: Option<String>,
653 pub local_endpoint: Option<LocalEndpoint>,
656 pub connection_info: ConnectionInfo,
657 pub connection_file_path: PathBuf,
658 pub connection_file_source: ConnectionFileSource,
659 #[cfg(windows)]
660 connection_file_copy: Option<PathBuf>,
661 pub machine_id: Option<crate::machine_id::MachineId>,
664 run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
668 publication_lock: Option<StartLock>,
674}
675
676pub fn connection_file_path() -> PathBuf {
683 connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
684}
685
686fn connection_file_path_with_source(
687 runtime_dir: Option<OsString>,
688) -> (PathBuf, ConnectionFileSource) {
689 if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
690 return (
691 PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
692 ConnectionFileSource::XdgRuntimeDir,
693 );
694 }
695
696 (
697 env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
698 ConnectionFileSource::TempDirFallback,
699 )
700}
701
702pub async fn run() -> Result<(), BootstrapError> {
708 run_with_config(
711 BootstrapConfig::from_env_for_daemon_binary()?
712 .with_cgroup_placement(CgroupPlacementConfig::Current),
713 )
714 .await
715}
716
717#[cfg(feature = "test-support")]
718mod config_test_seam {
719 #[cfg(not(test))]
720 use super::BootstrapError;
721
722 pub(super) fn enabled() -> bool {
723 std::env::var_os("SUBC_TEST_CONFIG_GATE").as_deref() == Some(std::ffi::OsStr::new("1"))
724 && std::env::current_exe().ok().is_some_and(|path| {
725 path.file_name()
726 .is_some_and(|name| name.to_string_lossy().starts_with("ckdev-"))
727 })
728 }
729
730 #[cfg(not(test))]
731 pub(super) fn open(
732 root: crate::config_store::StoreRoot,
733 ) -> Result<crate::config_store::ConfigStore, BootstrapError> {
734 if !enabled() {
735 return Ok(crate::config_store::ConfigStore::open(root));
736 }
737 crate::config_store::testing::TestSupport::new(
740 root.path().to_path_buf(),
741 Some(root.path().join("test-key")),
742 )
743 .and_then(|hooks| {
744 hooks
745 .with_anchor_failure_file(root.path().join("fail-anchor"))
746 .open()
747 })
748 .map_err(|error| BootstrapError::ConfigStore(error.to_string()))
749 }
750}
751
752pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
776 let Some(start_lock) = singleton_start_lock(&config.connection_file_path).await? else {
777 info!("subc daemon already running");
778 return Ok(());
779 };
780 let (config, opened_store, start_lock) = tokio::task::spawn_blocking(move || {
781 let mut opened_store = config
782 .config_store_root
783 .as_ref()
784 .map(|root| {
785 let (root, lock) = crate::config_store::StoreWriterLock::acquire(root)
786 .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?;
787 #[cfg(all(test, feature = "test-support"))]
788 let store = if let Some(factory) = config.test_store_factory {
789 crate::config_store::testing::TestSupport::new(root.path().to_path_buf(), None)
790 .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?;
791 factory(root)
792 } else {
793 crate::config_store::ConfigStore::open(root)
794 };
795 #[cfg(all(feature = "test-support", not(test)))]
796 let store = config_test_seam::open(root)?;
797 #[cfg(not(feature = "test-support"))]
798 let store = crate::config_store::ConfigStore::open(root);
799 Ok::<_, BootstrapError>((store, lock))
800 })
801 .transpose()?;
802 let config =
803 load_bootstrap_authority(config, opened_store.as_mut().map(|(store, _)| store))?;
804 #[cfg(all(test, feature = "test-support"))]
805 let config = {
806 let mut config = config;
807 if let Some(bind) = config.test_listener_bind {
808 bind(&mut config);
811 }
812 config
813 };
814 Ok::<_, BootstrapError>((config, opened_store, start_lock))
815 })
816 .await
817 .map_err(|e| BootstrapError::ConfigStore(e.to_string()))??;
818 #[cfg(feature = "test-support")]
819 let shutdown_intent_window = config.shutdown_intent_window;
820 let forwarding = Arc::new(ForwardingTable::default());
821 forwarding.operator_confirms().configure_bootstrap(&config);
822 let configured_modules = config.configured_modules.clone();
823 let storage_config = config.storage_config.clone();
824 let admission_facts = config.admission_facts.clone();
825 let scope_authority_owners = config.scope_authority_owners.clone();
826 let federated_scope_owner = config.federated_scope_owner.clone();
827 let privileged_operator_prompt = config.privileged_operator_prompt;
828 let activity_config = config.activity_config;
829 let daemon_config_path = config.daemon_config_path.clone();
830 let configured_port = config.configured_port;
831 let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
832 let reserved_capabilities = config.reserved_capabilities.clone();
833 let watchdog_config = config.watchdog_config.clone();
834 let cgroup_placement_config = config.cgroup_placement.clone();
835 let capture_logs_dir = config.capture_logs_dir.clone();
836 let terminal_journal_path = config.terminal_journal_path.clone();
837 let live_children_path = config.live_children_path.clone();
838 let privacy_trampoline = config.privacy_trampoline.clone();
839 #[cfg(all(test, feature = "test-support"))]
840 let test_control = config.test_control;
841 match ensure_singleton_with_start_lock(config, false, start_lock).await? {
842 Outcome::AlreadyRunning => {
843 info!("subc daemon already running");
844 Ok(())
845 }
846 Outcome::Bound(bound) => {
847 #[cfg(target_os = "linux")]
848 let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
849 #[cfg(not(target_os = "linux"))]
850 let _ = cgroup_placement_config;
851 serve_bound_daemon(
852 bound,
853 forwarding,
854 configured_modules,
855 storage_config,
856 admission_facts,
857 scope_authority_owners,
858 federated_scope_owner,
859 privileged_operator_prompt,
860 activity_config,
861 daemon_config_path,
862 opened_store,
863 configured_port,
864 route_bind_relay_default_ms,
865 reserved_capabilities,
866 watchdog_config,
867 capture_logs_dir,
868 terminal_journal_path,
869 live_children_path,
870 privacy_trampoline,
871 #[cfg(all(test, feature = "test-support"))]
872 test_control,
873 #[cfg(feature = "test-support")]
874 shutdown_intent_window,
875 #[cfg(target_os = "linux")]
876 cgroup_placement,
877 )
878 .await
879 }
880 }
881}
882
883fn load_bootstrap_authority(
886 config: BootstrapConfig,
887 mut store: Option<&mut crate::config_store::ConfigStore>,
888) -> Result<BootstrapConfig, BootstrapError> {
889 let store_state = store.as_mut().map(|store| store.status().state);
890 let recovery = matches!(
891 store_state,
892 Some(
893 crate::config_store::StoreState::StoreUnverified
894 | crate::config_store::StoreState::StoreMissing
895 )
896 );
897 let authority = if recovery {
898 BootstrapConfig::new(&config.connection_file_path, DEFAULT_SUBC_PORT)
899 } else if store_state == Some(crate::config_store::StoreState::Stamped) {
900 let store = store.expect("stamped store was just checked");
901 match stamped_authority(&config.connection_file_path, store) {
902 Ok(authority) => authority,
903 Err(reason) => {
904 tracing::error!(%reason, "approved configuration unusable; starting in recovery");
907 store.enter_recovery(&reason);
908 BootstrapConfig::new(&config.connection_file_path, DEFAULT_SUBC_PORT)
909 }
910 }
911 } else if config.defer_file_load {
912 BootstrapConfig::from_env_with_daemon_config_path(
913 config
914 .daemon_config_path
915 .as_ref()
916 .expect("binary config path"),
917 )?
918 } else {
919 return Ok(config);
920 };
921 Ok(BootstrapConfig {
924 connection_file_path: config.connection_file_path,
925 connection_file_source: config.connection_file_source,
926 daemon_ver: config.daemon_ver,
927 daemon_config_path: config.daemon_config_path,
928 config_store_root: config.config_store_root,
929 defer_file_load: config.defer_file_load,
930 operator_queue_wait: config.operator_queue_wait,
931 operator_timeout: config.operator_timeout,
932 operator_stuck_grace: config.operator_stuck_grace,
933 watchdog_config: config.watchdog_config,
934 cgroup_placement: config.cgroup_placement,
935 capture_logs_dir: config.capture_logs_dir,
936 terminal_journal_path: config.terminal_journal_path,
937 machine_id_path: config.machine_id_path,
938 local_listener: config.local_listener,
939 live_children_path: config.live_children_path,
940 privacy_trampoline: config.privacy_trampoline,
941 #[cfg(feature = "test-support")]
942 shutdown_intent_window: config.shutdown_intent_window,
943 #[cfg(feature = "test-support")]
944 operator_script: config.operator_script,
945 #[cfg(feature = "test-support")]
946 operator_events: config.operator_events,
947 #[cfg(all(test, windows))]
948 profile_override: config.profile_override,
949 #[cfg(all(test, feature = "test-support"))]
950 test_store_factory: config.test_store_factory,
951 #[cfg(all(test, feature = "test-support"))]
952 test_control: config.test_control,
953 #[cfg(all(test, feature = "test-support"))]
954 test_listener_bind: config.test_listener_bind,
955 ..authority
956 })
957}
958
959fn stamped_authority(
962 connection_file_path: &Path,
963 store: &mut crate::config_store::ConfigStore,
964) -> Result<BootstrapConfig, String> {
965 let generation = store
966 .select_boot_generation()
967 .map_err(|error| error.to_string())?;
968 let body = store.body().map_err(|error| error.to_string())?;
969 let layer = body
970 .generations
971 .get(&generation)
972 .and_then(|entry| entry.delivery.as_ref())
973 .and_then(|delivery| delivery.daemon.as_ref())
974 .ok_or("approved daemon snapshot is absent")?;
975 let bytes = layer
976 .bytes
977 .as_deref()
978 .ok_or("approved daemon bytes are absent")?;
979 let path = layer
980 .path
981 .as_deref()
982 .ok_or("approved daemon path is absent")?;
983 let doc = std::str::from_utf8(bytes).map_err(|error| error.to_string())?;
984 let (daemon, activity) = daemon_config::parse_with_activity(doc, Path::new(path))
985 .map_err(|error| format!("approved daemon configuration is invalid: {error}"))?;
986 let port = daemon.port;
987 Ok(
988 BootstrapConfig::new(connection_file_path, port.unwrap_or(DEFAULT_SUBC_PORT))
989 .with_parsed_daemon_config(Some(daemon), activity)
990 .with_daemon_config_source(PathBuf::from(path), port),
991 )
992}
993
994#[cfg(target_os = "linux")]
995fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
996 let result = match config {
997 CgroupPlacementConfig::Disabled => return None,
998 CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
999 CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
1000 };
1001
1002 match result {
1003 Ok(Some(placement)) => Some(placement),
1004 Ok(None) => {
1005 warn!(
1006 placement = ?config,
1007 "module cgroup placement is disabled: configured cgroup root is not delegated"
1008 );
1009 None
1010 }
1011 Err(error) => {
1012 warn!(
1013 placement = ?config,
1014 error = %error,
1015 "module cgroup placement is disabled by an unexpected cgroup probe error"
1016 );
1017 None
1018 }
1019 }
1020}
1021
1022#[cfg(unix)]
1029const NOFILE_TARGET: u64 = 65536;
1030
1031#[cfg(unix)]
1035fn raise_nofile_limit() {
1036 #[cfg(target_os = "macos")]
1037 let ceiling = subc_os::child_spawn::output(
1038 std::process::Command::new("/usr/sbin/sysctl").args(["-n", "kern.maxfilesperproc"]),
1039 )
1040 .ok()
1041 .filter(|output| output.status.success())
1042 .and_then(|output| String::from_utf8(output.stdout).ok())
1043 .and_then(|value| value.trim().parse::<u64>().ok())
1044 .filter(|value| *value > 0);
1045 #[cfg(not(target_os = "macos"))]
1046 let ceiling = None;
1047 raise_nofile_limit_with_ceiling(ceiling);
1048}
1049
1050#[cfg(unix)]
1051fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
1052 match rlimit::Resource::NOFILE.get() {
1053 Ok((soft, hard)) => {
1054 let target = NOFILE_TARGET
1058 .min(hard)
1059 .min(kernel_ceiling.unwrap_or(u64::MAX));
1060 if soft >= target {
1061 return;
1062 }
1063 match rlimit::Resource::NOFILE.set(target, hard) {
1064 Ok(()) => info!(
1065 previous_soft = soft,
1066 new_soft = target,
1067 hard,
1068 "raised open-file soft limit for daemon and module children"
1069 ),
1070 Err(err) => warn!(
1071 soft,
1072 hard,
1073 error = %err,
1074 "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
1075 ),
1076 }
1077 }
1078 Err(err) => warn!(error = %err, "could not read open-file limit"),
1079 }
1080}
1081
1082#[cfg(windows)]
1089fn raise_nofile_limit() {
1090 const MAXSTDIO_TARGET: u32 = 8192;
1091 let current = rlimit::getmaxstdio();
1092 if current >= MAXSTDIO_TARGET {
1093 return;
1094 }
1095 match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
1096 Ok(new_max) => info!(
1097 previous = current,
1098 new_max, "raised CRT stdio-stream limit for daemon"
1099 ),
1100 Err(err) => warn!(
1101 current,
1102 error = %err,
1103 "could not raise CRT stdio-stream limit"
1104 ),
1105 }
1106}
1107
1108#[cfg(not(any(unix, windows)))]
1109fn raise_nofile_limit() {}
1110
1111pub async fn run_with_daemon_config_path(
1112 config: BootstrapConfig,
1113 daemon_config_path: impl AsRef<Path>,
1114) -> Result<(), BootstrapError> {
1115 run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
1116}
1117
1118#[allow(clippy::too_many_arguments)]
1119async fn serve_bound_daemon(
1120 bound: BoundDaemon,
1121 forwarding: Arc<ForwardingTable>,
1122 configured_modules: Vec<ConfiguredModule>,
1123 storage_config: Option<daemon_config::StorageConfig>,
1124 admission_facts: AdmissionFactsConfig,
1125 scope_authority_owners: Vec<String>,
1126 federated_scope_owner: String,
1127 privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
1128 activity_config: daemon_config::ActivityConfig,
1129 daemon_config_path: Option<PathBuf>,
1130 opened_store: Option<(
1131 crate::config_store::ConfigStore,
1132 crate::config_store::StoreWriterLock,
1133 )>,
1134 configured_port: Option<u16>,
1135 route_bind_relay_default_ms: Option<u64>,
1136 reserved_capabilities: BTreeMap<String, String>,
1137 watchdog_config: DaemonSelfWatchdogConfig,
1138 capture_logs_dir: Option<PathBuf>,
1139 terminal_journal_path: Option<PathBuf>,
1140 live_children_path: Option<PathBuf>,
1141 privacy_trampoline: Option<PathBuf>,
1142 #[cfg(all(test, feature = "test-support"))] test_control: Option<
1143 fn(ControlHandler) -> ControlHandler,
1144 >,
1145 #[cfg(feature = "test-support")] shutdown_intent_window: Duration,
1146 #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
1147) -> Result<(), BootstrapError> {
1148 #[cfg(unix)]
1149 let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
1150 .map_err(BootstrapError::Signal)?;
1151 raise_nofile_limit();
1154 #[cfg(windows)]
1155 if let Err(error) = subc_jobobject::ensure_root_job() {
1156 warn!(%error, "module launches disabled: Windows root job setup failed");
1159 }
1160
1161 info!(
1162 connection_file = %bound.connection_file_path.display(),
1163 connection_file_source = %bound.connection_file_source,
1164 connection_file_source_reason = bound.connection_file_source.reason(),
1165 endpoints = ?bound.connection_info.endpoints,
1166 configured_modules = configured_modules.len(),
1167 machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
1168 "subc daemon starting"
1169 );
1170
1171 let run_dir_lock = bound.run_dir_lock;
1182 let publication_lock = bound.publication_lock;
1183 if let Some(owner) = &run_dir_lock {
1184 #[cfg(target_os = "linux")]
1185 crate::live_children::sweep_orphans_with_placement(
1186 owner,
1187 &crate::live_children::AdoptedPids::none(),
1188 crate::live_children::SweepBounds::default(),
1189 cgroup_placement.as_ref(),
1190 )
1191 .await;
1192 #[cfg(not(target_os = "linux"))]
1193 crate::live_children::sweep_orphans(
1194 owner,
1195 &crate::live_children::AdoptedPids::none(),
1196 crate::live_children::SweepBounds::default(),
1197 )
1198 .await;
1199 }
1200
1201 let registry = Arc::new(Registry::default());
1202 let process_liveness = Arc::new(SupervisorProcessLiveness::new());
1203 let supervisor_handle = SupervisorHandle::new();
1204 let _host_health_task = AbortOnDrop::new(supervisor_handle.spawn_host_health_monitor());
1205 #[cfg(feature = "test-support")]
1206 supervisor_handle.set_shutdown_intent_window(shutdown_intent_window);
1207 let connected_clients = ConnectedClients::new();
1208 let daemon_incarnation = format!(
1209 "{:032x}",
1210 u128::from_be_bytes(bound.connection_info.daemon_id)
1211 );
1212 let config_runtime = if let Some((store, writer_lock)) = opened_store {
1213 let path = daemon_config_path.clone().ok_or_else(|| {
1214 BootstrapError::ConfigStore(
1215 "an explicit daemon config path is required with a config store".into(),
1216 )
1217 })?;
1218 let config_home = daemon_config::default_config_home();
1219 let incarnation = daemon_incarnation.clone();
1220 let projects = Arc::new(
1221 crate::config_gate::project_registry::EntorhinalProjectRegistry::new(
1222 registry.clone(),
1223 forwarding.clone(),
1224 ),
1225 );
1226 Some(Arc::new(
1227 tokio::task::spawn_blocking(move || {
1228 crate::config_gate::runtime::RuntimeOwner::new(
1229 incarnation,
1230 store,
1231 crate::config_gate::capture::ConfigCapturer {
1232 daemon_path: path,
1233 config_home,
1234 projects,
1235 },
1236 )
1237 .retain_writer_lock(writer_lock)
1238 .map_err(io::Error::other)
1239 })
1240 .await
1241 .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?
1242 .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?,
1243 ))
1244 } else {
1245 None
1246 };
1247 let _config_plan_sweep = config_runtime.as_ref().map(|runtime| {
1248 let runtime = Arc::downgrade(runtime);
1249 AbortOnDrop::new(tokio::spawn(async move {
1250 let mut interval =
1251 tokio::time::interval(crate::config_gate::runtime::PLAN_SWEEP_INTERVAL);
1252 loop {
1253 interval.tick().await;
1254 let Some(runtime) = runtime.upgrade() else {
1255 break;
1256 };
1257 match tokio::task::spawn_blocking(move || runtime.sweep_expired()).await {
1258 Ok(Ok(())) => {}
1259 result => tracing::warn!(?result, "configuration plan expiry sweep failed"),
1260 }
1261 }
1262 }))
1263 });
1264 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
1265 .with_process_liveness(process_liveness.clone())
1266 .with_forwarding(Arc::clone(&forwarding))
1267 .with_handle(supervisor_handle.clone())
1268 .with_connection_file_path(bound.connection_file_path.clone())
1269 .with_daemon_incarnation(daemon_incarnation.clone());
1270 let supervisor = match &config_runtime {
1271 Some(owner) => supervisor.with_config_runtime(owner.clone()),
1272 None => supervisor,
1273 };
1274 let supervisor = match privacy_trampoline {
1275 Some(path) => supervisor.with_privacy_trampoline(path),
1276 None => supervisor,
1277 };
1278 let activity_run_dir = terminal_journal_path
1318 .as_ref()
1319 .and_then(|path| path.parent())
1320 .map(Path::to_path_buf);
1321 let supervisor = match terminal_journal_path {
1322 Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
1323 None => supervisor,
1324 };
1325 let supervisor = match capture_logs_dir {
1326 Some(dir) => supervisor.with_capture_logs_dir(dir),
1327 None => supervisor,
1328 };
1329 let supervisor = match live_children_path {
1330 Some(path) => supervisor.with_live_children_record(path),
1331 None => supervisor,
1332 };
1333 #[cfg(target_os = "linux")]
1334 let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
1335 let route_bind_relay_timeouts = configured_modules
1341 .iter()
1342 .filter_map(|module| {
1343 module
1344 .route_bind_relay_timeout_ms
1345 .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
1346 })
1347 .collect::<std::collections::BTreeMap<_, _>>();
1348 let control_start_clock = crate::clock::StartClock::capture();
1349 let mut control = ControlHandler::with_forwarding(Arc::clone(®istry), forwarding)
1350 .with_daemon_version(bound.connection_info.daemon_ver.clone())
1351 .with_process_liveness(process_liveness)
1352 .with_supervisor(supervisor_handle)
1353 .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
1356 .with_connected_clients(connected_clients.clone())
1357 .with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
1358 .with_storage_config(storage_config)
1359 .with_machine_id(bound.machine_id.clone())
1360 .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
1361 .with_scope_authority_owners(scope_authority_owners)
1362 .with_federated_scope_owner(federated_scope_owner)
1363 .with_privileged_operator_prompt(privileged_operator_prompt)
1364 .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
1365 .with_daemon_provenance(
1366 bound.connection_info.pid,
1367 control_start_clock.started_at_ms(),
1368 std::env::current_exe().ok(),
1369 normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
1370 normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
1371 )
1372 .with_daemon_start_clock(control_start_clock)
1373 .with_capability_config(
1374 configured_modules
1375 .iter()
1376 .map(|module| (module.module_id.clone(), module.enabled)),
1377 reserved_capabilities,
1378 );
1379 if let Some(run_dir) = activity_run_dir {
1380 control =
1381 control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
1382 }
1383 let daemon_ready_runtime = config_runtime.clone();
1384 if let Some(runtime) = config_runtime {
1385 control = control.with_explicit_config_store(runtime);
1386 }
1387 #[cfg(feature = "test-support")]
1388 if config_test_seam::enabled() {
1389 control = control.with_config_os_prompt_policy_for_tests();
1390 }
1391 if let Some(ms) = route_bind_relay_default_ms {
1392 control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
1395 }
1396 if let Some(config_path) = daemon_config_path {
1397 control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
1398 control = control.with_supervisor_rescan_entries(&configured_modules);
1399 }
1400 #[cfg(all(test, feature = "test-support"))]
1401 let control = if let Some(adapt) = test_control {
1402 let executable =
1404 std::env::current_exe().map_err(|e| BootstrapError::ConfigStore(e.to_string()))?;
1405 if !executable
1406 .file_name()
1407 .is_some_and(|name| name.to_string_lossy().starts_with("ckdev-"))
1408 {
1409 return Err(BootstrapError::ConfigStore(
1410 "test control requires a ckdev executable".into(),
1411 ));
1412 }
1413 adapt(control)
1414 } else {
1415 control
1416 };
1417 let control = Arc::new(control);
1418 let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
1419 let auth = ServerAuth::new(
1420 bound.connection_info.key.clone(),
1421 bound.connection_info.daemon_id,
1422 bound.connection_info.daemon_ver.clone(),
1423 )
1424 .with_connected_clients(connected_clients);
1425
1426 let mut registered_modules = Vec::with_capacity(configured_modules.len());
1430 for configured in configured_modules {
1431 let enabled = configured.enabled;
1432 let health = configured.health.clone();
1433 let module_id = configured.module_id.clone();
1434 let registered = match supervisor.register_configured_with_health(
1435 configured.module_spec(),
1436 enabled,
1437 health.clone(),
1438 configured.drain_timeout_ms,
1439 configured.restart,
1440 configured.register_timeout,
1441 ) {
1442 Ok(registered) => registered,
1443 Err(err) => {
1444 error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
1445 continue;
1446 }
1447 };
1448 registered_modules.push(registered);
1449 let default_threshold = HealthConfig::default().failure_threshold;
1452 if enabled && health.failure_threshold > default_threshold {
1453 warn!(
1454 module_id = %module_id,
1455 failure_threshold = health.failure_threshold,
1456 default_threshold,
1457 tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1458 "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1459 );
1460 }
1461 info!(module_id = %module_id, enabled, "configured module registered");
1462 }
1463 control.refresh_capability_requirements();
1464
1465 let local_serve_task = bound.local_listener.map(|local_listener| {
1468 AbortOnDrop::new(tokio::spawn(serve_local_listener(
1469 local_listener,
1470 Arc::clone(&router),
1471 auth.clone(),
1472 )))
1473 });
1474 #[cfg(windows)]
1477 let _local_serve_task = local_serve_task;
1478 let mut serve_task =
1479 AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1480 verify_serving(&bound.connection_info).await?;
1481 let (published, publication_lock) = publish_connection_file(
1488 &bound.connection_file_path,
1489 #[cfg(windows)]
1490 bound.connection_file_copy.as_deref(),
1491 &bound.connection_info,
1492 bound.local_endpoint.as_ref(),
1493 publication_lock,
1494 )
1495 .await;
1496 published?;
1497 if let Some(owner) = daemon_ready_runtime {
1501 if let Err(error) = owner.on_blocking(|owner| owner.record_daemon_ready()).await {
1502 warn!(%error, "daemon readiness was not recorded in the approved store");
1503 }
1504 }
1505 #[cfg(test)]
1506 startup_publication_hook::run(&bound.connection_file_path).await;
1507 drop(publication_lock);
1508 let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1509 #[cfg(target_os = "linux")]
1511 let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1512 crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1513 ));
1514 let _watchdog_task = AbortOnDrop::new(
1515 DaemonSelfWatchdog::new(
1516 bound.connection_info.clone(),
1517 bound.connection_file_path.clone(),
1518 )
1519 .with_config(watchdog_config)
1520 .spawn(),
1521 );
1522
1523 for registered in registered_modules {
1524 match supervisor
1525 .start_registered_configured_async(registered, None, true)
1526 .await
1527 {
1528 Ok(module) => {
1529 info!(module_id = %module.module_id(), "configured module supervised");
1530 }
1531 Err(err) => {
1532 error!(error = %err, "failed to start configured module; continuing daemon startup");
1533 }
1534 }
1535 }
1536
1537 control.refresh_capability_requirements();
1538 Arc::clone(&control).spawn_capability_deadline_loop();
1539 Arc::clone(&control).spawn_scope_expiry_loop();
1540 let _config_delivery_task = control.spawn_config_delivery_worker().map(AbortOnDrop::new);
1541 control.spawn_resource_sampler();
1542
1543 #[cfg(unix)]
1544 {
1545 tokio::select! {
1546 result = serve_task.join() => {
1547 return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1548 }
1549 _ = terminate.recv() => {}
1550 }
1551 supervisor.begin_daemon_shutdown();
1556 drop(_watchdog_task);
1560 drop(serve_task);
1564 drop(local_serve_task);
1565 let second_sigterm = async {
1566 terminate.recv().await;
1567 std::time::Instant::now()
1568 };
1569 tokio::pin!(second_sigterm);
1570 let mut escalation_started = tokio::select! {
1571 biased;
1572 at = second_sigterm.as_mut() => {
1573 info!("second SIGTERM: abandoning daemon shutdown wait");
1574 Some(at)
1575 }
1576 result = supervisor.drain_for_daemon_shutdown() => {
1577 if let Err(error) = result {
1578 warn!(%error, "daemon shutdown drain failed; exiting anyway");
1579 }
1580 None
1581 }
1582 };
1583 let already_escalated = escalation_started.is_some();
1590 supervisor
1591 .end_children_for_daemon_shutdown(already_escalated, async {
1592 if already_escalated {
1593 std::future::pending::<()>().await;
1594 } else {
1595 escalation_started = Some(second_sigterm.await);
1596 }
1597 })
1598 .await;
1599 if let Some(at) = escalation_started {
1600 info!(
1601 second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1602 "daemon shutdown wait ended after second SIGTERM"
1603 );
1604 }
1605 info!("daemon exiting: SIGTERM shutdown complete");
1606 Ok(())
1607 }
1608 #[cfg(not(unix))]
1609 serve_task
1610 .join()
1611 .await
1612 .map_err(BootstrapError::ServeJoin)?
1613 .map_err(BootstrapError::Serve)
1614}
1615
1616fn normalized_build_provenance(value: &str) -> Option<String> {
1617 match value.trim() {
1618 "" | "unavailable" => None,
1619 value => Some(value.to_string()),
1620 }
1621}
1622
1623pub async fn ensure_singleton(
1631 connection_file_path: impl AsRef<Path>,
1632 port: u16,
1633) -> Result<Outcome, BootstrapError> {
1634 ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1635}
1636
1637pub async fn ensure_singleton_with_config(
1638 config: BootstrapConfig,
1639) -> Result<Outcome, BootstrapError> {
1640 ensure_singleton_inner(config, true).await
1641}
1642
1643async fn ensure_singleton_inner(
1644 config: BootstrapConfig,
1645 publish: bool,
1646) -> Result<Outcome, BootstrapError> {
1647 let Some(lock) = singleton_start_lock(&config.connection_file_path).await? else {
1648 return Ok(Outcome::AlreadyRunning);
1649 };
1650 ensure_singleton_with_start_lock(config, publish, lock).await
1651}
1652
1653async fn singleton_start_lock(path: &Path) -> Result<Option<StartLock>, BootstrapError> {
1654 if stable_peer(path).await? {
1655 return Ok(None);
1656 }
1657
1658 let lock = StartLock::acquire(path).await?;
1659
1660 if stable_peer(path).await? {
1664 return Ok(None);
1665 }
1666 Ok(Some(lock))
1667}
1668
1669async fn ensure_singleton_with_start_lock(
1670 config: BootstrapConfig,
1671 publish: bool,
1672 lock: StartLock,
1673) -> Result<Outcome, BootstrapError> {
1674 #[cfg(windows)]
1675 let copy_path = windows_connection_copy::resolve(&config);
1676 let path = config.connection_file_path;
1677 let run_dir_lock = config
1683 .live_children_path
1684 .as_deref()
1685 .map(crate::run_dir_lock::RunDirLock::acquire)
1686 .transpose()?;
1687
1688 remove_stale_connection_file_if_present(&path)?;
1689 #[cfg(windows)]
1690 let copy_path = windows_connection_copy::reclaim(&path, copy_path).await;
1691
1692 let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1700 None => (None, lock, run_dir_lock),
1701 Some(machine_id_path) => {
1702 let task_path = machine_id_path.clone();
1703 match run_blocking_holding((lock, run_dir_lock), move || {
1704 crate::machine_id::load_or_mint(&task_path)
1705 })
1706 .await
1707 {
1708 Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1709 Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1710 Err(join_error) => {
1711 return Err(BootstrapError::MachineId(
1712 crate::machine_id::MachineIdFileError::Write {
1713 path: machine_id_path,
1714 source: io::Error::other(describe_blocking_task_failure(
1715 "loading or minting the machine id",
1716 &join_error,
1717 )),
1718 },
1719 ))
1720 }
1721 }
1722 }
1723 };
1724
1725 let (listeners, endpoints) = bind_loopback(config.port).await?;
1726 let (local_listener, local_endpoint, local_endpoint_unavailable) =
1732 if config.local_listener.configured_dir().is_some() {
1733 match local_listener::bind(&config.local_listener, &path) {
1734 Ok(local_listener) => {
1735 let endpoint = local_listener.endpoint();
1736 (Some(local_listener), Some(endpoint), None)
1737 }
1738 Err(error) => {
1739 let reason = error.to_string();
1740 warn!(
1741 error = %reason,
1742 "the local endpoint is unavailable; serving loopback TCP only"
1743 );
1744 (None, None, Some(reason))
1745 }
1746 }
1747 } else {
1748 (None, None, None)
1749 };
1750 let connection_info = ConnectionInfo {
1751 schema: SCHEMA_VERSION,
1752 wire_version: Some(PROTOCOL_VERSION),
1753 endpoints,
1754 key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1755 daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1756 pid: process::id(),
1757 daemon_ver: config.daemon_ver,
1758 };
1759
1760 let publication_lock = if publish {
1764 let (published, lock) = publish_connection_file(
1765 &path,
1766 #[cfg(windows)]
1767 copy_path.as_deref(),
1768 &connection_info,
1769 local_endpoint.as_ref(),
1770 lock,
1771 )
1772 .await;
1773 if let Err(error) = published {
1774 if let Some(local_listener) = local_listener.as_ref() {
1775 local_listener::release(local_listener);
1776 }
1777 drop(listeners);
1778 return Err(error);
1779 }
1780 drop(lock);
1781 None
1782 } else {
1783 Some(lock)
1784 };
1785
1786 Ok(Outcome::Bound(BoundDaemon {
1787 listeners,
1788 local_listener,
1789 local_endpoint,
1790 local_endpoint_unavailable,
1791 connection_info,
1792 connection_file_path: path,
1793 connection_file_source: config.connection_file_source,
1794 #[cfg(windows)]
1795 connection_file_copy: copy_path,
1796 machine_id,
1797 run_dir_lock,
1798 publication_lock,
1799 }))
1800}
1801
1802#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1803enum Probe {
1804 Live,
1805 StaleOrAbsent,
1806}
1807
1808async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1809 if !matches!(probe_existing(path).await?, Probe::Live) {
1810 return Ok(false);
1811 }
1812 tokio::time::sleep(Duration::from_millis(1500)).await;
1818 Ok(matches!(probe_existing(path).await?, Probe::Live))
1819}
1820
1821async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1822 let info = match connection_file::read(path) {
1823 Ok(info) => info,
1824 Err(source) if is_absent_or_stale_connection_file(&source) => {
1825 return Ok(Probe::StaleOrAbsent)
1826 }
1827 Err(source) => {
1828 return Err(BootstrapError::ConnectionFileRead {
1829 path: path.to_path_buf(),
1830 source,
1831 })
1832 }
1833 };
1834
1835 for endpoint in &info.endpoints {
1836 if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1837 return Ok(Probe::Live);
1838 }
1839 }
1840
1841 Ok(Probe::StaleOrAbsent)
1842}
1843
1844async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1845 let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1846 return Probe::StaleOrAbsent;
1847 };
1848 if !ip.is_loopback() {
1849 return Probe::StaleOrAbsent;
1850 }
1851 let addr = SocketAddr::new(ip, endpoint.port);
1852
1853 let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1854 Ok(Ok(stream)) => stream,
1855 Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1856 };
1857
1858 match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1859 Ok(()) => Probe::Live,
1860 Err(AuthError::DaemonIdMismatch)
1861 | Err(AuthError::InvalidServerProof)
1862 | Err(AuthError::UnexpectedEof { .. })
1863 | Err(AuthError::Timeout { .. })
1864 | Err(AuthError::JsonEncode { .. })
1865 | Err(AuthError::JsonDecode { .. })
1866 | Err(AuthError::Io { .. })
1867 | Err(AuthError::MessageTooLarge { .. })
1868 | Err(AuthError::KeyTooShort { .. })
1869 | Err(AuthError::Random(_))
1870 | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1871 }
1872}
1873
1874async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1875 if let Some(endpoint) = info.endpoints.first() {
1879 if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1880 return Ok(());
1881 }
1882 }
1883 Err(BootstrapError::StartupNotServing)
1884}
1885
1886async fn run_blocking_holding<G, T>(
1901 guards: G,
1902 work: impl FnOnce() -> T + Send + 'static,
1903) -> Result<(T, G), JoinError>
1904where
1905 G: Send + 'static,
1906 T: Send + 'static,
1907{
1908 tokio::task::spawn_blocking(move || {
1909 let output = work();
1910 (output, guards)
1911 })
1912 .await
1913}
1914
1915fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1917 if join_error.is_panic() {
1918 format!("the background thread {work} panicked before finishing: {join_error}")
1919 } else {
1920 format!(
1921 "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1922 )
1923 }
1924}
1925
1926async fn publish_connection_file<G: Send + 'static>(
1943 path: &Path,
1944 #[cfg(windows)] copy_path: Option<&Path>,
1945 info: &ConnectionInfo,
1946 local_endpoint: Option<&LocalEndpoint>,
1947 guard: G,
1948) -> (Result<(), BootstrapError>, Option<G>) {
1949 let task_path = path.to_path_buf();
1950 let task_info = info.clone();
1951 let task_endpoint = local_endpoint.cloned();
1952 #[cfg(windows)]
1953 let copy = windows_connection_copy::claim_for_publication(path, copy_path, info).await;
1954 let outcome = run_blocking_holding(guard, move || {
1955 write_connection_file(&task_path, &task_info, task_endpoint.as_ref())?;
1956 #[cfg(windows)]
1957 if let Some((copy_path, _lock)) = copy {
1958 if let Err(error) =
1959 write_connection_file(©_path, &task_info, task_endpoint.as_ref())
1960 {
1961 windows_connection_copy::warn_unavailable(&task_path, Some(©_path), &error);
1962 }
1963 }
1964 Ok(())
1965 })
1966 .await;
1967 match outcome {
1968 Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1969 Ok((Err(source), guard)) => (
1970 Err(BootstrapError::ConnectionFileWrite {
1971 path: path.to_path_buf(),
1972 source,
1973 }),
1974 Some(guard),
1975 ),
1976 Err(join_error) => (
1977 Err(BootstrapError::ConnectionFileWrite {
1978 path: path.to_path_buf(),
1979 source: ConnectionFileError::Io {
1980 op: "write",
1981 path: path.to_path_buf(),
1982 source: io::Error::other(describe_blocking_task_failure(
1983 "writing the connection file",
1984 &join_error,
1985 )),
1986 },
1987 }),
1988 None,
1989 ),
1990 }
1991}
1992
1993fn write_connection_file(
2000 path: &Path,
2001 info: &ConnectionInfo,
2002 local_endpoint: Option<&LocalEndpoint>,
2003) -> Result<(), ConnectionFileError> {
2004 #[cfg(test)]
2005 connection_file_write_hook::run(path);
2006 write_atomic_with_local_endpoint(path, info, local_endpoint)
2007}
2008
2009#[cfg(test)]
2012mod connection_file_write_hook {
2013 use std::{
2014 path::{Path, PathBuf},
2015 sync::{Mutex, PoisonError},
2016 };
2017
2018 type Hook = Box<dyn FnOnce() + Send>;
2019
2020 static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
2023
2024 pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
2026 HOOKS
2027 .lock()
2028 .unwrap_or_else(PoisonError::into_inner)
2029 .push((path.to_path_buf(), Box::new(hook)));
2030 }
2031
2032 pub(super) fn run(path: &Path) {
2033 let hook = {
2034 let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
2035 hooks
2036 .iter()
2037 .position(|(hooked, _)| hooked == path)
2038 .map(|index| hooks.swap_remove(index).1)
2039 };
2040 if let Some(hook) = hook {
2041 hook();
2042 }
2043 }
2044}
2045
2046#[cfg(test)]
2049mod startup_publication_hook {
2050 use std::{
2051 path::{Path, PathBuf},
2052 sync::{Mutex, PoisonError},
2053 };
2054 use tokio::sync::oneshot;
2055
2056 type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
2057 static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
2058
2059 pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
2060 let (entered_tx, entered_rx) = oneshot::channel();
2061 let (release_tx, release_rx) = oneshot::channel();
2062 BARRIERS
2063 .lock()
2064 .unwrap_or_else(PoisonError::into_inner)
2065 .push((path.to_path_buf(), (entered_tx, release_rx)));
2066 (entered_rx, release_tx)
2067 }
2068
2069 pub(super) async fn run(path: &Path) {
2070 let barrier = {
2071 let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
2072 barriers
2073 .iter()
2074 .position(|(hooked, _)| hooked == path)
2075 .map(|index| barriers.swap_remove(index).1)
2076 };
2077 if let Some((entered, release)) = barrier {
2078 let _ = entered.send(());
2079 let _ = release.await;
2080 }
2081 }
2082}
2083
2084fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
2085 match err {
2086 ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
2087 ConnectionFileError::JsonRead { .. }
2088 | ConnectionFileError::UnsupportedSchema { .. }
2089 | ConnectionFileError::Invalid { .. }
2090 | ConnectionFileError::KeyTooShort { .. }
2091 | ConnectionFileError::InsecurePermissions { .. } => true,
2095 ConnectionFileError::MissingParent { .. }
2096 | ConnectionFileError::MissingFileName { .. }
2097 | ConnectionFileError::InsecureParentDirectory { .. }
2101 | ConnectionFileError::Io { .. }
2102 | ConnectionFileError::JsonWrite { .. }
2103 | ConnectionFileError::Random(_)
2104 | ConnectionFileError::WireVersionMismatch { .. } => false,
2107 }
2108}
2109
2110fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
2111 match fs::remove_file(path) {
2112 Ok(()) => Ok(()),
2113 Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
2114 Err(source) => Err(BootstrapError::RemoveStale {
2115 path: path.to_path_buf(),
2116 source,
2117 }),
2118 }
2119}
2120
2121async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
2122 bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
2123}
2124
2125async fn bind_loopback_with<F, Fut>(
2126 port: u16,
2127 mut bind_v6: F,
2128) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
2129where
2130 F: FnMut(u16) -> Fut,
2131 Fut: std::future::Future<Output = io::Result<TcpListener>>,
2132{
2133 let v4_host = Ipv4Addr::LOCALHOST;
2134 let v4 = TcpListener::bind((v4_host, port))
2135 .await
2136 .map_err(|source| BootstrapError::Bind {
2137 host: v4_host.to_string(),
2138 port,
2139 source,
2140 })?;
2141 let actual_port = v4
2142 .local_addr()
2143 .map_err(|source| BootstrapError::LocalAddr {
2144 host: v4_host.to_string(),
2145 source,
2146 })?
2147 .port();
2148
2149 let mut listeners = vec![v4];
2150 let mut endpoints = vec![Endpoint {
2151 host: v4_host.to_string(),
2152 port: actual_port,
2153 }];
2154
2155 let v6_host = Ipv6Addr::LOCALHOST;
2156 match bind_v6(actual_port).await {
2157 Ok(v6) => {
2158 listeners.push(v6);
2159 endpoints.push(Endpoint {
2160 host: v6_host.to_string(),
2161 port: actual_port,
2162 });
2163 }
2164 Err(err)
2165 if ipv6_loopback_unavailable(&err)
2166 || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
2167 {
2168 warn!(
2169 port = actual_port,
2170 error = %err,
2171 "IPv6 loopback unavailable; serving only IPv4 loopback"
2172 );
2173 }
2174 Err(source) => {
2175 drop(listeners);
2176 return Err(BootstrapError::Bind {
2177 host: v6_host.to_string(),
2178 port: actual_port,
2179 source,
2180 });
2181 }
2182 }
2183
2184 Ok((listeners, endpoints))
2185}
2186
2187fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
2188 matches!(
2189 err.kind(),
2190 io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
2191 ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
2192}
2193
2194struct AbortOnDrop<T> {
2195 handle: JoinHandle<T>,
2196}
2197
2198impl<T> AbortOnDrop<T> {
2199 fn new(handle: JoinHandle<T>) -> Self {
2200 Self { handle }
2201 }
2202
2203 async fn join(&mut self) -> Result<T, JoinError> {
2204 (&mut self.handle).await
2205 }
2206}
2207
2208impl<T> Drop for AbortOnDrop<T> {
2209 fn drop(&mut self) {
2210 if !self.handle.is_finished() {
2211 self.handle.abort();
2212 }
2213 }
2214}
2215
2216#[derive(Debug)]
2217struct StartLock {
2218 _file: fs::File,
2220}
2221
2222impl Drop for StartLock {
2223 fn drop(&mut self) {
2224 if let Err(error) = FileExt::unlock(&self._file) {
2228 tracing::warn!(%error, "failed to release the singleton start lock");
2229 }
2230 }
2231}
2232
2233impl StartLock {
2234 async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
2235 let path = start_lock_path(connection_file_path);
2236 for _ in 0..START_LOCK_RETRIES {
2237 let file = match open_owner_only_lock(&path) {
2238 Ok(file) => file,
2239 Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
2240 };
2241 match FileExt::try_lock(&file) {
2242 Ok(()) => return Ok(Self { _file: file }),
2243 Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
2244 Err(TryLockError::Error(source)) => {
2245 return Err(BootstrapError::StartLockCreate { path, source });
2246 }
2247 }
2248 }
2249
2250 Err(BootstrapError::StartLockBusy {
2251 path,
2252 attempts: START_LOCK_RETRIES,
2253 })
2254 }
2255}
2256
2257pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
2258 let mut options = fs::OpenOptions::new();
2259 options.read(true).write(true).create(true);
2260 #[cfg(unix)]
2261 {
2262 use std::os::unix::fs::OpenOptionsExt;
2263 options.mode(0o600);
2264 }
2265 options.open(path)
2266}
2267
2268fn start_lock_path(connection_file_path: &Path) -> PathBuf {
2269 let file_name = connection_file_path
2270 .file_name()
2271 .map(|name| name.to_string_lossy())
2272 .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
2273 let lock_name = format!("{file_name}.start-lock");
2274 connection_file_path
2275 .parent()
2276 .filter(|parent| !parent.as_os_str().is_empty())
2277 .unwrap_or_else(|| Path::new("."))
2278 .join(lock_name)
2279}
2280
2281fn non_empty_os_var(key: &str) -> Option<OsString> {
2282 let value = env::var_os(key)?;
2283 if value.is_empty() {
2284 None
2285 } else {
2286 Some(value)
2287 }
2288}
2289
2290#[derive(Debug)]
2293#[non_exhaustive]
2294pub enum BootstrapError {
2295 ConfigStore(String),
2296 StartupNotServing,
2298 #[cfg(unix)]
2299 Signal(io::Error),
2300 InvalidPort {
2301 raw: String,
2302 source: std::num::ParseIntError,
2303 },
2304 ConnectionFileRead {
2305 path: PathBuf,
2306 source: ConnectionFileError,
2307 },
2308 ConnectionFileWrite {
2309 path: PathBuf,
2310 source: ConnectionFileError,
2311 },
2312 GenerateConnectionFile(ConnectionFileError),
2313 StartLockCreate {
2314 path: PathBuf,
2315 source: io::Error,
2316 },
2317 StartLockBusy {
2318 path: PathBuf,
2319 attempts: usize,
2320 },
2321 RunDirLockCreate {
2323 path: PathBuf,
2324 source: io::Error,
2325 },
2326 RunDirBusy {
2331 path: PathBuf,
2332 holder_pid: Option<u32>,
2333 },
2334 RemoveStale {
2335 path: PathBuf,
2336 source: io::Error,
2337 },
2338 Bind {
2339 host: String,
2340 port: u16,
2341 source: io::Error,
2342 },
2343 LocalAddr {
2344 host: String,
2345 source: io::Error,
2346 },
2347 DaemonConfig(DaemonConfigError),
2348 MachineId(crate::machine_id::MachineIdFileError),
2351 RunDir(daemon_config::DaemonRunDirError),
2354 Serve(ServerError),
2355 ServeJoin(tokio::task::JoinError),
2356}
2357
2358impl From<ServerError> for BootstrapError {
2359 fn from(error: ServerError) -> Self {
2360 Self::Serve(error)
2361 }
2362}
2363
2364impl fmt::Display for BootstrapError {
2365 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2366 match self {
2367 Self::ConfigStore(error) => write!(f, "configuration store startup refused: {error}"),
2368 Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
2369 #[cfg(unix)]
2370 Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
2371 Self::InvalidPort { raw, source } => {
2372 write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
2373 }
2374 Self::ConnectionFileRead { path, source } => write!(
2375 f,
2376 "failed to read connection file {}: {source}",
2377 path.display()
2378 ),
2379 Self::ConnectionFileWrite { path, source } => write!(
2380 f,
2381 "failed to publish connection file {}: {source}",
2382 path.display()
2383 ),
2384 Self::GenerateConnectionFile(err) => {
2385 write!(f, "failed to generate connection-file auth material: {err}")
2386 }
2387 Self::StartLockCreate { path, source } => {
2388 write!(
2389 f,
2390 "failed to create start lock {}: {source}",
2391 path.display()
2392 )
2393 }
2394 Self::StartLockBusy { path, attempts } => write!(
2395 f,
2396 "start lock {} remained busy after {attempts} attempts",
2397 path.display()
2398 ),
2399 Self::RunDirLockCreate { path, source } => write!(
2400 f,
2401 "refusing to start: failed to lock run directory via {}: {source}",
2402 path.display()
2403 ),
2404 Self::RunDirBusy { path, holder_pid } => {
2405 write!(
2406 f,
2407 "refusing to start: run directory lock {} is held by another daemon",
2408 path.display()
2409 )?;
2410 if let Some(pid) = holder_pid {
2411 write!(f, " (pid {pid})")?;
2412 }
2413 write!(
2414 f,
2415 "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
2416 )
2417 }
2418 Self::RemoveStale { path, source } => write!(
2419 f,
2420 "failed to remove stale connection file {}: {source}",
2421 path.display()
2422 ),
2423 Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
2424 write!(
2425 f,
2426 "port {port} in use on loopback {host}: {source}; set the port in config"
2427 )
2428 }
2429 Self::Bind { host, port, source } => {
2430 write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
2431 }
2432 Self::LocalAddr { host, source } => {
2433 write!(f, "failed to read local address for {host}: {source}")
2434 }
2435 Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
2436 Self::MachineId(err) => write!(f, "refusing to start: {err}"),
2437 Self::RunDir(err) => write!(f, "refusing to start: {err}"),
2438 Self::Serve(err) => write!(f, "daemon server failed: {err}"),
2439 Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
2440 }
2441 }
2442}
2443
2444impl Error for BootstrapError {
2445 fn source(&self) -> Option<&(dyn Error + 'static)> {
2446 match self {
2447 Self::ConfigStore(_) => None,
2448 Self::StartupNotServing => None,
2449 #[cfg(unix)]
2450 Self::Signal(source) => Some(source),
2451 Self::InvalidPort { source, .. } => Some(source),
2452 Self::ConnectionFileRead { source, .. }
2453 | Self::ConnectionFileWrite { source, .. }
2454 | Self::GenerateConnectionFile(source) => Some(source),
2455 Self::StartLockCreate { source, .. }
2456 | Self::RunDirLockCreate { source, .. }
2457 | Self::RemoveStale { source, .. }
2458 | Self::Bind { source, .. }
2459 | Self::LocalAddr { source, .. } => Some(source),
2460 Self::DaemonConfig(err) => Some(err),
2461 Self::MachineId(err) => Some(err),
2462 Self::RunDir(err) => Some(err),
2463 Self::Serve(err) => Some(err),
2464 Self::ServeJoin(err) => Some(err),
2465 Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
2466 }
2467 }
2468}
2469
2470#[cfg(all(test, feature = "test-support", unix))]
2471#[path = "bootstrap_recovery_tests.rs"]
2472mod recovery_tests;
2473
2474#[cfg(test)]
2475mod tests {
2476 mod activity {
2477 include!("activity_bootstrap_tests.rs");
2478 }
2479 mod startup_order {
2480 include!("startup_order_tests.rs");
2481 }
2482 use super::*;
2483 use crate::server::ServerAuth;
2484 use cortexkit_test_support::ScratchDir;
2485 #[cfg(target_os = "linux")]
2486 use std::collections::BTreeSet;
2487 use std::sync::Mutex;
2488 #[cfg(target_os = "linux")]
2489 use subc_control::ModuleProtocol;
2490 use subc_transport::{write_atomic, MIN_KEY_LEN};
2491 use tokio::io::AsyncReadExt;
2492 use tokio::task::JoinHandle;
2493
2494 #[cfg(unix)]
2495 use std::os::unix::fs::PermissionsExt;
2496
2497 static ENV_LOCK: Mutex<()> = Mutex::new(());
2498
2499 #[test]
2500 fn normalized_build_provenance_preserves_real_values() {
2501 assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
2502 }
2503
2504 #[test]
2505 fn config_recovery_bootstrap_preserves_all_loaded_fields_and_only_argument_overrides() {
2506 let _env_lock = ENV_LOCK.lock().unwrap();
2507 let scratch = unique_temp_dir("bootstrap-loaded-fields");
2508 let file = scratch.join("subc.jsonc");
2509 std::fs::write(
2510 &file,
2511 br#"{"version":1,"modules":{},"scope_authority_owners":["file-owner"]}"#,
2512 )
2513 .unwrap();
2514 let mut config = BootstrapConfig::new(scratch.join("connection.json"), 0)
2515 .with_daemon_config_source(file.clone(), None);
2516 config.defer_file_load = true;
2517 config.operator_timeout = Duration::from_secs(7);
2518 let selected = load_bootstrap_authority(config, None).unwrap();
2519 assert_eq!(
2520 selected.file_derived_test_probe,
2521 Some(vec!["file-owner".into()])
2522 );
2523 assert_eq!(selected.operator_timeout, Duration::from_secs(7));
2524 assert_eq!(selected.daemon_config_path, Some(file));
2525 assert_eq!(
2526 selected.connection_file_path,
2527 scratch.join("connection.json")
2528 );
2529 }
2530
2531 #[test]
2532 fn config_recovery_bootstrap_ignores_invalid_live_file_and_poisoned_settings() {
2533 let scratch = unique_temp_dir("recovery-bootstrap");
2534 let root = scratch.join("store");
2535 std::fs::create_dir_all(&root).unwrap();
2536 std::fs::write(
2537 root.join(crate::config_store::STORE_FILE),
2538 br#"{"format_version":999}"#,
2539 )
2540 .unwrap();
2541 let mut store = crate::config_store::ConfigStore::open_digest_only_for_tests(
2542 crate::config_store::StoreRoot::from_service_definition(root).unwrap(),
2543 );
2544 assert_eq!(
2545 store.status().state,
2546 crate::config_store::StoreState::StoreUnverified
2547 );
2548 let file = scratch.join("subc.jsonc");
2549 std::fs::write(&file, b"invalid live config").unwrap();
2550 let mut config = BootstrapConfig::new(scratch.join("connection.json"), 1)
2551 .with_daemon_config_source(file, Some(1))
2552 .with_scope_authority_owners(vec!["untrusted".into()])
2553 .with_reserved_capabilities(BTreeMap::from([("untrusted".into(), "untrusted".into())]));
2554 config.defer_file_load = true;
2555 config.file_derived_test_probe = Some(vec!["untrusted".into()]);
2556 let config = load_bootstrap_authority(config, Some(&mut store)).unwrap();
2557 assert_eq!(config.file_derived_test_probe, None);
2558 assert_eq!(config.port, DEFAULT_SUBC_PORT);
2559 assert!(config.configured_modules.is_empty());
2560 assert_eq!(
2561 config.scope_authority_owners,
2562 daemon_config::default_scope_authority_owners()
2563 );
2564 assert!(config.reserved_capabilities.is_empty());
2565 assert_eq!(
2566 config.privileged_operator_prompt,
2567 daemon_config::PrivilegedOperatorPrompt::default()
2568 );
2569 }
2570
2571 fn approved_port_entry(
2574 kind: crate::config_store::CommitKind,
2575 port: u16,
2576 ) -> crate::config_store::HistoryEntry {
2577 use subc_protocol::config::{ConfigLayer, ConfigLayerSnapshot};
2578 crate::config_store::HistoryEntry {
2579 generation: subc_config::Generation::new(
2580 0,
2581 subc_config::ApprovalChannel::OsPrompt,
2582 subc_config::raw_digest(b"approved-plan"),
2583 subc_config::InputVector::default(),
2584 subc_config::Changeset::new(),
2585 ),
2586 kind,
2587 approval_id: "approval".into(),
2588 outcome: "approved".into(),
2589 tuples: Default::default(),
2590 configuration: Default::default(),
2591 delivery: Some(crate::config_store::DeliverySnapshot {
2592 daemon: Some(ConfigLayerSnapshot::file(
2593 "subc",
2594 ConfigLayer::User,
2595 "/approved/subc.jsonc",
2596 Some(format!(r#"{{"version":1,"port":{port},"modules":{{}}}}"#).into_bytes()),
2597 )),
2598 modules: Default::default(),
2599 declarations: Default::default(),
2600 }),
2601 }
2602 }
2603
2604 fn approve_port(
2605 store: &mut crate::config_store::ConfigStore,
2606 kind: crate::config_store::CommitKind,
2607 port: u16,
2608 ) {
2609 store
2610 .commit_generation(
2611 approved_port_entry(kind, port),
2612 Default::default(),
2613 Default::default(),
2614 1,
2615 )
2616 .unwrap();
2617 }
2618
2619 fn stamped_boot(
2620 scratch: &Path,
2621 store: &mut crate::config_store::ConfigStore,
2622 ) -> BootstrapConfig {
2623 let mut config = BootstrapConfig::new(scratch.join("connection.json"), 1)
2624 .with_daemon_config_source(scratch.join("subc.jsonc"), Some(1));
2625 config.defer_file_load = true;
2626 load_bootstrap_authority(config, Some(store)).unwrap()
2627 }
2628
2629 #[test]
2630 fn config_stamped_boot_uses_last_active_generation_until_restart_intent() {
2631 use crate::config_store::{CommitKind, ConfigStore, StoreRoot};
2632 let scratch = unique_temp_dir("stamped-boot-selection");
2633 let root = scratch.join("store");
2634 std::fs::create_dir_all(&root).unwrap();
2635 let open = || {
2636 ConfigStore::open_digest_only_for_tests(
2637 StoreRoot::from_service_definition(root.clone()).unwrap(),
2638 )
2639 };
2640 let mut store = open();
2641 approve_port(&mut store, CommitKind::Stamp, 41001);
2642 approve_port(&mut store, CommitKind::Approved, 41002);
2643 drop(store);
2644 let mut store = open();
2645 assert_eq!(
2646 stamped_boot(&scratch, &mut store).port,
2647 41001,
2648 "an ordinary start must not boot an approval still waiting for a daemon restart"
2649 );
2650 assert_eq!(store.assign_daemon_restart().unwrap(), Some(2));
2651 drop(store);
2652 let mut store = open();
2653 assert_eq!(stamped_boot(&scratch, &mut store).port, 41002);
2654 }
2655
2656 #[test]
2657 fn config_stamped_boot_enters_recovery_when_anchor_retry_fails_at_start() {
2658 use crate::config_store::{CommitKind, ConfigStore, StoreRoot, StoreState, TestStoreKey};
2659 let scratch = unique_temp_dir("stamped-boot-anchor-pending");
2660 let root = scratch.join("store");
2661 std::fs::create_dir_all(&root).unwrap();
2662 let key = TestStoreKey::default();
2663 let open = || {
2664 ConfigStore::open_sealed_for_tests(
2665 StoreRoot::from_service_definition(root.clone()).unwrap(),
2666 key.clone(),
2667 )
2668 };
2669 let mut store = open();
2670 approve_port(&mut store, CommitKind::Stamp, 41001);
2671 key.fail_anchor(true);
2674 assert!(store
2675 .commit_generation(
2676 approved_port_entry(CommitKind::Approved, 41001),
2677 Default::default(),
2678 Default::default(),
2679 2,
2680 )
2681 .is_err());
2682 drop(store);
2683 let mut store = open();
2684 let status = store.status();
2685 assert_eq!(status.state, StoreState::Stamped);
2686 assert!(status.anchor_pending);
2687 let config = stamped_boot(&scratch, &mut store);
2688 assert_eq!(config.port, DEFAULT_SUBC_PORT);
2689 assert!(config.configured_modules.is_empty());
2690 assert_eq!(store.status().state, StoreState::StoreUnverified);
2691 }
2692
2693 #[test]
2694 fn normalized_build_provenance_omits_unavailable_and_empty_values() {
2695 assert_eq!(normalized_build_provenance("unavailable"), None);
2696 assert_eq!(normalized_build_provenance(""), None);
2697 }
2698
2699 #[cfg(target_os = "linux")]
2700 fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
2701 let cgroups = fs::read_to_string("/proc/self/cgroup")?;
2702 let relative = cgroups
2703 .lines()
2704 .find_map(|line| line.strip_prefix("0::"))
2705 .ok_or_else(|| {
2706 io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
2707 })?;
2708 Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
2709 }
2710
2711 #[cfg(target_os = "linux")]
2712 fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
2713 let modules = current_cgroup_path_for_test()?.join("subc-modules");
2714 let entries = match fs::read_dir(modules) {
2715 Ok(entries) => entries,
2716 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
2717 Err(error) => return Err(error),
2718 };
2719 let mut directories = BTreeSet::new();
2720 for entry in entries {
2721 let entry = entry?;
2722 if entry.file_type()?.is_dir() {
2723 directories.insert(entry.file_name());
2724 }
2725 }
2726 Ok(directories)
2727 }
2728
2729 #[cfg(target_os = "linux")]
2730 async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
2731 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2732 while !path.exists() && tokio::time::Instant::now() < deadline {
2733 assert!(
2734 !task.is_finished(),
2735 "daemon exited before creating {}",
2736 path.display()
2737 );
2738 sleep(Duration::from_millis(10)).await;
2739 }
2740 assert!(path.exists(), "daemon did not create {}", path.display());
2741 }
2742
2743 #[cfg(target_os = "linux")]
2748 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2749 async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
2750 let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
2751 let module_id = format!("cgroup-isolation-probe-{}", process::id());
2752 let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
2753 assert!(
2754 !before.contains(&OsString::from(&module_id)),
2755 "isolation probe cgroup already exists before this daemon starts"
2756 );
2757 let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
2758 let module = ConfiguredModule {
2759 script: None,
2760 carry_containment: false,
2761 module_id,
2762 program: PathBuf::from("sh"),
2763 args: vec!["-c".to_string(), "sleep 30".to_string()],
2764 env: Vec::new(),
2765 log: None,
2766 enabled: true,
2767 registration_channel: false,
2768 reserved: false,
2769 reserved_prefixes: Vec::new(),
2770 protocol: ModuleProtocol::None,
2771 overlap: Default::default(),
2772 health: HealthConfig::default(),
2773 drain_timeout_ms: None,
2774 register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2775 route_bind_relay_timeout_ms: None,
2776 restart: RestartPolicy::default(),
2777 };
2778 let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2779 .with_configured_modules([module])
2780 .with_capture_logs_dir(temp.join("logs"))
2781 .with_terminal_journal_path(temp.join("terminals.jsonl"));
2782 let task = tokio::spawn(run_with_config(config));
2783
2784 wait_for_path(&capture, &task).await;
2785 let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2786
2787 task.abort();
2788 assert!(task
2789 .await
2790 .expect_err("aborted daemon task must cancel")
2791 .is_cancelled());
2792 assert_eq!(
2793 before, after,
2794 "an in-process daemon must not create or reconcile ambient module cgroups"
2795 );
2796 }
2797
2798 #[cfg(target_os = "linux")]
2799 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2800 async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2801 let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2802 let cgroup_root = temp.join("cgroup");
2803 fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2804 fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2805 let modules = cgroup_root.join("subc-modules");
2806 let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2807 .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2808 .with_terminal_journal_path(temp.join("terminals.jsonl"));
2809 let task = tokio::spawn(run_with_config(config));
2810
2811 wait_for_path(&modules, &task).await;
2812
2813 task.abort();
2814 assert!(task
2815 .await
2816 .expect_err("aborted daemon task must cancel")
2817 .is_cancelled());
2818 assert!(
2819 fs::read_dir(&modules)
2820 .expect("read prepared modules directory")
2821 .next()
2822 .is_none(),
2823 "the delegation probe must clean up after itself"
2824 );
2825 }
2826
2827 struct EnvGuard {
2828 key: &'static str,
2829 previous: Option<OsString>,
2830 }
2831
2832 impl EnvGuard {
2833 fn set(key: &'static str, value: &Path) -> Self {
2834 let previous = env::var_os(key);
2835 env::set_var(key, value);
2836 Self { key, previous }
2837 }
2838
2839 fn set_str(key: &'static str, value: &str) -> Self {
2840 let previous = env::var_os(key);
2841 env::set_var(key, value);
2842 Self { key, previous }
2843 }
2844
2845 fn unset(key: &'static str) -> Self {
2846 let previous = env::var_os(key);
2847 env::remove_var(key);
2848 Self { key, previous }
2849 }
2850 }
2851
2852 impl Drop for EnvGuard {
2853 fn drop(&mut self) {
2854 match &self.previous {
2855 Some(value) => env::set_var(self.key, value),
2856 None => env::remove_var(self.key),
2857 }
2858 }
2859 }
2860
2861 fn unique_temp_dir(name: &str) -> ScratchDir {
2862 ScratchDir::new(name)
2863 }
2864
2865 fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2866 let dir = unique_temp_dir(name);
2867 let path = dir.join("conn.json");
2868 (dir, path)
2869 }
2870
2871 fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2872 ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2873 }
2874
2875 fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2876 let auth = auth_for(&bound.connection_info);
2877 tokio::spawn(serve_listeners(
2878 bound.listeners,
2879 Arc::new(Router::with_default_self_handler()),
2880 auth,
2881 ))
2882 }
2883
2884 fn expect_bound(outcome: Outcome) -> BoundDaemon {
2885 match outcome {
2886 Outcome::Bound(bound) => bound,
2887 Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2888 }
2889 }
2890
2891 async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2892 let endpoint = conn
2893 .endpoints
2894 .first()
2895 .expect("test connection file should have an endpoint");
2896 let ip: IpAddr = endpoint.host.parse().unwrap();
2897 TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2898 }
2899
2900 fn make_connection_info(port: u16) -> ConnectionInfo {
2901 ConnectionInfo {
2902 schema: SCHEMA_VERSION,
2903 wire_version: Some(PROTOCOL_VERSION),
2904 endpoints: vec![Endpoint {
2905 host: "127.0.0.1".to_owned(),
2906 port,
2907 }],
2908 key: generate_key().unwrap(),
2909 daemon_id: generate_daemon_id().unwrap(),
2910 pid: process::id(),
2911 daemon_ver: "test-subc".to_owned(),
2912 }
2913 }
2914
2915 fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2916 fs::write(path, contents).unwrap();
2917 #[cfg(unix)]
2918 fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2919 }
2920
2921 fn assert_owner_only_connection_file(path: &Path) {
2922 #[cfg(unix)]
2925 {
2926 let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2927 assert_eq!(mode, 0o600);
2928 }
2929 #[cfg(not(unix))]
2930 let _ = path;
2931 }
2932
2933 #[test]
2934 fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2935 let _env_lock = ENV_LOCK.lock().unwrap();
2936 let runtime_dir = unique_temp_dir("xdg-runtime");
2937 let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2938
2939 assert_eq!(
2940 connection_file_path(),
2941 runtime_dir.join(CONNECTION_FILE_NAME)
2942 );
2943 }
2944
2945 #[test]
2946 fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2947 let runtime_dir = OsString::from("/run/user/1000");
2948
2949 let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2950
2951 assert_eq!(
2952 path,
2953 PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2954 );
2955 assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2956 }
2957
2958 #[test]
2959 fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2960 let _env_lock = ENV_LOCK.lock().unwrap();
2961 let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2962
2963 assert_eq!(
2964 connection_file_path(),
2965 env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2966 );
2967 }
2968
2969 #[test]
2970 fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2971 let (path, source) = connection_file_path_with_source(None);
2972
2973 assert_eq!(
2974 path,
2975 env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2976 );
2977 assert_eq!(source, ConnectionFileSource::TempDirFallback);
2978 }
2979
2980 #[test]
2985 fn user_connection_token_is_stable_under_concurrent_callers() {
2986 let expected = user_connection_token();
2987 let workers: Vec<_> = (0..32)
2988 .map(|_| {
2989 std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2990 })
2991 .collect();
2992 for worker in workers {
2993 for token in worker.join().expect("probe thread") {
2994 assert_eq!(token, expected, "token diverged under concurrent probes");
2995 }
2996 }
2997 }
2998
2999 #[test]
3000 fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
3001 let (path, source) = connection_file_path_with_source(Some(OsString::new()));
3002
3003 assert_eq!(
3004 path,
3005 env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
3006 );
3007 assert_eq!(source, ConnectionFileSource::TempDirFallback);
3008 }
3009
3010 #[test]
3014 fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
3015 let _env_lock = ENV_LOCK.lock().unwrap();
3016 let root = unique_temp_dir("daemon-binary-config");
3017 let data_home = root.join("data");
3018 let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3019 let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3020 let _port = EnvGuard::unset(SUBC_PORT_ENV);
3021
3022 let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3023
3024 let run_dir = data_home.join("cortexkit").join("run");
3025 assert_eq!(
3026 config.terminal_journal_path,
3027 Some(run_dir.join("terminals.jsonl"))
3028 );
3029 assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
3030 assert_eq!(
3031 BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
3032 None,
3033 "an in-process config must not journal anywhere unless asked to"
3034 );
3035 }
3036
3037 #[test]
3038 fn daemon_binary_config_refuses_a_relative_data_home() {
3039 let _env_lock = ENV_LOCK.lock().unwrap();
3040 let root = unique_temp_dir("daemon-binary-relative-data");
3041 let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
3042 let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3043 let _port = EnvGuard::unset(SUBC_PORT_ENV);
3044
3045 let error = BootstrapConfig::from_env_for_daemon_binary()
3046 .expect_err("a relative data home must refuse the daemon binary's config");
3047 assert!(
3048 matches!(error, BootstrapError::RunDir(_)),
3049 "expected a run-directory refusal, got {error}"
3050 );
3051 assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
3052 }
3053
3054 #[test]
3055 fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
3056 let _env_lock = ENV_LOCK.lock().unwrap();
3057 let _port = EnvGuard::unset(SUBC_PORT_ENV);
3058 let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
3059 let config_path = conn_path.with_file_name("subc.jsonc");
3060 for (doc, expected) in [
3061 (None, daemon_config::PrivilegedOperatorPrompt::Log),
3062 (
3063 Some(r#"{"version":1}"#),
3064 daemon_config::PrivilegedOperatorPrompt::Log,
3065 ),
3066 (
3067 Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
3068 daemon_config::PrivilegedOperatorPrompt::Log,
3069 ),
3070 (
3071 Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
3072 daemon_config::PrivilegedOperatorPrompt::Enforce,
3073 ),
3074 ] {
3075 if let Some(doc) = doc {
3076 fs::write(&config_path, doc).unwrap();
3077 }
3078 assert_eq!(
3079 BootstrapConfig::from_env_with_daemon_config_path(&config_path)
3080 .unwrap()
3081 .privileged_operator_prompt,
3082 expected
3083 );
3084 assert_eq!(
3085 BootstrapConfig::new(&conn_path, 0)
3086 .with_daemon_config_path(&config_path)
3087 .unwrap()
3088 .privileged_operator_prompt,
3089 expected
3090 );
3091 }
3092 }
3093
3094 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3095 async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
3096 let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
3097 let connection_path = dir.join("connection.json");
3098 let config_path = dir.join("subc.jsonc");
3099 fs::write(
3100 &config_path,
3101 r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
3102 )
3103 .unwrap();
3104 let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
3105 .with_daemon_config_path(&config_path)
3106 .unwrap();
3107 let connection_path = config.connection_file_path.clone();
3108 let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
3109 tokio::time::timeout(Duration::from_secs(5), async {
3110 while !connection_path.exists() {
3111 assert!(
3112 !daemon.handle.is_finished(),
3113 "daemon exited before publishing"
3114 );
3115 tokio::task::yield_now().await;
3116 }
3117 })
3118 .await
3119 .unwrap();
3120 let info = connection_file::read(&connection_path).unwrap();
3121 let mut client = connect_from_info(&info).await.unwrap();
3122 authenticate_client(&mut client, &info, Duration::from_secs(5))
3123 .await
3124 .unwrap();
3125 for (correlation, mode, expected) in [
3126 (1, "enforce", Vec::new()),
3127 (2, "log", vec!["privileged_operator_prompt"]),
3128 (3, "log", vec!["privileged_operator_prompt"]),
3129 ] {
3130 fs::write(
3131 &config_path,
3132 serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
3133 )
3134 .unwrap();
3135 let frame = crate::Frame::build(
3136 subc_protocol::FrameType::Request,
3137 subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
3138 0,
3139 0,
3140 correlation,
3141 serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
3142 preview: true,
3143 })
3144 .unwrap(),
3145 )
3146 .unwrap();
3147 crate::write_frame(&mut client, &frame).await.unwrap();
3148 let reply =
3149 tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
3150 .await
3151 .unwrap()
3152 .unwrap()
3153 .unwrap();
3154 assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
3155 assert_eq!(reply.header.corr, correlation);
3156 let subc_control::ClientControlResponse::SupervisorRescan { result } =
3157 serde_json::from_slice(&reply.body).unwrap()
3158 else {
3159 panic!("expected a rescan response");
3160 };
3161 assert_eq!(result.restart_required, expected);
3162 }
3163 }
3164
3165 #[test]
3166 fn configured_port_uses_default_config_and_env_override() {
3167 let _env_lock = ENV_LOCK.lock().unwrap();
3168 let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
3169 let config_path = conn_path.with_file_name("subc.jsonc");
3170
3171 let _port = EnvGuard::unset(SUBC_PORT_ENV);
3172 assert_eq!(
3173 BootstrapConfig::from_env_with_daemon_config_path(&config_path)
3174 .unwrap()
3175 .port,
3176 DEFAULT_SUBC_PORT
3177 );
3178
3179 fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
3180 assert_eq!(
3181 BootstrapConfig::from_env_with_daemon_config_path(&config_path)
3182 .unwrap()
3183 .port,
3184 8123
3185 );
3186
3187 let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
3188 assert_eq!(
3189 BootstrapConfig::from_env_with_daemon_config_path(&config_path)
3190 .unwrap()
3191 .port,
3192 9012
3193 );
3194 }
3195
3196 #[tokio::test]
3197 async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
3198 let (_dir, path) = temp_connection_file_path("departing-peer");
3199 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3200 let info = make_connection_info(listener.local_addr().unwrap().port());
3201 write_connection_file(&path, &info, None).unwrap();
3202 let peer = tokio::spawn(async move {
3203 let (mut stream, _) = listener.accept().await.unwrap();
3204 subc_transport::authenticate_server(
3205 &mut stream,
3206 &info.key,
3207 &info.daemon_id,
3208 &info.daemon_ver,
3209 PROBE_AUTH_DEADLINE,
3210 )
3211 .await
3212 .unwrap();
3213 });
3214 let outcome = ensure_singleton(&path, 0).await.unwrap();
3215 peer.await.unwrap();
3216 assert!(
3217 matches!(outcome, Outcome::Bound(_)),
3218 "a departing peer is not AlreadyRunning"
3219 );
3220 }
3221
3222 #[tokio::test]
3223 async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
3224 let (_dir, path) = temp_connection_file_path("already-running");
3225
3226 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3227 let server = start_server(bound);
3228
3229 let second = ensure_singleton(&path, 0).await.unwrap();
3230 assert!(matches!(second, Outcome::AlreadyRunning));
3231
3232 server.abort();
3233 let _ = server.await;
3234 }
3235
3236 #[tokio::test]
3237 async fn daemon_connection_file_publishes_protocol_wire_version() {
3238 let (_dir, path) = temp_connection_file_path("wire-version");
3239 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3240 assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
3241 assert_eq!(
3242 connection_file::read(&path).unwrap().wire_version,
3243 Some(PROTOCOL_VERSION)
3244 );
3245
3246 drop(bound.listeners);
3247 }
3248
3249 #[tokio::test]
3250 async fn stale_unbound_connection_file_is_reclaimed() {
3251 let (_dir, path) = temp_connection_file_path("stale-reclaim");
3252 let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
3253 let stale_port = stale.local_addr().unwrap().port();
3254 drop(stale);
3255 let stale_info = make_connection_info(stale_port);
3256 write_atomic(&path, &stale_info).unwrap();
3257
3258 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3259 assert_ne!(bound.connection_info.key, stale_info.key);
3260 drop(bound.listeners);
3261 }
3262
3263 #[cfg(unix)]
3264 #[tokio::test]
3265 async fn ensure_singleton_reclaims_insecure_connection_file() {
3266 let (_dir, path) = temp_connection_file_path("insecure-reclaim");
3267 let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
3268 let stale_port = stale.local_addr().unwrap().port();
3269 drop(stale);
3270 let stale_info = make_connection_info(stale_port);
3271 write_atomic(&path, &stale_info).unwrap();
3272 fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
3273
3274 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3275 assert_ne!(bound.connection_info.key, stale_info.key);
3276 assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
3277 assert_owner_only_connection_file(&path);
3278
3279 drop(bound.listeners);
3280 }
3281
3282 #[tokio::test]
3283 async fn ensure_singleton_reclaims_non_loopback_connection_file() {
3284 let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
3285 let mut stale_info = make_connection_info(8757);
3286 stale_info.endpoints = vec![Endpoint {
3287 host: "192.0.2.10".to_owned(),
3288 port: 8757,
3289 }];
3290 write_atomic(&path, &stale_info).unwrap();
3291
3292 let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
3293 let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3294 assert_ne!(bound.connection_info.key, stale_info.key);
3295 assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
3296 assert!(bound
3297 .connection_info
3298 .endpoints
3299 .iter()
3300 .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
3301 assert_owner_only_connection_file(&path);
3302
3303 drop(bound.listeners);
3304 }
3305
3306 #[tokio::test]
3307 async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
3308 let mut unsupported_schema = make_connection_info(8757);
3309 unsupported_schema.schema = SCHEMA_VERSION + 1;
3310
3311 let mut empty_endpoints = make_connection_info(8757);
3312 empty_endpoints.endpoints.clear();
3313
3314 let mut short_key = make_connection_info(8757);
3315 short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
3316
3317 let cases = vec![
3318 (
3319 "unsupported-schema",
3320 serde_json::to_vec(&unsupported_schema).unwrap(),
3321 Some(unsupported_schema),
3322 ),
3323 (
3324 "empty-endpoints",
3325 serde_json::to_vec(&empty_endpoints).unwrap(),
3326 Some(empty_endpoints),
3327 ),
3328 (
3329 "short-key",
3330 serde_json::to_vec(&short_key).unwrap(),
3331 Some(short_key),
3332 ),
3333 ("invalid-json", b"{not valid connection json".to_vec(), None),
3334 ];
3335
3336 for (label, contents, old_info) in cases {
3337 let (_dir, path) = temp_connection_file_path(label);
3338 write_raw_owner_only_connection_file(&path, &contents);
3339
3340 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3341 if let Some(old_info) = old_info {
3342 assert_ne!(bound.connection_info.key, old_info.key, "{label}");
3343 assert_ne!(
3344 bound.connection_info.daemon_id, old_info.daemon_id,
3345 "{label}"
3346 );
3347 }
3348 assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
3349 assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
3350 assert_owner_only_connection_file(&path);
3351
3352 drop(bound.listeners);
3353 }
3354 }
3355
3356 #[tokio::test]
3357 async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
3358 let (_dir, path) = temp_connection_file_path("foreign-reclaim");
3359 let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
3360 let foreign_port = foreign.local_addr().unwrap().port();
3361 write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
3362 let foreign_task = tokio::spawn(async move {
3363 if let Ok((mut stream, _)) = foreign.accept().await {
3364 let mut buf = [0u8; 64];
3365 let _ = stream.read(&mut buf).await;
3366 }
3367 });
3368
3369 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3370 assert!(bound
3371 .connection_info
3372 .endpoints
3373 .iter()
3374 .all(|endpoint| endpoint.port != foreign_port));
3375
3376 drop(bound.listeners);
3377 let _ = foreign_task.await;
3378 }
3379
3380 #[tokio::test]
3381 async fn stale_start_lock_file_is_reclaimable() {
3382 let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
3383 let lock_path = start_lock_path(&path);
3384 drop(open_owner_only_lock(&lock_path).unwrap());
3385 assert!(lock_path.is_file());
3386
3387 let lock = StartLock::acquire(&path).await.unwrap();
3388 assert!(lock_path.is_file());
3389
3390 drop(lock);
3391 assert!(lock_path.is_file());
3392 }
3393
3394 #[tokio::test]
3395 async fn held_start_lock_blocks_second_acquire_until_release() {
3396 let (_dir, path) = temp_connection_file_path("start-lock-held");
3397 let lock_path = start_lock_path(&path);
3398 let first = StartLock::acquire(&path).await.unwrap();
3399
3400 let err = match StartLock::acquire(&path).await {
3401 Ok(_) => panic!("second acquire while held must stay busy"),
3402 Err(err) => err,
3403 };
3404 assert!(matches!(
3405 err,
3406 BootstrapError::StartLockBusy {
3407 ref path,
3408 attempts: START_LOCK_RETRIES,
3409 } if path == &lock_path
3410 ));
3411
3412 drop(first);
3413
3414 let second = StartLock::acquire(&path)
3415 .await
3416 .expect("released advisory lock should be reclaimable");
3417 drop(second);
3418 }
3419
3420 #[tokio::test]
3421 async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
3422 let (_dir, path) = temp_connection_file_path("bind-conflict");
3423 let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
3424 let occupied_port = occupied.local_addr().unwrap().port();
3425
3426 let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
3427 assert!(matches!(
3428 err,
3429 BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
3430 ));
3431 assert!(err.to_string().contains("set the port in config"));
3432
3433 drop(occupied);
3434 }
3435
3436 #[tokio::test]
3437 async fn ephemeral_ipv6_collision_still_serves_ipv4() {
3438 let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
3439 let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
3440 let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
3441 drop(occupied);
3442 result
3443 })
3444 .await
3445 .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
3446 assert_eq!(endpoints.len(), 1);
3447 assert_eq!(endpoints[0].host, "127.0.0.1");
3448 TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
3449 .await
3450 .unwrap();
3451 drop(listeners);
3452 }
3453
3454 #[tokio::test]
3455 async fn unpublished_boot_holds_singleton_lock_until_publication() {
3456 let dir = ScratchDir::new("ul");
3457 let path = dir.join("c");
3458 let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
3459 let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
3460 assert!(!path.exists());
3461 let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
3462 assert!(
3463 matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
3464 "an unpublished boot must keep its singleton claim"
3465 );
3466 drop(bound);
3467 FileExt::try_lock(&contender).unwrap();
3468 }
3469
3470 #[tokio::test]
3471 async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
3472 let dir = ScratchDir::new("sl");
3473 let path = dir.join("c");
3474 let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
3475 let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
3476 let inherited = bound
3479 .publication_lock
3480 .as_ref()
3481 .unwrap()
3482 ._file
3483 .try_clone()
3484 .unwrap();
3485 let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
3486 assert!(matches!(
3487 FileExt::try_lock(&contender),
3488 Err(TryLockError::WouldBlock)
3489 ));
3490 drop(bound);
3491 FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
3492 drop(inherited);
3493 }
3494
3495 #[tokio::test]
3496 async fn readiness_probe_requires_an_authenticating_server() {
3497 let dir = ScratchDir::new("unserved-readiness");
3498 let bound = expect_bound(
3499 ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
3500 .await
3501 .unwrap(),
3502 );
3503 assert!(matches!(
3504 verify_serving(&bound.connection_info).await,
3505 Err(BootstrapError::StartupNotServing)
3506 ));
3507 }
3508
3509 #[cfg(unix)]
3510 #[tokio::test]
3511 async fn orphan_sweep_finishes_before_connection_file_publication() {
3512 use crate::live_children::{ExecutableIdentity, LiveChild};
3513 let dir = ScratchDir::new("pas");
3514 let path = dir.join("c");
3518 let record = dir.join("live-children.json");
3519 let ready = dir.join("ready");
3520 let mut child = std::process::Command::new("/bin/sh")
3521 .args([
3522 "-c",
3523 "trap '' TERM; : > \"$0\"; read -r _",
3524 ready.to_str().unwrap(),
3525 ])
3526 .stdin(std::process::Stdio::piped())
3527 .env("XDG_DATA_HOME", dir.path())
3528 .env("XDG_RUNTIME_DIR", dir.path())
3529 .env("XDG_CONFIG_HOME", dir.path())
3530 .spawn()
3531 .unwrap();
3532 timeout(Duration::from_secs(5), async {
3533 while !ready.exists() {
3534 sleep(Duration::from_millis(5)).await;
3535 }
3536 })
3537 .await
3538 .unwrap();
3539 let observed = subc_os::Process::open(child.id())
3540 .unwrap()
3541 .unwrap()
3542 .observe()
3543 .unwrap();
3544 crate::live_children::write_record(
3545 &record,
3546 &[LiveChild {
3547 container: None,
3548 module_id: "old-sleep".into(),
3549 pid: child.id(),
3550 protocol: subc_control::ModuleProtocol::None,
3551 start_time: Some(observed.start_time),
3552 executable: observed.executable.map(ExecutableIdentity::from),
3553 cgroup_name: None,
3554 }],
3555 )
3556 .unwrap();
3557 let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
3558 .with_live_children_record(&record);
3559 let daemon = tokio::spawn(run_with_config(config));
3560 timeout(Duration::from_secs(5), async {
3561 while !path.exists() {
3562 sleep(Duration::from_millis(5)).await;
3563 }
3564 })
3565 .await
3566 .unwrap();
3567 assert!(path.exists());
3568 let ended = child.try_wait().unwrap().is_some();
3569 if !ended {
3570 child.kill().unwrap();
3571 }
3572 child.wait().unwrap();
3573 if ended {
3574 assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
3575 }
3576 daemon.abort();
3577 let _ = daemon.await;
3578 assert!(
3579 ended,
3580 "the published daemon must not still owe its orphan sweep"
3581 );
3582 }
3583
3584 #[tokio::test(flavor = "current_thread")]
3591 async fn connection_file_publication_leaves_the_async_runtime_free() {
3592 use std::sync::atomic::{AtomicBool, Ordering};
3593
3594 let (_dir, path) = temp_connection_file_path("publish-off-runtime");
3595 let write_started = Arc::new(AtomicBool::new(false));
3596 let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
3597 let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
3598 let hook_started = Arc::clone(&write_started);
3599 connection_file_write_hook::install(&path, move || {
3600 hook_started.store(true, Ordering::SeqCst);
3601 let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
3602 let _ = verdict_tx.send(progressed);
3603 });
3604 let ticker = tokio::spawn(async move {
3607 loop {
3608 if write_started.load(Ordering::SeqCst) {
3609 let _ = progress_tx.send(());
3610 return;
3611 }
3612 sleep(Duration::from_millis(5)).await;
3613 }
3614 });
3615
3616 let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
3617 let published = timeout(Duration::from_secs(15), async {
3618 while !path.exists() && !daemon.is_finished() {
3619 sleep(Duration::from_millis(5)).await;
3620 }
3621 })
3622 .await;
3623 let daemon_exited = daemon.is_finished();
3624 daemon.abort();
3625 let daemon_result = daemon.await;
3626 ticker.abort();
3627
3628 assert!(
3629 published.is_ok() && !daemon_exited,
3630 "the daemon must start and publish its connection file: {daemon_result:?}"
3631 );
3632 match verdict_rx.try_recv() {
3633 Ok(true) => {}
3634 Ok(false) => panic!(
3635 "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
3636 ),
3637 Err(_) => panic!("startup published the connection file without the hooked write"),
3638 }
3639 }
3640
3641 #[tokio::test]
3644 async fn a_panicking_connection_file_write_fails_startup() {
3645 let (_dir, path) = temp_connection_file_path("publish-panics");
3646 connection_file_write_hook::install(&path, || {
3647 panic!("deliberate panic inside the connection-file write")
3648 });
3649
3650 let error = timeout(
3651 Duration::from_secs(15),
3652 run_with_config(BootstrapConfig::new(&path, 0)),
3653 )
3654 .await
3655 .expect("startup must end once the write task fails")
3656 .expect_err("startup must fail when the connection-file write panics");
3657
3658 assert!(
3659 matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
3660 "expected a connection-file write error, got {error:?}"
3661 );
3662 let message = error.to_string();
3663 assert!(message.contains(&path.display().to_string()), "{message}");
3664 assert!(message.contains("panicked"), "{message}");
3665 assert!(!path.exists(), "no connection file may be published");
3666 }
3667
3668 #[cfg(target_os = "macos")]
3669 #[test]
3670 fn nofile_raise_worker() {
3671 let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
3672 return;
3673 };
3674 let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
3675 rlimit::Resource::NOFILE.set(256, hard).unwrap();
3676 raise_nofile_limit_with_ceiling(Some(10240));
3677 std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
3678 }
3679
3680 #[cfg(target_os = "macos")]
3681 #[test]
3682 fn macos_nofile_raise_clamps_to_kernel_ceiling() {
3683 let dir = ScratchDir::new("nofile-ceiling");
3684 let result = dir.join("soft");
3685 let status = std::process::Command::new(std::env::current_exe().unwrap())
3686 .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
3687 .env("SUBC_NOFILE_TEST_FILE", &result)
3688 .env("XDG_DATA_HOME", dir.path())
3689 .env("XDG_RUNTIME_DIR", dir.path())
3690 .env("XDG_CONFIG_HOME", dir.path())
3691 .status()
3692 .unwrap();
3693 assert!(status.success());
3694 let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
3695 assert!(
3696 soft > 256 && soft <= 10240,
3697 "the inherited limit must be raised within the kernel ceiling; got {soft}"
3698 );
3699 }
3700
3701 #[tokio::test]
3702 async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
3703 let (_dir, path) = temp_connection_file_path("key-rotation");
3704 let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3705 let old_info = first.connection_info.clone();
3706 let fixed_port = old_info.endpoints[0].port;
3707 drop(first.listeners);
3708
3709 let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
3710 let new_info = second.connection_info.clone();
3711 assert_ne!(old_info.key, new_info.key);
3712 assert_ne!(old_info.daemon_id, new_info.daemon_id);
3713 let server = start_server(second);
3714
3715 let mut old_stream = connect_from_info(&old_info).await.unwrap();
3716 let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
3717 assert!(
3718 old_auth.is_err(),
3719 "old key must not authenticate after restart"
3720 );
3721
3722 let reread = connection_file::read(&path).unwrap();
3723 let mut new_stream = connect_from_info(&reread).await.unwrap();
3724 authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
3725 .await
3726 .unwrap();
3727
3728 server.abort();
3729 let _ = server.await;
3730 }
3731
3732 #[cfg(unix)]
3733 #[tokio::test]
3734 async fn published_connection_file_permissions_are_owner_only() {
3735 let (_dir, path) = temp_connection_file_path("permissions");
3736 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3737
3738 let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
3739 assert_eq!(mode, 0o600);
3740
3741 drop(bound.listeners);
3742 }
3743
3744 fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
3752 let dir = config
3753 .connection_file_path
3754 .parent()
3755 .map(Path::to_path_buf)
3756 .unwrap_or_else(|| PathBuf::from("."));
3757 #[cfg(unix)]
3758 let config = BootstrapConfig {
3759 connection_file_path: dir.join("c"),
3760 local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
3761 ..config
3762 };
3763 #[cfg(not(unix))]
3764 let _ = dir;
3765 config
3766 }
3767
3768 #[cfg(unix)]
3771 #[test]
3772 fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
3773 let _env_lock = ENV_LOCK.lock().unwrap();
3774 let root = ScratchDir::new("binary-local-socket");
3775 let data_home = root.join("data");
3776 let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3777 let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3778 let _port = EnvGuard::unset(SUBC_PORT_ENV);
3779
3780 let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3781
3782 let run_dir = data_home.join("cortexkit").join("run");
3783 assert_eq!(
3784 config.local_listener.configured_dir(),
3785 Some(run_dir.as_path()),
3786 "the binary serves from its run directory"
3787 );
3788 let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
3789 .expect("the binary config names a directory");
3790 assert_eq!(socket.parent().unwrap(), run_dir);
3791 assert_eq!(
3792 socket.file_name().unwrap().to_string_lossy(),
3793 "subc.sock",
3794 "a fixed name: the run directory belongs to one data home"
3795 );
3796 }
3797
3798 #[cfg(unix)]
3805 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3806 async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
3807 let root = ScratchDir::new("in-process-no-local-socket");
3808 let data_home = root.join("data");
3809 fs::create_dir_all(&data_home).unwrap();
3810 let connection_path = root.join("c");
3811 let _data;
3812 let _config;
3813 let _port;
3814 {
3815 let _env_lock = ENV_LOCK.lock().unwrap();
3816 _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3817 _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3818 _port = EnvGuard::unset(SUBC_PORT_ENV);
3819 }
3820
3821 let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
3822
3823 assert!(bound.local_endpoint.is_none());
3824 assert!(bound.local_endpoint_unavailable.is_none());
3825 let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3826 assert!(published.local_endpoint.is_none());
3827 assert!(
3828 local_listener::socket_path_for(None, &connection_path).is_none(),
3829 "no configured directory, no socket"
3830 );
3831 let mut leftovers = Vec::new();
3832 let mut pending = vec![data_home.clone()];
3833 while let Some(dir) = pending.pop() {
3834 for entry in fs::read_dir(&dir).unwrap() {
3835 let entry = entry.unwrap();
3836 let path = entry.path();
3837 if entry.file_type().unwrap().is_dir() {
3838 pending.push(path.clone());
3839 }
3840 if entry.file_name().to_string_lossy().ends_with(".sock") {
3841 leftovers.push(path);
3842 }
3843 }
3844 }
3845 assert!(
3846 leftovers.is_empty(),
3847 "an in-process daemon must not create a socket in the data home: {leftovers:?}"
3848 );
3849 }
3850
3851 #[cfg(unix)]
3856 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3857 async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
3858 let dir = ScratchDir::new("tcp-only");
3859 let canonical = std::fs::canonicalize(&dir).unwrap();
3860 let connection_path = canonical.join("c");
3861 let deep = canonical.join("x".repeat(80));
3864 fs::create_dir(&deep).unwrap();
3865 fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
3866 let config =
3867 BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
3868 let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3869 assert!(
3870 bound.local_endpoint.is_none(),
3871 "a socket that cannot be bound is not published"
3872 );
3873 assert!(
3874 bound
3875 .local_endpoint_unavailable
3876 .as_deref()
3877 .is_some_and(|reason| reason.contains("too long")),
3878 "{:?}",
3879 bound.local_endpoint_unavailable
3880 );
3881 let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3882 assert!(published.local_endpoint.is_none());
3883 assert!(
3884 !published.info.endpoints.is_empty(),
3885 "TCP is still published"
3886 );
3887
3888 let server = start_server(bound);
3890 let info = connection_file::read(&connection_path).unwrap();
3891 let mut client = connect_from_info(&info).await.unwrap();
3892 authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
3893 .await
3894 .expect("a TCP client still completes the handshake");
3895 server.abort();
3896 let _ = server.await;
3897 }
3898
3899 #[cfg(unix)]
3902 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3903 async fn published_connection_file_names_the_served_local_endpoint() {
3904 let (_dir, path) = temp_connection_file_path("local-endpoint");
3905 let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
3909 let config =
3910 BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
3911 let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3912 let endpoint = connection_file::read_with_local_endpoint(&path)
3913 .unwrap()
3914 .local_endpoint
3915 .expect("the daemon serves a local endpoint, so the file names it");
3916 assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
3917 let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
3918 panic!("a Unix daemon publishes a socket path");
3919 };
3920 assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
3921 assert_eq!(
3922 fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
3923 0o600
3924 );
3925
3926 let router = Arc::new(Router::with_default_self_handler());
3927 let auth = auth_for(&bound.connection_info);
3928 let server = tokio::spawn(serve_local_listener(
3929 bound.local_listener.expect("the local listener is bound"),
3930 router,
3931 auth,
3932 ));
3933 let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
3934 authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
3935 .await
3936 .expect("a socket client completes the same handshake as a TCP client");
3937
3938 server.abort();
3939 drop(bound.listeners);
3940 }
3941
3942 #[cfg(unix)]
3945 #[test]
3946 fn a_socket_directory_that_is_a_file_refuses() {
3947 let dir = ScratchDir::new("local-listener-file-as-dir");
3948 let file = dir.join("not-a-directory");
3949 fs::write(&file, b"x").unwrap();
3950 let error = local_listener::bind(
3951 &LocalListenerConfig::in_socket_dir(file),
3952 &dir.join("connection.json"),
3953 )
3954 .expect_err("a path that is not a directory cannot hold the socket");
3955 assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
3956 }
3957
3958 #[cfg(unix)]
3962 #[test]
3963 fn the_socket_path_stays_within_the_unix_limit() {
3964 let run_dir = PathBuf::from("/tmp/short-run");
3965 assert!(
3966 local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
3967 "no configured directory means no socket"
3968 );
3969 let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
3970 .expect("a configured directory gets a socket");
3971 assert_eq!(socket, run_dir.join("subc.sock"));
3972 assert_eq!(
3973 socket,
3974 local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
3975 "the name does not come from the connection file"
3976 );
3977
3978 let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
3979 let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
3980 .expect("a configured directory gets a socket");
3981 assert!(
3982 long.as_os_str().len() < 104,
3983 "{} is {} bytes",
3984 long.display(),
3985 long.as_os_str().len()
3986 );
3987 }
3988}