1use std::{
2 collections::BTreeMap,
3 env,
4 error::Error,
5 ffi::OsString,
6 fmt, fs, io,
7 net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8 path::{Path, PathBuf},
9 process,
10 time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17 authenticate_client, connection_file, generate_daemon_id, generate_key,
18 write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
19 LocalEndpoint, SCHEMA_VERSION,
20};
21use tokio::{
22 net::{TcpListener, TcpStream},
23 task::{JoinError, JoinHandle},
24 time::{sleep, timeout},
25};
26use tracing::{error, info, warn};
27
28use crate::{
29 daemon_config::{self, ConfiguredModule, DaemonConfigError},
30 local_listener::{self, BoundLocalListener, LocalListenerConfig},
31 server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
32 supervise::HealthConfig,
33 ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
34 ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
35 SupervisorProcessLiveness,
36};
37use std::sync::Arc;
38
39#[cfg(windows)]
40mod windows_connection_copy;
41
42pub const DEFAULT_SUBC_PORT: u16 = 8757;
43pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
44use subc_transport::CONNECTION_FILE_NAME;
45const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
46const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
47const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
48const START_LOCK_RETRIES: usize = 40;
49const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum ConnectionFileSource {
53 XdgRuntimeDir,
54 TempDirFallback,
55 Explicit,
56}
57
58impl ConnectionFileSource {
59 fn reason(self) -> &'static str {
60 match self {
61 Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
62 Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
63 Self::Explicit => "configured path",
64 }
65 }
66}
67
68impl fmt::Display for ConnectionFileSource {
69 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
70 formatter.write_str(match self {
71 Self::XdgRuntimeDir => "xdg_runtime_dir",
72 Self::TempDirFallback => "temp_dir_fallback",
73 Self::Explicit => "explicit",
74 })
75 }
76}
77
78#[derive(Debug, Clone, Default)]
82struct AdmissionFactsConfig {
83 carrier_module_id: Option<String>,
84 targets: Option<Vec<String>>,
85}
86
87#[derive(Debug, Clone, Default, PartialEq, Eq)]
96pub enum CgroupPlacementConfig {
97 #[default]
98 Disabled,
99 Current,
100 Root(PathBuf),
101}
102
103#[derive(Debug, Clone)]
104pub struct BootstrapConfig {
105 #[cfg(feature = "test-support")]
106 shutdown_intent_window: Duration,
107 pub operator_queue_wait: Duration,
108 pub operator_timeout: Duration,
109 pub operator_stuck_grace: Duration,
110 #[cfg(feature = "test-support")]
111 pub operator_script: Option<PathBuf>,
112 #[cfg(feature = "test-support")]
113 pub operator_events: Option<PathBuf>,
114 pub connection_file_path: PathBuf,
115 pub port: u16,
116 pub daemon_ver: String,
117 configured_modules: Vec<ConfiguredModule>,
118 storage_config: Option<daemon_config::StorageConfig>,
119 admission_facts: AdmissionFactsConfig,
120 scope_authority_owners: Vec<String>,
122 federated_scope_owner: String,
123 privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
124 activity_config: daemon_config::ActivityConfig,
125 daemon_config_path: Option<PathBuf>,
126 configured_port: Option<u16>,
127 route_bind_relay_default_ms: Option<u64>,
131 reserved_capabilities: BTreeMap<String, String>,
132 watchdog_config: DaemonSelfWatchdogConfig,
133 connection_file_source: ConnectionFileSource,
134 #[cfg(all(test, windows))]
135 profile_override: Option<PathBuf>,
136 cgroup_placement: CgroupPlacementConfig,
139 capture_logs_dir: Option<PathBuf>,
143 terminal_journal_path: Option<PathBuf>,
151 machine_id_path: Option<PathBuf>,
157
158 local_listener: LocalListenerConfig,
165 live_children_path: Option<PathBuf>,
173 privacy_trampoline: Option<PathBuf>,
174}
175
176impl BootstrapConfig {
177 #[cfg(feature = "test-support")]
179 pub fn with_shutdown_intent_window(mut self, window: Duration) -> Self {
180 self.shutdown_intent_window = window;
181 self
182 }
183
184 pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
185 Self {
186 #[cfg(feature = "test-support")]
187 shutdown_intent_window: Duration::from_secs(120),
188 connection_file_path: connection_file_path.into(),
189 operator_queue_wait: Duration::from_secs(150),
190 operator_timeout: Duration::from_secs(120),
191 operator_stuck_grace: Duration::from_secs(10),
192 #[cfg(feature = "test-support")]
193 operator_script: None,
194 #[cfg(feature = "test-support")]
195 operator_events: None,
196 port,
197 daemon_ver: DAEMON_VERSION.to_owned(),
198 configured_modules: Vec::new(),
199 storage_config: None,
200 admission_facts: AdmissionFactsConfig::default(),
201 scope_authority_owners: daemon_config::default_scope_authority_owners(),
202 federated_scope_owner: daemon_config::default_federated_scope_owner(),
203 privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
204 activity_config: daemon_config::ActivityConfig::default(),
205 daemon_config_path: None,
206 configured_port: None,
207 route_bind_relay_default_ms: None,
208 reserved_capabilities: BTreeMap::new(),
209 watchdog_config: DaemonSelfWatchdogConfig::default(),
210 connection_file_source: ConnectionFileSource::Explicit,
211 #[cfg(all(test, windows))]
212 profile_override: None,
213 cgroup_placement: CgroupPlacementConfig::default(),
214 capture_logs_dir: None,
215 terminal_journal_path: None,
216 machine_id_path: None,
217 local_listener: LocalListenerConfig { socket_dir: None },
218 live_children_path: None,
219 privacy_trampoline: None,
220 }
221 }
222
223 pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
226 self.local_listener = LocalListenerConfig::in_socket_dir(dir);
227 self
228 }
229
230 pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
232 self.local_listener = config;
233 self
234 }
235
236 pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
240 self.live_children_path = Some(path.into());
241 self
242 }
243
244 pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
247 self.privacy_trampoline = Some(path.into());
248 self
249 }
250
251 pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
255 self.machine_id_path = Some(path.into());
256 self
257 }
258
259 pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
261 self.cgroup_placement = placement;
262 self
263 }
264
265 pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
269 self.capture_logs_dir = Some(dir.into());
270 self
271 }
272
273 pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
276 self.terminal_journal_path = Some(path.into());
277 self
278 }
279
280 pub fn from_env() -> Result<Self, BootstrapError> {
281 Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
282 }
283
284 pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
293 let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
294 let machine_id_path =
295 crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
296 let config = Self::from_env()?
297 .with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
298 .with_capture_logs_dir(run_dir.join("logs"))
299 .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
300 .with_live_children_record(crate::live_children::record_path(&run_dir))
301 .with_machine_id_path(machine_id_path);
302 #[cfg(feature = "test-support")]
303 let config = {
304 let mut config = config;
305 config.operator_script =
306 std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
307 config.operator_events =
308 std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
309 for (name, value) in [
310 (
311 "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
312 &mut config.operator_queue_wait,
313 ),
314 (
315 "SUBC_TEST_OPERATOR_TIMEOUT_MS",
316 &mut config.operator_timeout,
317 ),
318 (
319 "SUBC_TEST_OPERATOR_STUCK_MS",
320 &mut config.operator_stuck_grace,
321 ),
322 ] {
323 if let Ok(ms) = std::env::var(name) {
324 if let Ok(ms) = ms.parse::<u64>() {
325 *value = Duration::from_millis(ms);
326 }
327 }
328 }
329 config
330 };
331 Ok(config)
332 }
333
334 pub fn from_env_with_daemon_config_path(
335 daemon_config_path: impl AsRef<Path>,
336 ) -> Result<Self, BootstrapError> {
337 let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
338 let (daemon_config, activity_config) =
339 daemon_config::load_with_activity(&daemon_config_path)
340 .map_err(BootstrapError::DaemonConfig)?
341 .map(|(config, activity)| (Some(config), activity))
342 .unwrap_or_default();
343 let config_port = daemon_config.as_ref().and_then(|config| config.port);
344 let storage_config = daemon_config
345 .as_ref()
346 .and_then(|config| config.storage.clone());
347 let admission_facts_carrier_module_id = daemon_config
348 .as_ref()
349 .and_then(|config| config.admission_facts_carrier_module_id.clone());
350 let admission_facts_targets = daemon_config
351 .as_ref()
352 .and_then(|config| config.admission_facts_targets.clone());
353 let scope_authority_owners = daemon_config
354 .as_ref()
355 .map(|config| config.scope_authority_owners.clone())
356 .unwrap_or_else(daemon_config::default_scope_authority_owners);
357 let federated_scope_owner = daemon_config
358 .as_ref()
359 .map(|config| config.federated_scope_owner.clone())
360 .unwrap_or_else(daemon_config::default_federated_scope_owner);
361 let privileged_operator_prompt = daemon_config
362 .as_ref()
363 .map(|config| config.privileged_operator_prompt)
364 .unwrap_or_default();
365 let route_bind_relay_default_ms = daemon_config
366 .as_ref()
367 .and_then(|config| config.route_bind_relay_timeout_ms);
368 let reserved_capabilities = daemon_config
369 .as_ref()
370 .map(|config| config.reserved_capabilities.clone())
371 .unwrap_or_default();
372 let configured_modules = daemon_config
373 .map(|config| config.modules)
374 .unwrap_or_default();
375
376 let port = match env::var(SUBC_PORT_ENV) {
377 Ok(raw) if !raw.trim().is_empty() => {
378 let port = raw
379 .parse::<u16>()
380 .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
381 if let Some(config_port) = config_port {
382 info!(
383 env = SUBC_PORT_ENV,
384 env_port = port,
385 config_port,
386 "SUBC_PORT overrides daemon config port"
387 );
388 }
389 port
390 }
391 Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
392 };
393
394 let (connection_file_path, connection_file_source) =
395 connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
396 Ok(Self::new(connection_file_path, port)
397 .with_configured_modules(configured_modules)
398 .with_storage_config(storage_config)
399 .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
400 .with_scope_authority_owners(scope_authority_owners)
401 .with_federated_scope_owner(federated_scope_owner)
402 .with_privileged_operator_prompt(privileged_operator_prompt)
403 .with_activity_config(activity_config)
404 .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
405 .with_reserved_capabilities(reserved_capabilities)
406 .with_daemon_config_source(daemon_config_path, config_port)
407 .with_connection_file_source(connection_file_source))
408 }
409
410 pub fn with_daemon_config_path(
411 self,
412 daemon_config_path: impl AsRef<Path>,
413 ) -> Result<Self, BootstrapError> {
414 let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
415 let (daemon_config, activity_config) =
416 daemon_config::load_with_activity(&daemon_config_path)
417 .map_err(BootstrapError::DaemonConfig)?
418 .map(|(config, activity)| (Some(config), activity))
419 .unwrap_or_default();
420 let configured_port = daemon_config.as_ref().and_then(|config| config.port);
421 let storage_config = daemon_config
422 .as_ref()
423 .and_then(|config| config.storage.clone());
424 let admission_facts_carrier_module_id = daemon_config
425 .as_ref()
426 .and_then(|config| config.admission_facts_carrier_module_id.clone());
427 let admission_facts_targets = daemon_config
428 .as_ref()
429 .and_then(|config| config.admission_facts_targets.clone());
430 let scope_authority_owners = daemon_config
431 .as_ref()
432 .map(|config| config.scope_authority_owners.clone())
433 .unwrap_or_else(daemon_config::default_scope_authority_owners);
434 let federated_scope_owner = daemon_config
435 .as_ref()
436 .map(|config| config.federated_scope_owner.clone())
437 .unwrap_or_else(daemon_config::default_federated_scope_owner);
438 let privileged_operator_prompt = daemon_config
439 .as_ref()
440 .map(|config| config.privileged_operator_prompt)
441 .unwrap_or_default();
442 let route_bind_relay_default_ms = daemon_config
443 .as_ref()
444 .and_then(|config| config.route_bind_relay_timeout_ms);
445 let reserved_capabilities = daemon_config
446 .as_ref()
447 .map(|config| config.reserved_capabilities.clone())
448 .unwrap_or_default();
449 let configured_modules = daemon_config
450 .map(|config| config.modules)
451 .unwrap_or_default();
452 Ok(self
453 .with_configured_modules(configured_modules)
454 .with_storage_config(storage_config)
455 .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
456 .with_scope_authority_owners(scope_authority_owners)
457 .with_federated_scope_owner(federated_scope_owner)
458 .with_privileged_operator_prompt(privileged_operator_prompt)
459 .with_activity_config(activity_config)
460 .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
461 .with_reserved_capabilities(reserved_capabilities)
462 .with_daemon_config_source(daemon_config_path, configured_port))
463 }
464
465 pub fn with_configured_modules(
466 mut self,
467 modules: impl IntoIterator<Item = ConfiguredModule>,
468 ) -> Self {
469 self.configured_modules = modules.into_iter().collect();
470 self.configured_modules
471 .sort_by(|left, right| left.module_id.cmp(&right.module_id));
472 self
473 }
474
475 pub fn with_storage_config(
476 mut self,
477 storage_config: Option<daemon_config::StorageConfig>,
478 ) -> Self {
479 self.storage_config = storage_config;
480 self
481 }
482
483 pub fn with_admission_facts_config(
484 mut self,
485 carrier_module_id: Option<String>,
486 targets: Option<Vec<String>>,
487 ) -> Self {
488 self.admission_facts = AdmissionFactsConfig {
489 carrier_module_id,
490 targets,
491 };
492 self
493 }
494
495 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
498 self.scope_authority_owners = owners;
499 self
500 }
501
502 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
503 self.federated_scope_owner = owner;
504 self
505 }
506
507 pub fn with_privileged_operator_prompt(
509 mut self,
510 mode: daemon_config::PrivilegedOperatorPrompt,
511 ) -> Self {
512 self.privileged_operator_prompt = mode;
513 self
514 }
515
516 pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
517 self.activity_config = config;
518 self
519 }
520
521 pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
526 self.route_bind_relay_default_ms = ms;
527 self
528 }
529
530 pub fn with_reserved_capabilities(
531 mut self,
532 reserved_capabilities: BTreeMap<String, String>,
533 ) -> Self {
534 self.reserved_capabilities = reserved_capabilities;
535 self
536 }
537
538 fn with_daemon_config_source(
539 mut self,
540 daemon_config_path: PathBuf,
541 configured_port: Option<u16>,
542 ) -> Self {
543 self.daemon_config_path = Some(daemon_config_path);
544 self.configured_port = configured_port;
545 self
546 }
547
548 fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
549 self.connection_file_source = source;
550 self
551 }
552
553 pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
554 self.watchdog_config = watchdog_config;
555 self
556 }
557}
558
559#[allow(clippy::large_enum_variant)]
563#[derive(Debug)]
564pub enum Outcome {
565 AlreadyRunning,
567 Bound(BoundDaemon),
570}
571
572#[derive(Debug)]
573pub struct BoundDaemon {
574 pub listeners: Vec<TcpListener>,
575 pub local_listener: Option<BoundLocalListener>,
579 pub local_endpoint_unavailable: Option<String>,
582 pub local_endpoint: Option<LocalEndpoint>,
585 pub connection_info: ConnectionInfo,
586 pub connection_file_path: PathBuf,
587 pub connection_file_source: ConnectionFileSource,
588 #[cfg(windows)]
589 connection_file_copy: Option<PathBuf>,
590 pub machine_id: Option<crate::machine_id::MachineId>,
593 run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
597 publication_lock: Option<StartLock>,
603}
604
605pub fn connection_file_path() -> PathBuf {
612 connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
613}
614
615fn connection_file_path_with_source(
616 runtime_dir: Option<OsString>,
617) -> (PathBuf, ConnectionFileSource) {
618 if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
619 return (
620 PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
621 ConnectionFileSource::XdgRuntimeDir,
622 );
623 }
624
625 (
626 env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
627 ConnectionFileSource::TempDirFallback,
628 )
629}
630
631pub async fn run() -> Result<(), BootstrapError> {
637 run_with_config(
640 BootstrapConfig::from_env_for_daemon_binary()?
641 .with_cgroup_placement(CgroupPlacementConfig::Current),
642 )
643 .await
644}
645
646pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
670 #[cfg(feature = "test-support")]
671 let shutdown_intent_window = config.shutdown_intent_window;
672 let forwarding = Arc::new(ForwardingTable::default());
673 forwarding.operator_confirms().configure_bootstrap(&config);
674 let configured_modules = config.configured_modules.clone();
675 let storage_config = config.storage_config.clone();
676 let admission_facts = config.admission_facts.clone();
677 let scope_authority_owners = config.scope_authority_owners.clone();
678 let federated_scope_owner = config.federated_scope_owner.clone();
679 let privileged_operator_prompt = config.privileged_operator_prompt;
680 let activity_config = config.activity_config;
681 let daemon_config_path = config.daemon_config_path.clone();
682 let configured_port = config.configured_port;
683 let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
684 let reserved_capabilities = config.reserved_capabilities.clone();
685 let watchdog_config = config.watchdog_config.clone();
686 let cgroup_placement_config = config.cgroup_placement.clone();
687 let capture_logs_dir = config.capture_logs_dir.clone();
688 let terminal_journal_path = config.terminal_journal_path.clone();
689 let live_children_path = config.live_children_path.clone();
690 let privacy_trampoline = config.privacy_trampoline.clone();
691 match ensure_singleton_inner(config, false).await? {
692 Outcome::AlreadyRunning => {
693 info!("subc daemon already running");
694 Ok(())
695 }
696 Outcome::Bound(bound) => {
697 #[cfg(target_os = "linux")]
698 let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
699 #[cfg(not(target_os = "linux"))]
700 let _ = cgroup_placement_config;
701 serve_bound_daemon(
702 bound,
703 forwarding,
704 configured_modules,
705 storage_config,
706 admission_facts,
707 scope_authority_owners,
708 federated_scope_owner,
709 privileged_operator_prompt,
710 activity_config,
711 daemon_config_path,
712 configured_port,
713 route_bind_relay_default_ms,
714 reserved_capabilities,
715 watchdog_config,
716 capture_logs_dir,
717 terminal_journal_path,
718 live_children_path,
719 privacy_trampoline,
720 #[cfg(feature = "test-support")]
721 shutdown_intent_window,
722 #[cfg(target_os = "linux")]
723 cgroup_placement,
724 )
725 .await
726 }
727 }
728}
729
730#[cfg(target_os = "linux")]
731fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
732 let result = match config {
733 CgroupPlacementConfig::Disabled => return None,
734 CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
735 CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
736 };
737
738 match result {
739 Ok(Some(placement)) => Some(placement),
740 Ok(None) => {
741 warn!(
742 placement = ?config,
743 "module cgroup placement is disabled: configured cgroup root is not delegated"
744 );
745 None
746 }
747 Err(error) => {
748 warn!(
749 placement = ?config,
750 error = %error,
751 "module cgroup placement is disabled by an unexpected cgroup probe error"
752 );
753 None
754 }
755 }
756}
757
758#[cfg(unix)]
765const NOFILE_TARGET: u64 = 65536;
766
767#[cfg(unix)]
771fn raise_nofile_limit() {
772 #[cfg(target_os = "macos")]
773 let ceiling = std::process::Command::new("/usr/sbin/sysctl")
774 .args(["-n", "kern.maxfilesperproc"])
775 .output()
776 .ok()
777 .filter(|output| output.status.success())
778 .and_then(|output| String::from_utf8(output.stdout).ok())
779 .and_then(|value| value.trim().parse::<u64>().ok())
780 .filter(|value| *value > 0);
781 #[cfg(not(target_os = "macos"))]
782 let ceiling = None;
783 raise_nofile_limit_with_ceiling(ceiling);
784}
785
786#[cfg(unix)]
787fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
788 match rlimit::Resource::NOFILE.get() {
789 Ok((soft, hard)) => {
790 let target = NOFILE_TARGET
794 .min(hard)
795 .min(kernel_ceiling.unwrap_or(u64::MAX));
796 if soft >= target {
797 return;
798 }
799 match rlimit::Resource::NOFILE.set(target, hard) {
800 Ok(()) => info!(
801 previous_soft = soft,
802 new_soft = target,
803 hard,
804 "raised open-file soft limit for daemon and module children"
805 ),
806 Err(err) => warn!(
807 soft,
808 hard,
809 error = %err,
810 "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
811 ),
812 }
813 }
814 Err(err) => warn!(error = %err, "could not read open-file limit"),
815 }
816}
817
818#[cfg(windows)]
825fn raise_nofile_limit() {
826 const MAXSTDIO_TARGET: u32 = 8192;
827 let current = rlimit::getmaxstdio();
828 if current >= MAXSTDIO_TARGET {
829 return;
830 }
831 match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
832 Ok(new_max) => info!(
833 previous = current,
834 new_max, "raised CRT stdio-stream limit for daemon"
835 ),
836 Err(err) => warn!(
837 current,
838 error = %err,
839 "could not raise CRT stdio-stream limit"
840 ),
841 }
842}
843
844#[cfg(not(any(unix, windows)))]
845fn raise_nofile_limit() {}
846
847pub async fn run_with_daemon_config_path(
848 config: BootstrapConfig,
849 daemon_config_path: impl AsRef<Path>,
850) -> Result<(), BootstrapError> {
851 run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
852}
853
854#[allow(clippy::too_many_arguments)]
855async fn serve_bound_daemon(
856 bound: BoundDaemon,
857 forwarding: Arc<ForwardingTable>,
858 configured_modules: Vec<ConfiguredModule>,
859 storage_config: Option<daemon_config::StorageConfig>,
860 admission_facts: AdmissionFactsConfig,
861 scope_authority_owners: Vec<String>,
862 federated_scope_owner: String,
863 privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
864 activity_config: daemon_config::ActivityConfig,
865 daemon_config_path: Option<PathBuf>,
866 configured_port: Option<u16>,
867 route_bind_relay_default_ms: Option<u64>,
868 reserved_capabilities: BTreeMap<String, String>,
869 watchdog_config: DaemonSelfWatchdogConfig,
870 capture_logs_dir: Option<PathBuf>,
871 terminal_journal_path: Option<PathBuf>,
872 live_children_path: Option<PathBuf>,
873 privacy_trampoline: Option<PathBuf>,
874 #[cfg(feature = "test-support")] shutdown_intent_window: Duration,
875 #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
876) -> Result<(), BootstrapError> {
877 #[cfg(unix)]
878 let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
879 .map_err(BootstrapError::Signal)?;
880 raise_nofile_limit();
883 #[cfg(windows)]
884 if let Err(error) = subc_jobobject::ensure_root_job() {
885 warn!(%error, "module launches disabled: Windows root job setup failed");
888 }
889
890 info!(
891 connection_file = %bound.connection_file_path.display(),
892 connection_file_source = %bound.connection_file_source,
893 connection_file_source_reason = bound.connection_file_source.reason(),
894 endpoints = ?bound.connection_info.endpoints,
895 configured_modules = configured_modules.len(),
896 machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
897 "subc daemon starting"
898 );
899
900 let run_dir_lock = bound.run_dir_lock;
911 let publication_lock = bound.publication_lock;
912 if let Some(owner) = &run_dir_lock {
913 #[cfg(target_os = "linux")]
914 crate::live_children::sweep_orphans_with_placement(
915 owner,
916 &crate::live_children::AdoptedPids::none(),
917 crate::live_children::SweepBounds::default(),
918 cgroup_placement.as_ref(),
919 )
920 .await;
921 #[cfg(not(target_os = "linux"))]
922 crate::live_children::sweep_orphans(
923 owner,
924 &crate::live_children::AdoptedPids::none(),
925 crate::live_children::SweepBounds::default(),
926 )
927 .await;
928 }
929
930 let registry = Arc::new(Registry::default());
931 let process_liveness = Arc::new(SupervisorProcessLiveness::new());
932 let supervisor_handle = SupervisorHandle::new();
933 let _host_health_task = AbortOnDrop::new(supervisor_handle.spawn_host_health_monitor());
934 #[cfg(feature = "test-support")]
935 supervisor_handle.set_shutdown_intent_window(shutdown_intent_window);
936 let connected_clients = ConnectedClients::new();
937 let daemon_incarnation = format!(
938 "{:032x}",
939 u128::from_be_bytes(bound.connection_info.daemon_id)
940 );
941 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
942 .with_process_liveness(process_liveness.clone())
943 .with_forwarding(Arc::clone(&forwarding))
944 .with_handle(supervisor_handle.clone())
945 .with_connection_file_path(bound.connection_file_path.clone())
946 .with_daemon_incarnation(daemon_incarnation.clone());
947 let supervisor = match privacy_trampoline {
948 Some(path) => supervisor.with_privacy_trampoline(path),
949 None => supervisor,
950 };
951 let activity_run_dir = terminal_journal_path
991 .as_ref()
992 .and_then(|path| path.parent())
993 .map(Path::to_path_buf);
994 let supervisor = match terminal_journal_path {
995 Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
996 None => supervisor,
997 };
998 let supervisor = match capture_logs_dir {
999 Some(dir) => supervisor.with_capture_logs_dir(dir),
1000 None => supervisor,
1001 };
1002 let supervisor = match live_children_path {
1003 Some(path) => supervisor.with_live_children_record(path),
1004 None => supervisor,
1005 };
1006 #[cfg(target_os = "linux")]
1007 let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
1008 let route_bind_relay_timeouts = configured_modules
1014 .iter()
1015 .filter_map(|module| {
1016 module
1017 .route_bind_relay_timeout_ms
1018 .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
1019 })
1020 .collect::<std::collections::BTreeMap<_, _>>();
1021 let control_start_clock = crate::clock::StartClock::capture();
1022 let mut control = ControlHandler::with_forwarding(Arc::clone(®istry), forwarding)
1023 .with_daemon_version(bound.connection_info.daemon_ver.clone())
1024 .with_process_liveness(process_liveness)
1025 .with_supervisor(supervisor_handle)
1026 .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
1029 .with_connected_clients(connected_clients.clone())
1030 .with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
1031 .with_storage_config(storage_config)
1032 .with_machine_id(bound.machine_id.clone())
1033 .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
1034 .with_scope_authority_owners(scope_authority_owners)
1035 .with_federated_scope_owner(federated_scope_owner)
1036 .with_privileged_operator_prompt(privileged_operator_prompt)
1037 .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
1038 .with_daemon_provenance(
1039 bound.connection_info.pid,
1040 control_start_clock.started_at_ms(),
1041 std::env::current_exe().ok(),
1042 normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
1043 normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
1044 )
1045 .with_daemon_start_clock(control_start_clock)
1046 .with_capability_config(
1047 configured_modules
1048 .iter()
1049 .map(|module| (module.module_id.clone(), module.enabled)),
1050 reserved_capabilities,
1051 );
1052 if let Some(run_dir) = activity_run_dir {
1053 control =
1054 control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
1055 }
1056 if let Some(ms) = route_bind_relay_default_ms {
1057 control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
1060 }
1061 if let Some(config_path) = daemon_config_path {
1062 control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
1063 }
1064 let control = Arc::new(control);
1065 let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
1066 let auth = ServerAuth::new(
1067 bound.connection_info.key.clone(),
1068 bound.connection_info.daemon_id,
1069 bound.connection_info.daemon_ver.clone(),
1070 )
1071 .with_connected_clients(connected_clients);
1072
1073 let mut registered_modules = Vec::with_capacity(configured_modules.len());
1077 for configured in configured_modules {
1078 let enabled = configured.enabled;
1079 let health = configured.health.clone();
1080 let module_id = configured.module_id.clone();
1081 let registered = match supervisor.register_configured_with_health(
1082 configured.module_spec(),
1083 enabled,
1084 health.clone(),
1085 configured.drain_timeout_ms,
1086 configured.restart,
1087 configured.register_timeout,
1088 ) {
1089 Ok(registered) => registered,
1090 Err(err) => {
1091 error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
1092 continue;
1093 }
1094 };
1095 registered_modules.push(registered);
1096 let default_threshold = HealthConfig::default().failure_threshold;
1099 if enabled && health.failure_threshold > default_threshold {
1100 warn!(
1101 module_id = %module_id,
1102 failure_threshold = health.failure_threshold,
1103 default_threshold,
1104 tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1105 "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1106 );
1107 }
1108 info!(module_id = %module_id, enabled, "configured module registered");
1109 }
1110 control.refresh_capability_requirements();
1111
1112 let local_serve_task = bound.local_listener.map(|local_listener| {
1115 AbortOnDrop::new(tokio::spawn(serve_local_listener(
1116 local_listener,
1117 Arc::clone(&router),
1118 auth.clone(),
1119 )))
1120 });
1121 #[cfg(windows)]
1124 let _local_serve_task = local_serve_task;
1125 let mut serve_task =
1126 AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1127 verify_serving(&bound.connection_info).await?;
1128 let (published, publication_lock) = publish_connection_file(
1135 &bound.connection_file_path,
1136 #[cfg(windows)]
1137 bound.connection_file_copy.as_deref(),
1138 &bound.connection_info,
1139 bound.local_endpoint.as_ref(),
1140 publication_lock,
1141 )
1142 .await;
1143 published?;
1144 #[cfg(test)]
1145 startup_publication_hook::run(&bound.connection_file_path).await;
1146 drop(publication_lock);
1147 let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1148 #[cfg(target_os = "linux")]
1150 let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1151 crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1152 ));
1153 let _watchdog_task = AbortOnDrop::new(
1154 DaemonSelfWatchdog::new(
1155 bound.connection_info.clone(),
1156 bound.connection_file_path.clone(),
1157 )
1158 .with_config(watchdog_config)
1159 .spawn(),
1160 );
1161
1162 for registered in registered_modules {
1163 match supervisor.start_registered_configured(registered, None, true) {
1164 Ok(module) => {
1165 info!(module_id = %module.module_id(), "configured module supervised");
1166 }
1167 Err(err) => {
1168 error!(error = %err, "failed to start configured module; continuing daemon startup");
1169 }
1170 }
1171 }
1172
1173 control.refresh_capability_requirements();
1174 Arc::clone(&control).spawn_capability_deadline_loop();
1175 Arc::clone(&control).spawn_scope_expiry_loop();
1176 control.spawn_resource_sampler();
1177
1178 #[cfg(unix)]
1179 {
1180 tokio::select! {
1181 result = serve_task.join() => {
1182 return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1183 }
1184 _ = terminate.recv() => {}
1185 }
1186 supervisor.begin_daemon_shutdown();
1191 drop(_watchdog_task);
1195 drop(serve_task);
1199 drop(local_serve_task);
1200 let second_sigterm = async {
1201 terminate.recv().await;
1202 std::time::Instant::now()
1203 };
1204 tokio::pin!(second_sigterm);
1205 let mut escalation_started = tokio::select! {
1206 biased;
1207 at = second_sigterm.as_mut() => {
1208 info!("second SIGTERM: abandoning daemon shutdown wait");
1209 Some(at)
1210 }
1211 result = supervisor.drain_for_daemon_shutdown() => {
1212 if let Err(error) = result {
1213 warn!(%error, "daemon shutdown drain failed; exiting anyway");
1214 }
1215 None
1216 }
1217 };
1218 let already_escalated = escalation_started.is_some();
1225 supervisor
1226 .end_children_for_daemon_shutdown(already_escalated, async {
1227 if already_escalated {
1228 std::future::pending::<()>().await;
1229 } else {
1230 escalation_started = Some(second_sigterm.await);
1231 }
1232 })
1233 .await;
1234 if let Some(at) = escalation_started {
1235 info!(
1236 second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1237 "daemon shutdown wait ended after second SIGTERM"
1238 );
1239 }
1240 info!("daemon exiting: SIGTERM shutdown complete");
1241 Ok(())
1242 }
1243 #[cfg(not(unix))]
1244 serve_task
1245 .join()
1246 .await
1247 .map_err(BootstrapError::ServeJoin)?
1248 .map_err(BootstrapError::Serve)
1249}
1250
1251fn normalized_build_provenance(value: &str) -> Option<String> {
1252 match value.trim() {
1253 "" | "unavailable" => None,
1254 value => Some(value.to_string()),
1255 }
1256}
1257
1258pub async fn ensure_singleton(
1266 connection_file_path: impl AsRef<Path>,
1267 port: u16,
1268) -> Result<Outcome, BootstrapError> {
1269 ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1270}
1271
1272pub async fn ensure_singleton_with_config(
1273 config: BootstrapConfig,
1274) -> Result<Outcome, BootstrapError> {
1275 ensure_singleton_inner(config, true).await
1276}
1277
1278async fn ensure_singleton_inner(
1279 config: BootstrapConfig,
1280 publish: bool,
1281) -> Result<Outcome, BootstrapError> {
1282 #[cfg(windows)]
1283 let copy_path = windows_connection_copy::resolve(&config);
1284 let path = config.connection_file_path;
1285
1286 if stable_peer(&path).await? {
1287 return Ok(Outcome::AlreadyRunning);
1288 }
1289
1290 let lock = StartLock::acquire(&path).await?;
1291
1292 if stable_peer(&path).await? {
1296 return Ok(Outcome::AlreadyRunning);
1297 }
1298
1299 let run_dir_lock = config
1305 .live_children_path
1306 .as_deref()
1307 .map(crate::run_dir_lock::RunDirLock::acquire)
1308 .transpose()?;
1309
1310 remove_stale_connection_file_if_present(&path)?;
1311 #[cfg(windows)]
1312 let copy_path = windows_connection_copy::reclaim(&path, copy_path).await;
1313
1314 let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1322 None => (None, lock, run_dir_lock),
1323 Some(machine_id_path) => {
1324 let task_path = machine_id_path.clone();
1325 match run_blocking_holding((lock, run_dir_lock), move || {
1326 crate::machine_id::load_or_mint(&task_path)
1327 })
1328 .await
1329 {
1330 Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1331 Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1332 Err(join_error) => {
1333 return Err(BootstrapError::MachineId(
1334 crate::machine_id::MachineIdFileError::Write {
1335 path: machine_id_path,
1336 source: io::Error::other(describe_blocking_task_failure(
1337 "loading or minting the machine id",
1338 &join_error,
1339 )),
1340 },
1341 ))
1342 }
1343 }
1344 }
1345 };
1346
1347 let (listeners, endpoints) = bind_loopback(config.port).await?;
1348 let (local_listener, local_endpoint, local_endpoint_unavailable) =
1354 if config.local_listener.configured_dir().is_some() {
1355 match local_listener::bind(&config.local_listener, &path) {
1356 Ok(local_listener) => {
1357 let endpoint = local_listener.endpoint();
1358 (Some(local_listener), Some(endpoint), None)
1359 }
1360 Err(error) => {
1361 let reason = error.to_string();
1362 warn!(
1363 error = %reason,
1364 "the local endpoint is unavailable; serving loopback TCP only"
1365 );
1366 (None, None, Some(reason))
1367 }
1368 }
1369 } else {
1370 (None, None, None)
1371 };
1372 let connection_info = ConnectionInfo {
1373 schema: SCHEMA_VERSION,
1374 wire_version: Some(PROTOCOL_VERSION),
1375 endpoints,
1376 key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1377 daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1378 pid: process::id(),
1379 daemon_ver: config.daemon_ver,
1380 };
1381
1382 let publication_lock = if publish {
1386 let (published, lock) = publish_connection_file(
1387 &path,
1388 #[cfg(windows)]
1389 copy_path.as_deref(),
1390 &connection_info,
1391 local_endpoint.as_ref(),
1392 lock,
1393 )
1394 .await;
1395 if let Err(error) = published {
1396 if let Some(local_listener) = local_listener.as_ref() {
1397 local_listener::release(local_listener);
1398 }
1399 drop(listeners);
1400 return Err(error);
1401 }
1402 drop(lock);
1403 None
1404 } else {
1405 Some(lock)
1406 };
1407
1408 Ok(Outcome::Bound(BoundDaemon {
1409 listeners,
1410 local_listener,
1411 local_endpoint,
1412 local_endpoint_unavailable,
1413 connection_info,
1414 connection_file_path: path,
1415 connection_file_source: config.connection_file_source,
1416 #[cfg(windows)]
1417 connection_file_copy: copy_path,
1418 machine_id,
1419 run_dir_lock,
1420 publication_lock,
1421 }))
1422}
1423
1424#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1425enum Probe {
1426 Live,
1427 StaleOrAbsent,
1428}
1429
1430async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1431 if !matches!(probe_existing(path).await?, Probe::Live) {
1432 return Ok(false);
1433 }
1434 tokio::time::sleep(Duration::from_millis(1500)).await;
1440 Ok(matches!(probe_existing(path).await?, Probe::Live))
1441}
1442
1443async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1444 let info = match connection_file::read(path) {
1445 Ok(info) => info,
1446 Err(source) if is_absent_or_stale_connection_file(&source) => {
1447 return Ok(Probe::StaleOrAbsent)
1448 }
1449 Err(source) => {
1450 return Err(BootstrapError::ConnectionFileRead {
1451 path: path.to_path_buf(),
1452 source,
1453 })
1454 }
1455 };
1456
1457 for endpoint in &info.endpoints {
1458 if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1459 return Ok(Probe::Live);
1460 }
1461 }
1462
1463 Ok(Probe::StaleOrAbsent)
1464}
1465
1466async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1467 let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1468 return Probe::StaleOrAbsent;
1469 };
1470 if !ip.is_loopback() {
1471 return Probe::StaleOrAbsent;
1472 }
1473 let addr = SocketAddr::new(ip, endpoint.port);
1474
1475 let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1476 Ok(Ok(stream)) => stream,
1477 Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1478 };
1479
1480 match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1481 Ok(()) => Probe::Live,
1482 Err(AuthError::DaemonIdMismatch)
1483 | Err(AuthError::InvalidServerProof)
1484 | Err(AuthError::UnexpectedEof { .. })
1485 | Err(AuthError::Timeout { .. })
1486 | Err(AuthError::JsonEncode { .. })
1487 | Err(AuthError::JsonDecode { .. })
1488 | Err(AuthError::Io { .. })
1489 | Err(AuthError::MessageTooLarge { .. })
1490 | Err(AuthError::KeyTooShort { .. })
1491 | Err(AuthError::Random(_))
1492 | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1493 }
1494}
1495
1496async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1497 if let Some(endpoint) = info.endpoints.first() {
1501 if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1502 return Ok(());
1503 }
1504 }
1505 Err(BootstrapError::StartupNotServing)
1506}
1507
1508async fn run_blocking_holding<G, T>(
1523 guards: G,
1524 work: impl FnOnce() -> T + Send + 'static,
1525) -> Result<(T, G), JoinError>
1526where
1527 G: Send + 'static,
1528 T: Send + 'static,
1529{
1530 tokio::task::spawn_blocking(move || {
1531 let output = work();
1532 (output, guards)
1533 })
1534 .await
1535}
1536
1537fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1539 if join_error.is_panic() {
1540 format!("the background thread {work} panicked before finishing: {join_error}")
1541 } else {
1542 format!(
1543 "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1544 )
1545 }
1546}
1547
1548async fn publish_connection_file<G: Send + 'static>(
1565 path: &Path,
1566 #[cfg(windows)] copy_path: Option<&Path>,
1567 info: &ConnectionInfo,
1568 local_endpoint: Option<&LocalEndpoint>,
1569 guard: G,
1570) -> (Result<(), BootstrapError>, Option<G>) {
1571 let task_path = path.to_path_buf();
1572 let task_info = info.clone();
1573 let task_endpoint = local_endpoint.cloned();
1574 #[cfg(windows)]
1575 let copy = windows_connection_copy::claim_for_publication(path, copy_path, info).await;
1576 let outcome = run_blocking_holding(guard, move || {
1577 write_connection_file(&task_path, &task_info, task_endpoint.as_ref())?;
1578 #[cfg(windows)]
1579 if let Some((copy_path, _lock)) = copy {
1580 if let Err(error) =
1581 write_connection_file(©_path, &task_info, task_endpoint.as_ref())
1582 {
1583 windows_connection_copy::warn_unavailable(&task_path, Some(©_path), &error);
1584 }
1585 }
1586 Ok(())
1587 })
1588 .await;
1589 match outcome {
1590 Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1591 Ok((Err(source), guard)) => (
1592 Err(BootstrapError::ConnectionFileWrite {
1593 path: path.to_path_buf(),
1594 source,
1595 }),
1596 Some(guard),
1597 ),
1598 Err(join_error) => (
1599 Err(BootstrapError::ConnectionFileWrite {
1600 path: path.to_path_buf(),
1601 source: ConnectionFileError::Io {
1602 op: "write",
1603 path: path.to_path_buf(),
1604 source: io::Error::other(describe_blocking_task_failure(
1605 "writing the connection file",
1606 &join_error,
1607 )),
1608 },
1609 }),
1610 None,
1611 ),
1612 }
1613}
1614
1615fn write_connection_file(
1622 path: &Path,
1623 info: &ConnectionInfo,
1624 local_endpoint: Option<&LocalEndpoint>,
1625) -> Result<(), ConnectionFileError> {
1626 #[cfg(test)]
1627 connection_file_write_hook::run(path);
1628 write_atomic_with_local_endpoint(path, info, local_endpoint)
1629}
1630
1631#[cfg(test)]
1634mod connection_file_write_hook {
1635 use std::{
1636 path::{Path, PathBuf},
1637 sync::{Mutex, PoisonError},
1638 };
1639
1640 type Hook = Box<dyn FnOnce() + Send>;
1641
1642 static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
1645
1646 pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
1648 HOOKS
1649 .lock()
1650 .unwrap_or_else(PoisonError::into_inner)
1651 .push((path.to_path_buf(), Box::new(hook)));
1652 }
1653
1654 pub(super) fn run(path: &Path) {
1655 let hook = {
1656 let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
1657 hooks
1658 .iter()
1659 .position(|(hooked, _)| hooked == path)
1660 .map(|index| hooks.swap_remove(index).1)
1661 };
1662 if let Some(hook) = hook {
1663 hook();
1664 }
1665 }
1666}
1667
1668#[cfg(test)]
1671mod startup_publication_hook {
1672 use std::{
1673 path::{Path, PathBuf},
1674 sync::{Mutex, PoisonError},
1675 };
1676 use tokio::sync::oneshot;
1677
1678 type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
1679 static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
1680
1681 pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
1682 let (entered_tx, entered_rx) = oneshot::channel();
1683 let (release_tx, release_rx) = oneshot::channel();
1684 BARRIERS
1685 .lock()
1686 .unwrap_or_else(PoisonError::into_inner)
1687 .push((path.to_path_buf(), (entered_tx, release_rx)));
1688 (entered_rx, release_tx)
1689 }
1690
1691 pub(super) async fn run(path: &Path) {
1692 let barrier = {
1693 let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
1694 barriers
1695 .iter()
1696 .position(|(hooked, _)| hooked == path)
1697 .map(|index| barriers.swap_remove(index).1)
1698 };
1699 if let Some((entered, release)) = barrier {
1700 let _ = entered.send(());
1701 let _ = release.await;
1702 }
1703 }
1704}
1705
1706fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
1707 match err {
1708 ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
1709 ConnectionFileError::JsonRead { .. }
1710 | ConnectionFileError::UnsupportedSchema { .. }
1711 | ConnectionFileError::Invalid { .. }
1712 | ConnectionFileError::KeyTooShort { .. }
1713 | ConnectionFileError::InsecurePermissions { .. } => true,
1717 ConnectionFileError::MissingParent { .. }
1718 | ConnectionFileError::MissingFileName { .. }
1719 | ConnectionFileError::InsecureParentDirectory { .. }
1723 | ConnectionFileError::Io { .. }
1724 | ConnectionFileError::JsonWrite { .. }
1725 | ConnectionFileError::Random(_)
1726 | ConnectionFileError::WireVersionMismatch { .. } => false,
1729 }
1730}
1731
1732fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
1733 match fs::remove_file(path) {
1734 Ok(()) => Ok(()),
1735 Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
1736 Err(source) => Err(BootstrapError::RemoveStale {
1737 path: path.to_path_buf(),
1738 source,
1739 }),
1740 }
1741}
1742
1743async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
1744 bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
1745}
1746
1747async fn bind_loopback_with<F, Fut>(
1748 port: u16,
1749 mut bind_v6: F,
1750) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
1751where
1752 F: FnMut(u16) -> Fut,
1753 Fut: std::future::Future<Output = io::Result<TcpListener>>,
1754{
1755 let v4_host = Ipv4Addr::LOCALHOST;
1756 let v4 = TcpListener::bind((v4_host, port))
1757 .await
1758 .map_err(|source| BootstrapError::Bind {
1759 host: v4_host.to_string(),
1760 port,
1761 source,
1762 })?;
1763 let actual_port = v4
1764 .local_addr()
1765 .map_err(|source| BootstrapError::LocalAddr {
1766 host: v4_host.to_string(),
1767 source,
1768 })?
1769 .port();
1770
1771 let mut listeners = vec![v4];
1772 let mut endpoints = vec![Endpoint {
1773 host: v4_host.to_string(),
1774 port: actual_port,
1775 }];
1776
1777 let v6_host = Ipv6Addr::LOCALHOST;
1778 match bind_v6(actual_port).await {
1779 Ok(v6) => {
1780 listeners.push(v6);
1781 endpoints.push(Endpoint {
1782 host: v6_host.to_string(),
1783 port: actual_port,
1784 });
1785 }
1786 Err(err)
1787 if ipv6_loopback_unavailable(&err)
1788 || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
1789 {
1790 warn!(
1791 port = actual_port,
1792 error = %err,
1793 "IPv6 loopback unavailable; serving only IPv4 loopback"
1794 );
1795 }
1796 Err(source) => {
1797 drop(listeners);
1798 return Err(BootstrapError::Bind {
1799 host: v6_host.to_string(),
1800 port: actual_port,
1801 source,
1802 });
1803 }
1804 }
1805
1806 Ok((listeners, endpoints))
1807}
1808
1809fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
1810 matches!(
1811 err.kind(),
1812 io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
1813 ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
1814}
1815
1816struct AbortOnDrop<T> {
1817 handle: JoinHandle<T>,
1818}
1819
1820impl<T> AbortOnDrop<T> {
1821 fn new(handle: JoinHandle<T>) -> Self {
1822 Self { handle }
1823 }
1824
1825 async fn join(&mut self) -> Result<T, JoinError> {
1826 (&mut self.handle).await
1827 }
1828}
1829
1830impl<T> Drop for AbortOnDrop<T> {
1831 fn drop(&mut self) {
1832 if !self.handle.is_finished() {
1833 self.handle.abort();
1834 }
1835 }
1836}
1837
1838#[derive(Debug)]
1839struct StartLock {
1840 _file: fs::File,
1842}
1843
1844impl Drop for StartLock {
1845 fn drop(&mut self) {
1846 if let Err(error) = FileExt::unlock(&self._file) {
1850 tracing::warn!(%error, "failed to release the singleton start lock");
1851 }
1852 }
1853}
1854
1855impl StartLock {
1856 async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
1857 let path = start_lock_path(connection_file_path);
1858 for _ in 0..START_LOCK_RETRIES {
1859 let file = match open_owner_only_lock(&path) {
1860 Ok(file) => file,
1861 Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
1862 };
1863 match FileExt::try_lock(&file) {
1864 Ok(()) => return Ok(Self { _file: file }),
1865 Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
1866 Err(TryLockError::Error(source)) => {
1867 return Err(BootstrapError::StartLockCreate { path, source });
1868 }
1869 }
1870 }
1871
1872 Err(BootstrapError::StartLockBusy {
1873 path,
1874 attempts: START_LOCK_RETRIES,
1875 })
1876 }
1877}
1878
1879pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
1880 let mut options = fs::OpenOptions::new();
1881 options.read(true).write(true).create(true);
1882 #[cfg(unix)]
1883 {
1884 use std::os::unix::fs::OpenOptionsExt;
1885 options.mode(0o600);
1886 }
1887 options.open(path)
1888}
1889
1890fn start_lock_path(connection_file_path: &Path) -> PathBuf {
1891 let file_name = connection_file_path
1892 .file_name()
1893 .map(|name| name.to_string_lossy())
1894 .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
1895 let lock_name = format!("{file_name}.start-lock");
1896 connection_file_path
1897 .parent()
1898 .filter(|parent| !parent.as_os_str().is_empty())
1899 .unwrap_or_else(|| Path::new("."))
1900 .join(lock_name)
1901}
1902
1903fn non_empty_os_var(key: &str) -> Option<OsString> {
1904 let value = env::var_os(key)?;
1905 if value.is_empty() {
1906 None
1907 } else {
1908 Some(value)
1909 }
1910}
1911
1912#[derive(Debug)]
1915pub enum BootstrapError {
1916 StartupNotServing,
1918 #[cfg(unix)]
1919 Signal(io::Error),
1920 InvalidPort {
1921 raw: String,
1922 source: std::num::ParseIntError,
1923 },
1924 ConnectionFileRead {
1925 path: PathBuf,
1926 source: ConnectionFileError,
1927 },
1928 ConnectionFileWrite {
1929 path: PathBuf,
1930 source: ConnectionFileError,
1931 },
1932 GenerateConnectionFile(ConnectionFileError),
1933 StartLockCreate {
1934 path: PathBuf,
1935 source: io::Error,
1936 },
1937 StartLockBusy {
1938 path: PathBuf,
1939 attempts: usize,
1940 },
1941 RunDirLockCreate {
1943 path: PathBuf,
1944 source: io::Error,
1945 },
1946 RunDirBusy {
1951 path: PathBuf,
1952 holder_pid: Option<u32>,
1953 },
1954 RemoveStale {
1955 path: PathBuf,
1956 source: io::Error,
1957 },
1958 Bind {
1959 host: String,
1960 port: u16,
1961 source: io::Error,
1962 },
1963 LocalAddr {
1964 host: String,
1965 source: io::Error,
1966 },
1967 DaemonConfig(DaemonConfigError),
1968 MachineId(crate::machine_id::MachineIdFileError),
1971 RunDir(daemon_config::DaemonRunDirError),
1974 Serve(ServerError),
1975 ServeJoin(tokio::task::JoinError),
1976}
1977
1978impl From<ServerError> for BootstrapError {
1979 fn from(error: ServerError) -> Self {
1980 Self::Serve(error)
1981 }
1982}
1983
1984impl fmt::Display for BootstrapError {
1985 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1986 match self {
1987 Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
1988 #[cfg(unix)]
1989 Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
1990 Self::InvalidPort { raw, source } => {
1991 write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
1992 }
1993 Self::ConnectionFileRead { path, source } => write!(
1994 f,
1995 "failed to read connection file {}: {source}",
1996 path.display()
1997 ),
1998 Self::ConnectionFileWrite { path, source } => write!(
1999 f,
2000 "failed to publish connection file {}: {source}",
2001 path.display()
2002 ),
2003 Self::GenerateConnectionFile(err) => {
2004 write!(f, "failed to generate connection-file auth material: {err}")
2005 }
2006 Self::StartLockCreate { path, source } => {
2007 write!(
2008 f,
2009 "failed to create start lock {}: {source}",
2010 path.display()
2011 )
2012 }
2013 Self::StartLockBusy { path, attempts } => write!(
2014 f,
2015 "start lock {} remained busy after {attempts} attempts",
2016 path.display()
2017 ),
2018 Self::RunDirLockCreate { path, source } => write!(
2019 f,
2020 "refusing to start: failed to lock run directory via {}: {source}",
2021 path.display()
2022 ),
2023 Self::RunDirBusy { path, holder_pid } => {
2024 write!(
2025 f,
2026 "refusing to start: run directory lock {} is held by another daemon",
2027 path.display()
2028 )?;
2029 if let Some(pid) = holder_pid {
2030 write!(f, " (pid {pid})")?;
2031 }
2032 write!(
2033 f,
2034 "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
2035 )
2036 }
2037 Self::RemoveStale { path, source } => write!(
2038 f,
2039 "failed to remove stale connection file {}: {source}",
2040 path.display()
2041 ),
2042 Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
2043 write!(
2044 f,
2045 "port {port} in use on loopback {host}: {source}; set the port in config"
2046 )
2047 }
2048 Self::Bind { host, port, source } => {
2049 write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
2050 }
2051 Self::LocalAddr { host, source } => {
2052 write!(f, "failed to read local address for {host}: {source}")
2053 }
2054 Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
2055 Self::MachineId(err) => write!(f, "refusing to start: {err}"),
2056 Self::RunDir(err) => write!(f, "refusing to start: {err}"),
2057 Self::Serve(err) => write!(f, "daemon server failed: {err}"),
2058 Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
2059 }
2060 }
2061}
2062
2063impl Error for BootstrapError {
2064 fn source(&self) -> Option<&(dyn Error + 'static)> {
2065 match self {
2066 Self::StartupNotServing => None,
2067 #[cfg(unix)]
2068 Self::Signal(source) => Some(source),
2069 Self::InvalidPort { source, .. } => Some(source),
2070 Self::ConnectionFileRead { source, .. }
2071 | Self::ConnectionFileWrite { source, .. }
2072 | Self::GenerateConnectionFile(source) => Some(source),
2073 Self::StartLockCreate { source, .. }
2074 | Self::RunDirLockCreate { source, .. }
2075 | Self::RemoveStale { source, .. }
2076 | Self::Bind { source, .. }
2077 | Self::LocalAddr { source, .. } => Some(source),
2078 Self::DaemonConfig(err) => Some(err),
2079 Self::MachineId(err) => Some(err),
2080 Self::RunDir(err) => Some(err),
2081 Self::Serve(err) => Some(err),
2082 Self::ServeJoin(err) => Some(err),
2083 Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
2084 }
2085 }
2086}
2087
2088#[cfg(test)]
2089mod tests {
2090 mod activity {
2091 include!("activity_bootstrap_tests.rs");
2092 }
2093 mod startup_order {
2094 include!("startup_order_tests.rs");
2095 }
2096 use super::*;
2097 use crate::server::ServerAuth;
2098 use cortexkit_test_support::ScratchDir;
2099 #[cfg(target_os = "linux")]
2100 use std::collections::BTreeSet;
2101 use std::sync::Mutex;
2102 #[cfg(target_os = "linux")]
2103 use subc_control::ModuleProtocol;
2104 use subc_transport::{write_atomic, MIN_KEY_LEN};
2105 use tokio::io::AsyncReadExt;
2106 use tokio::task::JoinHandle;
2107
2108 #[cfg(unix)]
2109 use std::os::unix::fs::PermissionsExt;
2110
2111 static ENV_LOCK: Mutex<()> = Mutex::new(());
2112
2113 #[test]
2114 fn normalized_build_provenance_preserves_real_values() {
2115 assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
2116 }
2117
2118 #[test]
2119 fn normalized_build_provenance_omits_unavailable_and_empty_values() {
2120 assert_eq!(normalized_build_provenance("unavailable"), None);
2121 assert_eq!(normalized_build_provenance(""), None);
2122 }
2123
2124 #[cfg(target_os = "linux")]
2125 fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
2126 let cgroups = fs::read_to_string("/proc/self/cgroup")?;
2127 let relative = cgroups
2128 .lines()
2129 .find_map(|line| line.strip_prefix("0::"))
2130 .ok_or_else(|| {
2131 io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
2132 })?;
2133 Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
2134 }
2135
2136 #[cfg(target_os = "linux")]
2137 fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
2138 let modules = current_cgroup_path_for_test()?.join("subc-modules");
2139 let entries = match fs::read_dir(modules) {
2140 Ok(entries) => entries,
2141 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
2142 Err(error) => return Err(error),
2143 };
2144 let mut directories = BTreeSet::new();
2145 for entry in entries {
2146 let entry = entry?;
2147 if entry.file_type()?.is_dir() {
2148 directories.insert(entry.file_name());
2149 }
2150 }
2151 Ok(directories)
2152 }
2153
2154 #[cfg(target_os = "linux")]
2155 async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
2156 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2157 while !path.exists() && tokio::time::Instant::now() < deadline {
2158 assert!(
2159 !task.is_finished(),
2160 "daemon exited before creating {}",
2161 path.display()
2162 );
2163 sleep(Duration::from_millis(10)).await;
2164 }
2165 assert!(path.exists(), "daemon did not create {}", path.display());
2166 }
2167
2168 #[cfg(target_os = "linux")]
2173 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2174 async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
2175 let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
2176 let module_id = format!("cgroup-isolation-probe-{}", process::id());
2177 let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
2178 assert!(
2179 !before.contains(&OsString::from(&module_id)),
2180 "isolation probe cgroup already exists before this daemon starts"
2181 );
2182 let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
2183 let module = ConfiguredModule {
2184 carry_containment: false,
2185 module_id,
2186 program: PathBuf::from("sh"),
2187 args: vec!["-c".to_string(), "sleep 30".to_string()],
2188 env: Vec::new(),
2189 log: None,
2190 enabled: true,
2191 reserved: false,
2192 reserved_prefixes: Vec::new(),
2193 protocol: ModuleProtocol::None,
2194 overlap: Default::default(),
2195 health: HealthConfig::default(),
2196 drain_timeout_ms: None,
2197 register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2198 route_bind_relay_timeout_ms: None,
2199 restart: RestartPolicy::default(),
2200 };
2201 let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2202 .with_configured_modules([module])
2203 .with_capture_logs_dir(temp.join("logs"))
2204 .with_terminal_journal_path(temp.join("terminals.jsonl"));
2205 let task = tokio::spawn(run_with_config(config));
2206
2207 wait_for_path(&capture, &task).await;
2208 let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2209
2210 task.abort();
2211 assert!(task
2212 .await
2213 .expect_err("aborted daemon task must cancel")
2214 .is_cancelled());
2215 assert_eq!(
2216 before, after,
2217 "an in-process daemon must not create or reconcile ambient module cgroups"
2218 );
2219 }
2220
2221 #[cfg(target_os = "linux")]
2222 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2223 async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2224 let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2225 let cgroup_root = temp.join("cgroup");
2226 fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2227 fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2228 let modules = cgroup_root.join("subc-modules");
2229 let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2230 .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2231 .with_terminal_journal_path(temp.join("terminals.jsonl"));
2232 let task = tokio::spawn(run_with_config(config));
2233
2234 wait_for_path(&modules, &task).await;
2235
2236 task.abort();
2237 assert!(task
2238 .await
2239 .expect_err("aborted daemon task must cancel")
2240 .is_cancelled());
2241 assert!(
2242 fs::read_dir(&modules)
2243 .expect("read prepared modules directory")
2244 .next()
2245 .is_none(),
2246 "the delegation probe must clean up after itself"
2247 );
2248 }
2249
2250 struct EnvGuard {
2251 key: &'static str,
2252 previous: Option<OsString>,
2253 }
2254
2255 impl EnvGuard {
2256 fn set(key: &'static str, value: &Path) -> Self {
2257 let previous = env::var_os(key);
2258 env::set_var(key, value);
2259 Self { key, previous }
2260 }
2261
2262 fn set_str(key: &'static str, value: &str) -> Self {
2263 let previous = env::var_os(key);
2264 env::set_var(key, value);
2265 Self { key, previous }
2266 }
2267
2268 fn unset(key: &'static str) -> Self {
2269 let previous = env::var_os(key);
2270 env::remove_var(key);
2271 Self { key, previous }
2272 }
2273 }
2274
2275 impl Drop for EnvGuard {
2276 fn drop(&mut self) {
2277 match &self.previous {
2278 Some(value) => env::set_var(self.key, value),
2279 None => env::remove_var(self.key),
2280 }
2281 }
2282 }
2283
2284 fn unique_temp_dir(name: &str) -> ScratchDir {
2285 ScratchDir::new(name)
2286 }
2287
2288 fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2289 let dir = unique_temp_dir(name);
2290 let path = dir.join("conn.json");
2291 (dir, path)
2292 }
2293
2294 fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2295 ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2296 }
2297
2298 fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2299 let auth = auth_for(&bound.connection_info);
2300 tokio::spawn(serve_listeners(
2301 bound.listeners,
2302 Arc::new(Router::with_default_self_handler()),
2303 auth,
2304 ))
2305 }
2306
2307 fn expect_bound(outcome: Outcome) -> BoundDaemon {
2308 match outcome {
2309 Outcome::Bound(bound) => bound,
2310 Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2311 }
2312 }
2313
2314 async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2315 let endpoint = conn
2316 .endpoints
2317 .first()
2318 .expect("test connection file should have an endpoint");
2319 let ip: IpAddr = endpoint.host.parse().unwrap();
2320 TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2321 }
2322
2323 fn make_connection_info(port: u16) -> ConnectionInfo {
2324 ConnectionInfo {
2325 schema: SCHEMA_VERSION,
2326 wire_version: Some(PROTOCOL_VERSION),
2327 endpoints: vec![Endpoint {
2328 host: "127.0.0.1".to_owned(),
2329 port,
2330 }],
2331 key: generate_key().unwrap(),
2332 daemon_id: generate_daemon_id().unwrap(),
2333 pid: process::id(),
2334 daemon_ver: "test-subc".to_owned(),
2335 }
2336 }
2337
2338 fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2339 fs::write(path, contents).unwrap();
2340 #[cfg(unix)]
2341 fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2342 }
2343
2344 fn assert_owner_only_connection_file(path: &Path) {
2345 #[cfg(unix)]
2348 {
2349 let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2350 assert_eq!(mode, 0o600);
2351 }
2352 #[cfg(not(unix))]
2353 let _ = path;
2354 }
2355
2356 #[test]
2357 fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2358 let _env_lock = ENV_LOCK.lock().unwrap();
2359 let runtime_dir = unique_temp_dir("xdg-runtime");
2360 let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2361
2362 assert_eq!(
2363 connection_file_path(),
2364 runtime_dir.join(CONNECTION_FILE_NAME)
2365 );
2366 }
2367
2368 #[test]
2369 fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2370 let runtime_dir = OsString::from("/run/user/1000");
2371
2372 let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2373
2374 assert_eq!(
2375 path,
2376 PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2377 );
2378 assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2379 }
2380
2381 #[test]
2382 fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2383 let _env_lock = ENV_LOCK.lock().unwrap();
2384 let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2385
2386 assert_eq!(
2387 connection_file_path(),
2388 env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2389 );
2390 }
2391
2392 #[test]
2393 fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2394 let (path, source) = connection_file_path_with_source(None);
2395
2396 assert_eq!(
2397 path,
2398 env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2399 );
2400 assert_eq!(source, ConnectionFileSource::TempDirFallback);
2401 }
2402
2403 #[test]
2408 fn user_connection_token_is_stable_under_concurrent_callers() {
2409 let expected = user_connection_token();
2410 let workers: Vec<_> = (0..32)
2411 .map(|_| {
2412 std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2413 })
2414 .collect();
2415 for worker in workers {
2416 for token in worker.join().expect("probe thread") {
2417 assert_eq!(token, expected, "token diverged under concurrent probes");
2418 }
2419 }
2420 }
2421
2422 #[test]
2423 fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
2424 let (path, source) = connection_file_path_with_source(Some(OsString::new()));
2425
2426 assert_eq!(
2427 path,
2428 env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2429 );
2430 assert_eq!(source, ConnectionFileSource::TempDirFallback);
2431 }
2432
2433 #[test]
2437 fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
2438 let _env_lock = ENV_LOCK.lock().unwrap();
2439 let root = unique_temp_dir("daemon-binary-config");
2440 let data_home = root.join("data");
2441 let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
2442 let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2443 let _port = EnvGuard::unset(SUBC_PORT_ENV);
2444
2445 let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
2446
2447 let run_dir = data_home.join("cortexkit").join("run");
2448 assert_eq!(
2449 config.terminal_journal_path,
2450 Some(run_dir.join("terminals.jsonl"))
2451 );
2452 assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
2453 assert_eq!(
2454 BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
2455 None,
2456 "an in-process config must not journal anywhere unless asked to"
2457 );
2458 }
2459
2460 #[test]
2461 fn daemon_binary_config_refuses_a_relative_data_home() {
2462 let _env_lock = ENV_LOCK.lock().unwrap();
2463 let root = unique_temp_dir("daemon-binary-relative-data");
2464 let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
2465 let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2466 let _port = EnvGuard::unset(SUBC_PORT_ENV);
2467
2468 let error = BootstrapConfig::from_env_for_daemon_binary()
2469 .expect_err("a relative data home must refuse the daemon binary's config");
2470 assert!(
2471 matches!(error, BootstrapError::RunDir(_)),
2472 "expected a run-directory refusal, got {error}"
2473 );
2474 assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
2475 }
2476
2477 #[test]
2478 fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
2479 let _env_lock = ENV_LOCK.lock().unwrap();
2480 let _port = EnvGuard::unset(SUBC_PORT_ENV);
2481 let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
2482 let config_path = conn_path.with_file_name("subc.jsonc");
2483 for (doc, expected) in [
2484 (None, daemon_config::PrivilegedOperatorPrompt::Log),
2485 (
2486 Some(r#"{"version":1}"#),
2487 daemon_config::PrivilegedOperatorPrompt::Log,
2488 ),
2489 (
2490 Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
2491 daemon_config::PrivilegedOperatorPrompt::Log,
2492 ),
2493 (
2494 Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
2495 daemon_config::PrivilegedOperatorPrompt::Enforce,
2496 ),
2497 ] {
2498 if let Some(doc) = doc {
2499 fs::write(&config_path, doc).unwrap();
2500 }
2501 assert_eq!(
2502 BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2503 .unwrap()
2504 .privileged_operator_prompt,
2505 expected
2506 );
2507 assert_eq!(
2508 BootstrapConfig::new(&conn_path, 0)
2509 .with_daemon_config_path(&config_path)
2510 .unwrap()
2511 .privileged_operator_prompt,
2512 expected
2513 );
2514 }
2515 }
2516
2517 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2518 async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
2519 let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
2520 let connection_path = dir.join("connection.json");
2521 let config_path = dir.join("subc.jsonc");
2522 fs::write(
2523 &config_path,
2524 r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
2525 )
2526 .unwrap();
2527 let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
2528 .with_daemon_config_path(&config_path)
2529 .unwrap();
2530 let connection_path = config.connection_file_path.clone();
2531 let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
2532 tokio::time::timeout(Duration::from_secs(5), async {
2533 while !connection_path.exists() {
2534 assert!(
2535 !daemon.handle.is_finished(),
2536 "daemon exited before publishing"
2537 );
2538 tokio::task::yield_now().await;
2539 }
2540 })
2541 .await
2542 .unwrap();
2543 let info = connection_file::read(&connection_path).unwrap();
2544 let mut client = connect_from_info(&info).await.unwrap();
2545 authenticate_client(&mut client, &info, Duration::from_secs(5))
2546 .await
2547 .unwrap();
2548 for (correlation, mode, expected) in [
2549 (1, "enforce", Vec::new()),
2550 (2, "log", vec!["privileged_operator_prompt"]),
2551 (3, "log", vec!["privileged_operator_prompt"]),
2552 ] {
2553 fs::write(
2554 &config_path,
2555 serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
2556 )
2557 .unwrap();
2558 let frame = crate::Frame::build(
2559 subc_protocol::FrameType::Request,
2560 subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
2561 0,
2562 0,
2563 correlation,
2564 serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
2565 preview: true,
2566 })
2567 .unwrap(),
2568 )
2569 .unwrap();
2570 crate::write_frame(&mut client, &frame).await.unwrap();
2571 let reply =
2572 tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
2573 .await
2574 .unwrap()
2575 .unwrap()
2576 .unwrap();
2577 assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
2578 assert_eq!(reply.header.corr, correlation);
2579 let subc_control::ClientControlResponse::SupervisorRescan { result } =
2580 serde_json::from_slice(&reply.body).unwrap()
2581 else {
2582 panic!("expected a rescan response");
2583 };
2584 assert_eq!(result.restart_required, expected);
2585 }
2586 }
2587
2588 #[test]
2589 fn configured_port_uses_default_config_and_env_override() {
2590 let _env_lock = ENV_LOCK.lock().unwrap();
2591 let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
2592 let config_path = conn_path.with_file_name("subc.jsonc");
2593
2594 let _port = EnvGuard::unset(SUBC_PORT_ENV);
2595 assert_eq!(
2596 BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2597 .unwrap()
2598 .port,
2599 DEFAULT_SUBC_PORT
2600 );
2601
2602 fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
2603 assert_eq!(
2604 BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2605 .unwrap()
2606 .port,
2607 8123
2608 );
2609
2610 let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
2611 assert_eq!(
2612 BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2613 .unwrap()
2614 .port,
2615 9012
2616 );
2617 }
2618
2619 #[tokio::test]
2620 async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
2621 let (_dir, path) = temp_connection_file_path("departing-peer");
2622 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2623 let info = make_connection_info(listener.local_addr().unwrap().port());
2624 write_connection_file(&path, &info, None).unwrap();
2625 let peer = tokio::spawn(async move {
2626 let (mut stream, _) = listener.accept().await.unwrap();
2627 subc_transport::authenticate_server(
2628 &mut stream,
2629 &info.key,
2630 &info.daemon_id,
2631 &info.daemon_ver,
2632 PROBE_AUTH_DEADLINE,
2633 )
2634 .await
2635 .unwrap();
2636 });
2637 let outcome = ensure_singleton(&path, 0).await.unwrap();
2638 peer.await.unwrap();
2639 assert!(
2640 matches!(outcome, Outcome::Bound(_)),
2641 "a departing peer is not AlreadyRunning"
2642 );
2643 }
2644
2645 #[tokio::test]
2646 async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
2647 let (_dir, path) = temp_connection_file_path("already-running");
2648
2649 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2650 let server = start_server(bound);
2651
2652 let second = ensure_singleton(&path, 0).await.unwrap();
2653 assert!(matches!(second, Outcome::AlreadyRunning));
2654
2655 server.abort();
2656 let _ = server.await;
2657 }
2658
2659 #[tokio::test]
2660 async fn daemon_connection_file_publishes_protocol_wire_version() {
2661 let (_dir, path) = temp_connection_file_path("wire-version");
2662 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2663 assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
2664 assert_eq!(
2665 connection_file::read(&path).unwrap().wire_version,
2666 Some(PROTOCOL_VERSION)
2667 );
2668
2669 drop(bound.listeners);
2670 }
2671
2672 #[tokio::test]
2673 async fn stale_unbound_connection_file_is_reclaimed() {
2674 let (_dir, path) = temp_connection_file_path("stale-reclaim");
2675 let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2676 let stale_port = stale.local_addr().unwrap().port();
2677 drop(stale);
2678 let stale_info = make_connection_info(stale_port);
2679 write_atomic(&path, &stale_info).unwrap();
2680
2681 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2682 assert_ne!(bound.connection_info.key, stale_info.key);
2683 drop(bound.listeners);
2684 }
2685
2686 #[cfg(unix)]
2687 #[tokio::test]
2688 async fn ensure_singleton_reclaims_insecure_connection_file() {
2689 let (_dir, path) = temp_connection_file_path("insecure-reclaim");
2690 let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2691 let stale_port = stale.local_addr().unwrap().port();
2692 drop(stale);
2693 let stale_info = make_connection_info(stale_port);
2694 write_atomic(&path, &stale_info).unwrap();
2695 fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
2696
2697 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2698 assert_ne!(bound.connection_info.key, stale_info.key);
2699 assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2700 assert_owner_only_connection_file(&path);
2701
2702 drop(bound.listeners);
2703 }
2704
2705 #[tokio::test]
2706 async fn ensure_singleton_reclaims_non_loopback_connection_file() {
2707 let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
2708 let mut stale_info = make_connection_info(8757);
2709 stale_info.endpoints = vec![Endpoint {
2710 host: "192.0.2.10".to_owned(),
2711 port: 8757,
2712 }];
2713 write_atomic(&path, &stale_info).unwrap();
2714
2715 let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2716 let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
2717 assert_ne!(bound.connection_info.key, stale_info.key);
2718 assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2719 assert!(bound
2720 .connection_info
2721 .endpoints
2722 .iter()
2723 .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
2724 assert_owner_only_connection_file(&path);
2725
2726 drop(bound.listeners);
2727 }
2728
2729 #[tokio::test]
2730 async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
2731 let mut unsupported_schema = make_connection_info(8757);
2732 unsupported_schema.schema = SCHEMA_VERSION + 1;
2733
2734 let mut empty_endpoints = make_connection_info(8757);
2735 empty_endpoints.endpoints.clear();
2736
2737 let mut short_key = make_connection_info(8757);
2738 short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
2739
2740 let cases = vec![
2741 (
2742 "unsupported-schema",
2743 serde_json::to_vec(&unsupported_schema).unwrap(),
2744 Some(unsupported_schema),
2745 ),
2746 (
2747 "empty-endpoints",
2748 serde_json::to_vec(&empty_endpoints).unwrap(),
2749 Some(empty_endpoints),
2750 ),
2751 (
2752 "short-key",
2753 serde_json::to_vec(&short_key).unwrap(),
2754 Some(short_key),
2755 ),
2756 ("invalid-json", b"{not valid connection json".to_vec(), None),
2757 ];
2758
2759 for (label, contents, old_info) in cases {
2760 let (_dir, path) = temp_connection_file_path(label);
2761 write_raw_owner_only_connection_file(&path, &contents);
2762
2763 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2764 if let Some(old_info) = old_info {
2765 assert_ne!(bound.connection_info.key, old_info.key, "{label}");
2766 assert_ne!(
2767 bound.connection_info.daemon_id, old_info.daemon_id,
2768 "{label}"
2769 );
2770 }
2771 assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
2772 assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
2773 assert_owner_only_connection_file(&path);
2774
2775 drop(bound.listeners);
2776 }
2777 }
2778
2779 #[tokio::test]
2780 async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
2781 let (_dir, path) = temp_connection_file_path("foreign-reclaim");
2782 let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2783 let foreign_port = foreign.local_addr().unwrap().port();
2784 write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
2785 let foreign_task = tokio::spawn(async move {
2786 if let Ok((mut stream, _)) = foreign.accept().await {
2787 let mut buf = [0u8; 64];
2788 let _ = stream.read(&mut buf).await;
2789 }
2790 });
2791
2792 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2793 assert!(bound
2794 .connection_info
2795 .endpoints
2796 .iter()
2797 .all(|endpoint| endpoint.port != foreign_port));
2798
2799 drop(bound.listeners);
2800 let _ = foreign_task.await;
2801 }
2802
2803 #[tokio::test]
2804 async fn stale_start_lock_file_is_reclaimable() {
2805 let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
2806 let lock_path = start_lock_path(&path);
2807 drop(open_owner_only_lock(&lock_path).unwrap());
2808 assert!(lock_path.is_file());
2809
2810 let lock = StartLock::acquire(&path).await.unwrap();
2811 assert!(lock_path.is_file());
2812
2813 drop(lock);
2814 assert!(lock_path.is_file());
2815 }
2816
2817 #[tokio::test]
2818 async fn held_start_lock_blocks_second_acquire_until_release() {
2819 let (_dir, path) = temp_connection_file_path("start-lock-held");
2820 let lock_path = start_lock_path(&path);
2821 let first = StartLock::acquire(&path).await.unwrap();
2822
2823 let err = match StartLock::acquire(&path).await {
2824 Ok(_) => panic!("second acquire while held must stay busy"),
2825 Err(err) => err,
2826 };
2827 assert!(matches!(
2828 err,
2829 BootstrapError::StartLockBusy {
2830 ref path,
2831 attempts: START_LOCK_RETRIES,
2832 } if path == &lock_path
2833 ));
2834
2835 drop(first);
2836
2837 let second = StartLock::acquire(&path)
2838 .await
2839 .expect("released advisory lock should be reclaimable");
2840 drop(second);
2841 }
2842
2843 #[tokio::test]
2844 async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
2845 let (_dir, path) = temp_connection_file_path("bind-conflict");
2846 let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2847 let occupied_port = occupied.local_addr().unwrap().port();
2848
2849 let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
2850 assert!(matches!(
2851 err,
2852 BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
2853 ));
2854 assert!(err.to_string().contains("set the port in config"));
2855
2856 drop(occupied);
2857 }
2858
2859 #[tokio::test]
2860 async fn ephemeral_ipv6_collision_still_serves_ipv4() {
2861 let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
2862 let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
2863 let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
2864 drop(occupied);
2865 result
2866 })
2867 .await
2868 .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
2869 assert_eq!(endpoints.len(), 1);
2870 assert_eq!(endpoints[0].host, "127.0.0.1");
2871 TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
2872 .await
2873 .unwrap();
2874 drop(listeners);
2875 }
2876
2877 #[tokio::test]
2878 async fn unpublished_boot_holds_singleton_lock_until_publication() {
2879 let dir = ScratchDir::new("ul");
2880 let path = dir.join("c");
2881 let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2882 let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2883 assert!(!path.exists());
2884 let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2885 assert!(
2886 matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
2887 "an unpublished boot must keep its singleton claim"
2888 );
2889 drop(bound);
2890 FileExt::try_lock(&contender).unwrap();
2891 }
2892
2893 #[tokio::test]
2894 async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
2895 let dir = ScratchDir::new("sl");
2896 let path = dir.join("c");
2897 let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2898 let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2899 let inherited = bound
2902 .publication_lock
2903 .as_ref()
2904 .unwrap()
2905 ._file
2906 .try_clone()
2907 .unwrap();
2908 let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2909 assert!(matches!(
2910 FileExt::try_lock(&contender),
2911 Err(TryLockError::WouldBlock)
2912 ));
2913 drop(bound);
2914 FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
2915 drop(inherited);
2916 }
2917
2918 #[tokio::test]
2919 async fn readiness_probe_requires_an_authenticating_server() {
2920 let dir = ScratchDir::new("unserved-readiness");
2921 let bound = expect_bound(
2922 ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
2923 .await
2924 .unwrap(),
2925 );
2926 assert!(matches!(
2927 verify_serving(&bound.connection_info).await,
2928 Err(BootstrapError::StartupNotServing)
2929 ));
2930 }
2931
2932 #[cfg(unix)]
2933 #[tokio::test]
2934 async fn orphan_sweep_finishes_before_connection_file_publication() {
2935 use crate::live_children::{ExecutableIdentity, LiveChild};
2936 let dir = ScratchDir::new("pas");
2937 let path = dir.join("c");
2941 let record = dir.join("live-children.json");
2942 let ready = dir.join("ready");
2943 let mut child = std::process::Command::new("/bin/sh")
2944 .args([
2945 "-c",
2946 "trap '' TERM; : > \"$0\"; read -r _",
2947 ready.to_str().unwrap(),
2948 ])
2949 .stdin(std::process::Stdio::piped())
2950 .env("XDG_DATA_HOME", dir.path())
2951 .env("XDG_RUNTIME_DIR", dir.path())
2952 .env("XDG_CONFIG_HOME", dir.path())
2953 .spawn()
2954 .unwrap();
2955 timeout(Duration::from_secs(5), async {
2956 while !ready.exists() {
2957 sleep(Duration::from_millis(5)).await;
2958 }
2959 })
2960 .await
2961 .unwrap();
2962 let observed = subc_os::Process::open(child.id())
2963 .unwrap()
2964 .unwrap()
2965 .observe()
2966 .unwrap();
2967 crate::live_children::write_record(
2968 &record,
2969 &[LiveChild {
2970 container: None,
2971 module_id: "old-sleep".into(),
2972 pid: child.id(),
2973 protocol: subc_control::ModuleProtocol::None,
2974 start_time: Some(observed.start_time),
2975 executable: observed.executable.map(ExecutableIdentity::from),
2976 cgroup_name: None,
2977 }],
2978 )
2979 .unwrap();
2980 let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
2981 .with_live_children_record(&record);
2982 let daemon = tokio::spawn(run_with_config(config));
2983 timeout(Duration::from_secs(5), async {
2984 while !path.exists() {
2985 sleep(Duration::from_millis(5)).await;
2986 }
2987 })
2988 .await
2989 .unwrap();
2990 assert!(path.exists());
2991 let ended = child.try_wait().unwrap().is_some();
2992 if !ended {
2993 child.kill().unwrap();
2994 }
2995 child.wait().unwrap();
2996 if ended {
2997 assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
2998 }
2999 daemon.abort();
3000 let _ = daemon.await;
3001 assert!(
3002 ended,
3003 "the published daemon must not still owe its orphan sweep"
3004 );
3005 }
3006
3007 #[tokio::test(flavor = "current_thread")]
3014 async fn connection_file_publication_leaves_the_async_runtime_free() {
3015 use std::sync::atomic::{AtomicBool, Ordering};
3016
3017 let (_dir, path) = temp_connection_file_path("publish-off-runtime");
3018 let write_started = Arc::new(AtomicBool::new(false));
3019 let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
3020 let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
3021 let hook_started = Arc::clone(&write_started);
3022 connection_file_write_hook::install(&path, move || {
3023 hook_started.store(true, Ordering::SeqCst);
3024 let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
3025 let _ = verdict_tx.send(progressed);
3026 });
3027 let ticker = tokio::spawn(async move {
3030 loop {
3031 if write_started.load(Ordering::SeqCst) {
3032 let _ = progress_tx.send(());
3033 return;
3034 }
3035 sleep(Duration::from_millis(5)).await;
3036 }
3037 });
3038
3039 let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
3040 let published = timeout(Duration::from_secs(15), async {
3041 while !path.exists() && !daemon.is_finished() {
3042 sleep(Duration::from_millis(5)).await;
3043 }
3044 })
3045 .await;
3046 let daemon_exited = daemon.is_finished();
3047 daemon.abort();
3048 let daemon_result = daemon.await;
3049 ticker.abort();
3050
3051 assert!(
3052 published.is_ok() && !daemon_exited,
3053 "the daemon must start and publish its connection file: {daemon_result:?}"
3054 );
3055 match verdict_rx.try_recv() {
3056 Ok(true) => {}
3057 Ok(false) => panic!(
3058 "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
3059 ),
3060 Err(_) => panic!("startup published the connection file without the hooked write"),
3061 }
3062 }
3063
3064 #[tokio::test]
3067 async fn a_panicking_connection_file_write_fails_startup() {
3068 let (_dir, path) = temp_connection_file_path("publish-panics");
3069 connection_file_write_hook::install(&path, || {
3070 panic!("deliberate panic inside the connection-file write")
3071 });
3072
3073 let error = timeout(
3074 Duration::from_secs(15),
3075 run_with_config(BootstrapConfig::new(&path, 0)),
3076 )
3077 .await
3078 .expect("startup must end once the write task fails")
3079 .expect_err("startup must fail when the connection-file write panics");
3080
3081 assert!(
3082 matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
3083 "expected a connection-file write error, got {error:?}"
3084 );
3085 let message = error.to_string();
3086 assert!(message.contains(&path.display().to_string()), "{message}");
3087 assert!(message.contains("panicked"), "{message}");
3088 assert!(!path.exists(), "no connection file may be published");
3089 }
3090
3091 #[cfg(target_os = "macos")]
3092 #[test]
3093 fn nofile_raise_worker() {
3094 let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
3095 return;
3096 };
3097 let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
3098 rlimit::Resource::NOFILE.set(256, hard).unwrap();
3099 raise_nofile_limit_with_ceiling(Some(10240));
3100 std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
3101 }
3102
3103 #[cfg(target_os = "macos")]
3104 #[test]
3105 fn macos_nofile_raise_clamps_to_kernel_ceiling() {
3106 let dir = ScratchDir::new("nofile-ceiling");
3107 let result = dir.join("soft");
3108 let status = std::process::Command::new(std::env::current_exe().unwrap())
3109 .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
3110 .env("SUBC_NOFILE_TEST_FILE", &result)
3111 .env("XDG_DATA_HOME", dir.path())
3112 .env("XDG_RUNTIME_DIR", dir.path())
3113 .env("XDG_CONFIG_HOME", dir.path())
3114 .status()
3115 .unwrap();
3116 assert!(status.success());
3117 let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
3118 assert!(
3119 soft > 256 && soft <= 10240,
3120 "the inherited limit must be raised within the kernel ceiling; got {soft}"
3121 );
3122 }
3123
3124 #[tokio::test]
3125 async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
3126 let (_dir, path) = temp_connection_file_path("key-rotation");
3127 let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3128 let old_info = first.connection_info.clone();
3129 let fixed_port = old_info.endpoints[0].port;
3130 drop(first.listeners);
3131
3132 let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
3133 let new_info = second.connection_info.clone();
3134 assert_ne!(old_info.key, new_info.key);
3135 assert_ne!(old_info.daemon_id, new_info.daemon_id);
3136 let server = start_server(second);
3137
3138 let mut old_stream = connect_from_info(&old_info).await.unwrap();
3139 let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
3140 assert!(
3141 old_auth.is_err(),
3142 "old key must not authenticate after restart"
3143 );
3144
3145 let reread = connection_file::read(&path).unwrap();
3146 let mut new_stream = connect_from_info(&reread).await.unwrap();
3147 authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
3148 .await
3149 .unwrap();
3150
3151 server.abort();
3152 let _ = server.await;
3153 }
3154
3155 #[cfg(unix)]
3156 #[tokio::test]
3157 async fn published_connection_file_permissions_are_owner_only() {
3158 let (_dir, path) = temp_connection_file_path("permissions");
3159 let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3160
3161 let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
3162 assert_eq!(mode, 0o600);
3163
3164 drop(bound.listeners);
3165 }
3166
3167 fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
3175 let dir = config
3176 .connection_file_path
3177 .parent()
3178 .map(Path::to_path_buf)
3179 .unwrap_or_else(|| PathBuf::from("."));
3180 #[cfg(unix)]
3181 let config = BootstrapConfig {
3182 connection_file_path: dir.join("c"),
3183 local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
3184 ..config
3185 };
3186 #[cfg(not(unix))]
3187 let _ = dir;
3188 config
3189 }
3190
3191 #[cfg(unix)]
3194 #[test]
3195 fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
3196 let _env_lock = ENV_LOCK.lock().unwrap();
3197 let root = ScratchDir::new("binary-local-socket");
3198 let data_home = root.join("data");
3199 let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3200 let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3201 let _port = EnvGuard::unset(SUBC_PORT_ENV);
3202
3203 let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3204
3205 let run_dir = data_home.join("cortexkit").join("run");
3206 assert_eq!(
3207 config.local_listener.configured_dir(),
3208 Some(run_dir.as_path()),
3209 "the binary serves from its run directory"
3210 );
3211 let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
3212 .expect("the binary config names a directory");
3213 assert_eq!(socket.parent().unwrap(), run_dir);
3214 assert_eq!(
3215 socket.file_name().unwrap().to_string_lossy(),
3216 "subc.sock",
3217 "a fixed name: the run directory belongs to one data home"
3218 );
3219 }
3220
3221 #[cfg(unix)]
3228 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3229 async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
3230 let root = ScratchDir::new("in-process-no-local-socket");
3231 let data_home = root.join("data");
3232 fs::create_dir_all(&data_home).unwrap();
3233 let connection_path = root.join("c");
3234 let _data;
3235 let _config;
3236 let _port;
3237 {
3238 let _env_lock = ENV_LOCK.lock().unwrap();
3239 _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3240 _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3241 _port = EnvGuard::unset(SUBC_PORT_ENV);
3242 }
3243
3244 let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
3245
3246 assert!(bound.local_endpoint.is_none());
3247 assert!(bound.local_endpoint_unavailable.is_none());
3248 let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3249 assert!(published.local_endpoint.is_none());
3250 assert!(
3251 local_listener::socket_path_for(None, &connection_path).is_none(),
3252 "no configured directory, no socket"
3253 );
3254 let mut leftovers = Vec::new();
3255 let mut pending = vec![data_home.clone()];
3256 while let Some(dir) = pending.pop() {
3257 for entry in fs::read_dir(&dir).unwrap() {
3258 let entry = entry.unwrap();
3259 let path = entry.path();
3260 if entry.file_type().unwrap().is_dir() {
3261 pending.push(path.clone());
3262 }
3263 if entry.file_name().to_string_lossy().ends_with(".sock") {
3264 leftovers.push(path);
3265 }
3266 }
3267 }
3268 assert!(
3269 leftovers.is_empty(),
3270 "an in-process daemon must not create a socket in the data home: {leftovers:?}"
3271 );
3272 }
3273
3274 #[cfg(unix)]
3279 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3280 async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
3281 let dir = ScratchDir::new("tcp-only");
3282 let canonical = std::fs::canonicalize(&dir).unwrap();
3283 let connection_path = canonical.join("c");
3284 let deep = canonical.join("x".repeat(80));
3287 fs::create_dir(&deep).unwrap();
3288 fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
3289 let config =
3290 BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
3291 let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3292 assert!(
3293 bound.local_endpoint.is_none(),
3294 "a socket that cannot be bound is not published"
3295 );
3296 assert!(
3297 bound
3298 .local_endpoint_unavailable
3299 .as_deref()
3300 .is_some_and(|reason| reason.contains("too long")),
3301 "{:?}",
3302 bound.local_endpoint_unavailable
3303 );
3304 let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3305 assert!(published.local_endpoint.is_none());
3306 assert!(
3307 !published.info.endpoints.is_empty(),
3308 "TCP is still published"
3309 );
3310
3311 let server = start_server(bound);
3313 let info = connection_file::read(&connection_path).unwrap();
3314 let mut client = connect_from_info(&info).await.unwrap();
3315 authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
3316 .await
3317 .expect("a TCP client still completes the handshake");
3318 server.abort();
3319 let _ = server.await;
3320 }
3321
3322 #[cfg(unix)]
3325 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3326 async fn published_connection_file_names_the_served_local_endpoint() {
3327 let (_dir, path) = temp_connection_file_path("local-endpoint");
3328 let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
3332 let config =
3333 BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
3334 let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3335 let endpoint = connection_file::read_with_local_endpoint(&path)
3336 .unwrap()
3337 .local_endpoint
3338 .expect("the daemon serves a local endpoint, so the file names it");
3339 assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
3340 let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
3341 panic!("a Unix daemon publishes a socket path");
3342 };
3343 assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
3344 assert_eq!(
3345 fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
3346 0o600
3347 );
3348
3349 let router = Arc::new(Router::with_default_self_handler());
3350 let auth = auth_for(&bound.connection_info);
3351 let server = tokio::spawn(serve_local_listener(
3352 bound.local_listener.expect("the local listener is bound"),
3353 router,
3354 auth,
3355 ));
3356 let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
3357 authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
3358 .await
3359 .expect("a socket client completes the same handshake as a TCP client");
3360
3361 server.abort();
3362 drop(bound.listeners);
3363 }
3364
3365 #[cfg(unix)]
3368 #[test]
3369 fn a_socket_directory_that_is_a_file_refuses() {
3370 let dir = ScratchDir::new("local-listener-file-as-dir");
3371 let file = dir.join("not-a-directory");
3372 fs::write(&file, b"x").unwrap();
3373 let error = local_listener::bind(
3374 &LocalListenerConfig::in_socket_dir(file),
3375 &dir.join("connection.json"),
3376 )
3377 .expect_err("a path that is not a directory cannot hold the socket");
3378 assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
3379 }
3380
3381 #[cfg(unix)]
3385 #[test]
3386 fn the_socket_path_stays_within_the_unix_limit() {
3387 let run_dir = PathBuf::from("/tmp/short-run");
3388 assert!(
3389 local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
3390 "no configured directory means no socket"
3391 );
3392 let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
3393 .expect("a configured directory gets a socket");
3394 assert_eq!(socket, run_dir.join("subc.sock"));
3395 assert_eq!(
3396 socket,
3397 local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
3398 "the name does not come from the connection file"
3399 );
3400
3401 let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
3402 let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
3403 .expect("a configured directory gets a socket");
3404 assert!(
3405 long.as_os_str().len() < 104,
3406 "{} is {} bytes",
3407 long.display(),
3408 long.as_os_str().len()
3409 );
3410 }
3411}