use std::{
collections::BTreeMap,
env,
error::Error,
ffi::OsString,
fmt, fs, io,
net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
path::{Path, PathBuf},
process,
time::Duration,
};
use fs4::{FileExt, TryLockError};
use subc_protocol::PROTOCOL_VERSION;
pub use subc_transport::user_connection_token;
use subc_transport::{
authenticate_client, connection_file, generate_daemon_id, generate_key,
write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
LocalEndpoint, SCHEMA_VERSION,
};
use tokio::{
net::{TcpListener, TcpStream},
task::{JoinError, JoinHandle},
time::{sleep, timeout},
};
use tracing::{error, info, warn};
use crate::{
daemon_config::{self, ConfiguredModule, DaemonConfigError},
local_listener::{self, BoundLocalListener, LocalListenerConfig},
server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
supervise::HealthConfig,
ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
SupervisorProcessLiveness,
};
use std::sync::Arc;
#[cfg(windows)]
mod windows_connection_copy;
pub const DEFAULT_SUBC_PORT: u16 = 8757;
pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
use subc_transport::CONNECTION_FILE_NAME;
const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
const START_LOCK_RETRIES: usize = 40;
const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConnectionFileSource {
XdgRuntimeDir,
TempDirFallback,
Explicit,
}
impl ConnectionFileSource {
fn reason(self) -> &'static str {
match self {
Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
Self::Explicit => "configured path",
}
}
}
impl fmt::Display for ConnectionFileSource {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(match self {
Self::XdgRuntimeDir => "xdg_runtime_dir",
Self::TempDirFallback => "temp_dir_fallback",
Self::Explicit => "explicit",
})
}
}
#[derive(Debug, Clone, Default)]
struct AdmissionFactsConfig {
carrier_module_id: Option<String>,
targets: Option<Vec<String>>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub enum CgroupPlacementConfig {
#[default]
Disabled,
Current,
Root(PathBuf),
}
#[derive(Debug, Clone)]
pub struct BootstrapConfig {
#[cfg(feature = "test-support")]
shutdown_intent_window: Duration,
pub operator_queue_wait: Duration,
pub operator_timeout: Duration,
pub operator_stuck_grace: Duration,
#[cfg(feature = "test-support")]
pub operator_script: Option<PathBuf>,
#[cfg(feature = "test-support")]
pub operator_events: Option<PathBuf>,
pub connection_file_path: PathBuf,
pub port: u16,
pub daemon_ver: String,
configured_modules: Vec<ConfiguredModule>,
storage_config: Option<daemon_config::StorageConfig>,
admission_facts: AdmissionFactsConfig,
scope_authority_owners: Vec<String>,
privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
activity_config: daemon_config::ActivityConfig,
daemon_config_path: Option<PathBuf>,
configured_port: Option<u16>,
route_bind_relay_default_ms: Option<u64>,
reserved_capabilities: BTreeMap<String, String>,
watchdog_config: DaemonSelfWatchdogConfig,
connection_file_source: ConnectionFileSource,
#[cfg(all(test, windows))]
profile_override: Option<PathBuf>,
cgroup_placement: CgroupPlacementConfig,
capture_logs_dir: Option<PathBuf>,
terminal_journal_path: Option<PathBuf>,
machine_id_path: Option<PathBuf>,
local_listener: LocalListenerConfig,
live_children_path: Option<PathBuf>,
privacy_trampoline: Option<PathBuf>,
}
impl BootstrapConfig {
#[cfg(feature = "test-support")]
pub fn with_shutdown_intent_window(mut self, window: Duration) -> Self {
self.shutdown_intent_window = window;
self
}
pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
Self {
#[cfg(feature = "test-support")]
shutdown_intent_window: Duration::from_secs(120),
connection_file_path: connection_file_path.into(),
operator_queue_wait: Duration::from_secs(150),
operator_timeout: Duration::from_secs(120),
operator_stuck_grace: Duration::from_secs(10),
#[cfg(feature = "test-support")]
operator_script: None,
#[cfg(feature = "test-support")]
operator_events: None,
port,
daemon_ver: DAEMON_VERSION.to_owned(),
configured_modules: Vec::new(),
storage_config: None,
admission_facts: AdmissionFactsConfig::default(),
scope_authority_owners: daemon_config::default_scope_authority_owners(),
privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
activity_config: daemon_config::ActivityConfig::default(),
daemon_config_path: None,
configured_port: None,
route_bind_relay_default_ms: None,
reserved_capabilities: BTreeMap::new(),
watchdog_config: DaemonSelfWatchdogConfig::default(),
connection_file_source: ConnectionFileSource::Explicit,
#[cfg(all(test, windows))]
profile_override: None,
cgroup_placement: CgroupPlacementConfig::default(),
capture_logs_dir: None,
terminal_journal_path: None,
machine_id_path: None,
local_listener: LocalListenerConfig { socket_dir: None },
live_children_path: None,
privacy_trampoline: None,
}
}
pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
self.local_listener = LocalListenerConfig::in_socket_dir(dir);
self
}
pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
self.local_listener = config;
self
}
pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
self.live_children_path = Some(path.into());
self
}
pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
self.privacy_trampoline = Some(path.into());
self
}
pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
self.machine_id_path = Some(path.into());
self
}
pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
self.cgroup_placement = placement;
self
}
pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
self.capture_logs_dir = Some(dir.into());
self
}
pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
self.terminal_journal_path = Some(path.into());
self
}
pub fn from_env() -> Result<Self, BootstrapError> {
Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
}
pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
let machine_id_path =
crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
let config = Self::from_env()?
.with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
.with_capture_logs_dir(run_dir.join("logs"))
.with_terminal_journal_path(run_dir.join("terminals.jsonl"))
.with_live_children_record(crate::live_children::record_path(&run_dir))
.with_machine_id_path(machine_id_path);
#[cfg(feature = "test-support")]
let config = {
let mut config = config;
config.operator_script =
std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
config.operator_events =
std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
for (name, value) in [
(
"SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
&mut config.operator_queue_wait,
),
(
"SUBC_TEST_OPERATOR_TIMEOUT_MS",
&mut config.operator_timeout,
),
(
"SUBC_TEST_OPERATOR_STUCK_MS",
&mut config.operator_stuck_grace,
),
] {
if let Ok(ms) = std::env::var(name) {
if let Ok(ms) = ms.parse::<u64>() {
*value = Duration::from_millis(ms);
}
}
}
config
};
Ok(config)
}
pub fn from_env_with_daemon_config_path(
daemon_config_path: impl AsRef<Path>,
) -> Result<Self, BootstrapError> {
let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
let (daemon_config, activity_config) =
daemon_config::load_with_activity(&daemon_config_path)
.map_err(BootstrapError::DaemonConfig)?
.map(|(config, activity)| (Some(config), activity))
.unwrap_or_default();
let config_port = daemon_config.as_ref().and_then(|config| config.port);
let storage_config = daemon_config
.as_ref()
.and_then(|config| config.storage.clone());
let admission_facts_carrier_module_id = daemon_config
.as_ref()
.and_then(|config| config.admission_facts_carrier_module_id.clone());
let admission_facts_targets = daemon_config
.as_ref()
.and_then(|config| config.admission_facts_targets.clone());
let scope_authority_owners = daemon_config
.as_ref()
.map(|config| config.scope_authority_owners.clone())
.unwrap_or_else(daemon_config::default_scope_authority_owners);
let privileged_operator_prompt = daemon_config
.as_ref()
.map(|config| config.privileged_operator_prompt)
.unwrap_or_default();
let route_bind_relay_default_ms = daemon_config
.as_ref()
.and_then(|config| config.route_bind_relay_timeout_ms);
let reserved_capabilities = daemon_config
.as_ref()
.map(|config| config.reserved_capabilities.clone())
.unwrap_or_default();
let configured_modules = daemon_config
.map(|config| config.modules)
.unwrap_or_default();
let port = match env::var(SUBC_PORT_ENV) {
Ok(raw) if !raw.trim().is_empty() => {
let port = raw
.parse::<u16>()
.map_err(|source| BootstrapError::InvalidPort { raw, source })?;
if let Some(config_port) = config_port {
info!(
env = SUBC_PORT_ENV,
env_port = port,
config_port,
"SUBC_PORT overrides daemon config port"
);
}
port
}
Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
};
let (connection_file_path, connection_file_source) =
connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
Ok(Self::new(connection_file_path, port)
.with_configured_modules(configured_modules)
.with_storage_config(storage_config)
.with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
.with_scope_authority_owners(scope_authority_owners)
.with_privileged_operator_prompt(privileged_operator_prompt)
.with_activity_config(activity_config)
.with_route_bind_relay_default_ms(route_bind_relay_default_ms)
.with_reserved_capabilities(reserved_capabilities)
.with_daemon_config_source(daemon_config_path, config_port)
.with_connection_file_source(connection_file_source))
}
pub fn with_daemon_config_path(
self,
daemon_config_path: impl AsRef<Path>,
) -> Result<Self, BootstrapError> {
let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
let (daemon_config, activity_config) =
daemon_config::load_with_activity(&daemon_config_path)
.map_err(BootstrapError::DaemonConfig)?
.map(|(config, activity)| (Some(config), activity))
.unwrap_or_default();
let configured_port = daemon_config.as_ref().and_then(|config| config.port);
let storage_config = daemon_config
.as_ref()
.and_then(|config| config.storage.clone());
let admission_facts_carrier_module_id = daemon_config
.as_ref()
.and_then(|config| config.admission_facts_carrier_module_id.clone());
let admission_facts_targets = daemon_config
.as_ref()
.and_then(|config| config.admission_facts_targets.clone());
let scope_authority_owners = daemon_config
.as_ref()
.map(|config| config.scope_authority_owners.clone())
.unwrap_or_else(daemon_config::default_scope_authority_owners);
let privileged_operator_prompt = daemon_config
.as_ref()
.map(|config| config.privileged_operator_prompt)
.unwrap_or_default();
let route_bind_relay_default_ms = daemon_config
.as_ref()
.and_then(|config| config.route_bind_relay_timeout_ms);
let reserved_capabilities = daemon_config
.as_ref()
.map(|config| config.reserved_capabilities.clone())
.unwrap_or_default();
let configured_modules = daemon_config
.map(|config| config.modules)
.unwrap_or_default();
Ok(self
.with_configured_modules(configured_modules)
.with_storage_config(storage_config)
.with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
.with_scope_authority_owners(scope_authority_owners)
.with_privileged_operator_prompt(privileged_operator_prompt)
.with_activity_config(activity_config)
.with_route_bind_relay_default_ms(route_bind_relay_default_ms)
.with_reserved_capabilities(reserved_capabilities)
.with_daemon_config_source(daemon_config_path, configured_port))
}
pub fn with_configured_modules(
mut self,
modules: impl IntoIterator<Item = ConfiguredModule>,
) -> Self {
self.configured_modules = modules.into_iter().collect();
self.configured_modules
.sort_by(|left, right| left.module_id.cmp(&right.module_id));
self
}
pub fn with_storage_config(
mut self,
storage_config: Option<daemon_config::StorageConfig>,
) -> Self {
self.storage_config = storage_config;
self
}
pub fn with_admission_facts_config(
mut self,
carrier_module_id: Option<String>,
targets: Option<Vec<String>>,
) -> Self {
self.admission_facts = AdmissionFactsConfig {
carrier_module_id,
targets,
};
self
}
pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
self.scope_authority_owners = owners;
self
}
pub fn with_privileged_operator_prompt(
mut self,
mode: daemon_config::PrivilegedOperatorPrompt,
) -> Self {
self.privileged_operator_prompt = mode;
self
}
pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
self.activity_config = config;
self
}
pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
self.route_bind_relay_default_ms = ms;
self
}
pub fn with_reserved_capabilities(
mut self,
reserved_capabilities: BTreeMap<String, String>,
) -> Self {
self.reserved_capabilities = reserved_capabilities;
self
}
fn with_daemon_config_source(
mut self,
daemon_config_path: PathBuf,
configured_port: Option<u16>,
) -> Self {
self.daemon_config_path = Some(daemon_config_path);
self.configured_port = configured_port;
self
}
fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
self.connection_file_source = source;
self
}
pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
self.watchdog_config = watchdog_config;
self
}
}
#[allow(clippy::large_enum_variant)]
#[derive(Debug)]
pub enum Outcome {
AlreadyRunning,
Bound(BoundDaemon),
}
#[derive(Debug)]
pub struct BoundDaemon {
pub listeners: Vec<TcpListener>,
pub local_listener: Option<BoundLocalListener>,
pub local_endpoint_unavailable: Option<String>,
pub local_endpoint: Option<LocalEndpoint>,
pub connection_info: ConnectionInfo,
pub connection_file_path: PathBuf,
pub connection_file_source: ConnectionFileSource,
#[cfg(windows)]
connection_file_copy: Option<PathBuf>,
pub machine_id: Option<crate::machine_id::MachineId>,
run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
publication_lock: Option<StartLock>,
}
pub fn connection_file_path() -> PathBuf {
connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
}
fn connection_file_path_with_source(
runtime_dir: Option<OsString>,
) -> (PathBuf, ConnectionFileSource) {
if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
return (
PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
ConnectionFileSource::XdgRuntimeDir,
);
}
(
env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
ConnectionFileSource::TempDirFallback,
)
}
pub async fn run() -> Result<(), BootstrapError> {
run_with_config(
BootstrapConfig::from_env_for_daemon_binary()?
.with_cgroup_placement(CgroupPlacementConfig::Current),
)
.await
}
pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
#[cfg(feature = "test-support")]
let shutdown_intent_window = config.shutdown_intent_window;
let forwarding = Arc::new(ForwardingTable::default());
forwarding.operator_confirms().configure_bootstrap(&config);
let configured_modules = config.configured_modules.clone();
let storage_config = config.storage_config.clone();
let admission_facts = config.admission_facts.clone();
let scope_authority_owners = config.scope_authority_owners.clone();
let privileged_operator_prompt = config.privileged_operator_prompt;
let activity_config = config.activity_config;
let daemon_config_path = config.daemon_config_path.clone();
let configured_port = config.configured_port;
let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
let reserved_capabilities = config.reserved_capabilities.clone();
let watchdog_config = config.watchdog_config.clone();
let cgroup_placement_config = config.cgroup_placement.clone();
let capture_logs_dir = config.capture_logs_dir.clone();
let terminal_journal_path = config.terminal_journal_path.clone();
let live_children_path = config.live_children_path.clone();
let privacy_trampoline = config.privacy_trampoline.clone();
match ensure_singleton_inner(config, false).await? {
Outcome::AlreadyRunning => {
info!("subc daemon already running");
Ok(())
}
Outcome::Bound(bound) => {
#[cfg(target_os = "linux")]
let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
#[cfg(not(target_os = "linux"))]
let _ = cgroup_placement_config;
serve_bound_daemon(
bound,
forwarding,
configured_modules,
storage_config,
admission_facts,
scope_authority_owners,
privileged_operator_prompt,
activity_config,
daemon_config_path,
configured_port,
route_bind_relay_default_ms,
reserved_capabilities,
watchdog_config,
capture_logs_dir,
terminal_journal_path,
live_children_path,
privacy_trampoline,
#[cfg(feature = "test-support")]
shutdown_intent_window,
#[cfg(target_os = "linux")]
cgroup_placement,
)
.await
}
}
}
#[cfg(target_os = "linux")]
fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
let result = match config {
CgroupPlacementConfig::Disabled => return None,
CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
};
match result {
Ok(Some(placement)) => Some(placement),
Ok(None) => {
warn!(
placement = ?config,
"module cgroup placement is disabled: configured cgroup root is not delegated"
);
None
}
Err(error) => {
warn!(
placement = ?config,
error = %error,
"module cgroup placement is disabled by an unexpected cgroup probe error"
);
None
}
}
}
#[cfg(unix)]
const NOFILE_TARGET: u64 = 65536;
#[cfg(unix)]
fn raise_nofile_limit() {
#[cfg(target_os = "macos")]
let ceiling = std::process::Command::new("/usr/sbin/sysctl")
.args(["-n", "kern.maxfilesperproc"])
.output()
.ok()
.filter(|output| output.status.success())
.and_then(|output| String::from_utf8(output.stdout).ok())
.and_then(|value| value.trim().parse::<u64>().ok())
.filter(|value| *value > 0);
#[cfg(not(target_os = "macos"))]
let ceiling = None;
raise_nofile_limit_with_ceiling(ceiling);
}
#[cfg(unix)]
fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
match rlimit::Resource::NOFILE.get() {
Ok((soft, hard)) => {
let target = NOFILE_TARGET
.min(hard)
.min(kernel_ceiling.unwrap_or(u64::MAX));
if soft >= target {
return;
}
match rlimit::Resource::NOFILE.set(target, hard) {
Ok(()) => info!(
previous_soft = soft,
new_soft = target,
hard,
"raised open-file soft limit for daemon and module children"
),
Err(err) => warn!(
soft,
hard,
error = %err,
"could not raise open-file soft limit; multi-root modules may exhaust descriptors"
),
}
}
Err(err) => warn!(error = %err, "could not read open-file limit"),
}
}
#[cfg(windows)]
fn raise_nofile_limit() {
const MAXSTDIO_TARGET: u32 = 8192;
let current = rlimit::getmaxstdio();
if current >= MAXSTDIO_TARGET {
return;
}
match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
Ok(new_max) => info!(
previous = current,
new_max, "raised CRT stdio-stream limit for daemon"
),
Err(err) => warn!(
current,
error = %err,
"could not raise CRT stdio-stream limit"
),
}
}
#[cfg(not(any(unix, windows)))]
fn raise_nofile_limit() {}
pub async fn run_with_daemon_config_path(
config: BootstrapConfig,
daemon_config_path: impl AsRef<Path>,
) -> Result<(), BootstrapError> {
run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
}
#[allow(clippy::too_many_arguments)]
async fn serve_bound_daemon(
bound: BoundDaemon,
forwarding: Arc<ForwardingTable>,
configured_modules: Vec<ConfiguredModule>,
storage_config: Option<daemon_config::StorageConfig>,
admission_facts: AdmissionFactsConfig,
scope_authority_owners: Vec<String>,
privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
activity_config: daemon_config::ActivityConfig,
daemon_config_path: Option<PathBuf>,
configured_port: Option<u16>,
route_bind_relay_default_ms: Option<u64>,
reserved_capabilities: BTreeMap<String, String>,
watchdog_config: DaemonSelfWatchdogConfig,
capture_logs_dir: Option<PathBuf>,
terminal_journal_path: Option<PathBuf>,
live_children_path: Option<PathBuf>,
privacy_trampoline: Option<PathBuf>,
#[cfg(feature = "test-support")] shutdown_intent_window: Duration,
#[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
) -> Result<(), BootstrapError> {
#[cfg(unix)]
let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.map_err(BootstrapError::Signal)?;
raise_nofile_limit();
#[cfg(windows)]
if let Err(error) = subc_jobobject::ensure_root_job() {
warn!(%error, "module launches disabled: Windows root job setup failed");
}
info!(
connection_file = %bound.connection_file_path.display(),
connection_file_source = %bound.connection_file_source,
connection_file_source_reason = bound.connection_file_source.reason(),
endpoints = ?bound.connection_info.endpoints,
configured_modules = configured_modules.len(),
machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
"subc daemon starting"
);
let run_dir_lock = bound.run_dir_lock;
let publication_lock = bound.publication_lock;
if let Some(owner) = &run_dir_lock {
crate::live_children::sweep_orphans(
owner,
&crate::live_children::AdoptedPids::none(),
crate::live_children::SweepBounds::default(),
)
.await;
}
let registry = Arc::new(Registry::default());
let process_liveness = Arc::new(SupervisorProcessLiveness::new());
let supervisor_handle = SupervisorHandle::new();
#[cfg(feature = "test-support")]
supervisor_handle.set_shutdown_intent_window(shutdown_intent_window);
let connected_clients = ConnectedClients::new();
let daemon_incarnation = format!(
"{:032x}",
u128::from_be_bytes(bound.connection_info.daemon_id)
);
let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
.with_process_liveness(process_liveness.clone())
.with_forwarding(Arc::clone(&forwarding))
.with_handle(supervisor_handle.clone())
.with_connection_file_path(bound.connection_file_path.clone())
.with_daemon_incarnation(daemon_incarnation.clone());
let supervisor = match privacy_trampoline {
Some(path) => supervisor.with_privacy_trampoline(path),
None => supervisor,
};
let activity_run_dir = terminal_journal_path
.as_ref()
.and_then(|path| path.parent())
.map(Path::to_path_buf);
let supervisor = match terminal_journal_path {
Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
None => supervisor,
};
let supervisor = match capture_logs_dir {
Some(dir) => supervisor.with_capture_logs_dir(dir),
None => supervisor,
};
let supervisor = match live_children_path {
Some(path) => supervisor.with_live_children_record(path),
None => supervisor,
};
#[cfg(target_os = "linux")]
let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
let route_bind_relay_timeouts = configured_modules
.iter()
.filter_map(|module| {
module
.route_bind_relay_timeout_ms
.map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
})
.collect::<std::collections::BTreeMap<_, _>>();
let control_start_clock = crate::clock::StartClock::capture();
let mut control = ControlHandler::with_forwarding(Arc::clone(®istry), forwarding)
.with_process_liveness(process_liveness)
.with_supervisor(supervisor_handle)
.with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
.with_connected_clients(connected_clients.clone())
.with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
.with_storage_config(storage_config)
.with_machine_id(bound.machine_id.clone())
.with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
.with_scope_authority_owners(scope_authority_owners)
.with_privileged_operator_prompt(privileged_operator_prompt)
.with_route_bind_relay_timeouts(route_bind_relay_timeouts)
.with_daemon_provenance(
bound.connection_info.pid,
control_start_clock.started_at_ms(),
std::env::current_exe().ok(),
normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
)
.with_daemon_start_clock(control_start_clock)
.with_capability_config(
configured_modules
.iter()
.map(|module| (module.module_id.clone(), module.enabled)),
reserved_capabilities,
);
if let Some(run_dir) = activity_run_dir {
control =
control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
}
if let Some(ms) = route_bind_relay_default_ms {
control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
}
if let Some(config_path) = daemon_config_path {
control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
}
let control = Arc::new(control);
let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
let auth = ServerAuth::new(
bound.connection_info.key.clone(),
bound.connection_info.daemon_id,
bound.connection_info.daemon_ver.clone(),
)
.with_connected_clients(connected_clients);
let mut registered_modules = Vec::with_capacity(configured_modules.len());
for configured in configured_modules {
let enabled = configured.enabled;
let health = configured.health.clone();
let module_id = configured.module_id.clone();
let registered = match supervisor.register_configured_with_health(
configured.module_spec(),
enabled,
health.clone(),
configured.drain_timeout_ms,
configured.restart,
configured.register_timeout,
) {
Ok(registered) => registered,
Err(err) => {
error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
continue;
}
};
registered_modules.push(registered);
let default_threshold = HealthConfig::default().failure_threshold;
if enabled && health.failure_threshold > default_threshold {
warn!(
module_id = %module_id,
failure_threshold = health.failure_threshold,
default_threshold,
tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
"health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
);
}
info!(module_id = %module_id, enabled, "configured module registered");
}
control.refresh_capability_requirements();
let local_serve_task = bound.local_listener.map(|local_listener| {
AbortOnDrop::new(tokio::spawn(serve_local_listener(
local_listener,
Arc::clone(&router),
auth.clone(),
)))
});
#[cfg(windows)]
let _local_serve_task = local_serve_task;
let mut serve_task =
AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
verify_serving(&bound.connection_info).await?;
let (published, publication_lock) = publish_connection_file(
&bound.connection_file_path,
#[cfg(windows)]
bound.connection_file_copy.as_deref(),
&bound.connection_info,
bound.local_endpoint.as_ref(),
publication_lock,
)
.await;
published?;
#[cfg(test)]
startup_publication_hook::run(&bound.connection_file_path).await;
drop(publication_lock);
let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
#[cfg(target_os = "linux")]
let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
));
let _watchdog_task = AbortOnDrop::new(
DaemonSelfWatchdog::new(
bound.connection_info.clone(),
bound.connection_file_path.clone(),
)
.with_config(watchdog_config)
.spawn(),
);
for registered in registered_modules {
match supervisor.start_registered_configured(registered, None, true) {
Ok(module) => {
info!(module_id = %module.module_id(), "configured module supervised");
}
Err(err) => {
error!(error = %err, "failed to start configured module; continuing daemon startup");
}
}
}
control.refresh_capability_requirements();
Arc::clone(&control).spawn_capability_deadline_loop();
Arc::clone(&control).spawn_scope_expiry_loop();
#[cfg(unix)]
{
tokio::select! {
result = serve_task.join() => {
return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
}
_ = terminate.recv() => {}
}
supervisor.begin_daemon_shutdown();
drop(_watchdog_task);
drop(serve_task);
drop(local_serve_task);
let second_sigterm = async {
terminate.recv().await;
std::time::Instant::now()
};
tokio::pin!(second_sigterm);
let mut escalation_started = tokio::select! {
biased;
at = second_sigterm.as_mut() => {
info!("second SIGTERM: abandoning daemon shutdown wait");
Some(at)
}
result = supervisor.drain_for_daemon_shutdown() => {
if let Err(error) = result {
warn!(%error, "daemon shutdown drain failed; exiting anyway");
}
None
}
};
let already_escalated = escalation_started.is_some();
supervisor
.end_children_for_daemon_shutdown(already_escalated, async {
if already_escalated {
std::future::pending::<()>().await;
} else {
escalation_started = Some(second_sigterm.await);
}
})
.await;
if let Some(at) = escalation_started {
info!(
second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
"daemon shutdown wait ended after second SIGTERM"
);
}
info!("daemon exiting: SIGTERM shutdown complete");
Ok(())
}
#[cfg(not(unix))]
serve_task
.join()
.await
.map_err(BootstrapError::ServeJoin)?
.map_err(BootstrapError::Serve)
}
fn normalized_build_provenance(value: &str) -> Option<String> {
match value.trim() {
"" | "unavailable" => None,
value => Some(value.to_string()),
}
}
pub async fn ensure_singleton(
connection_file_path: impl AsRef<Path>,
port: u16,
) -> Result<Outcome, BootstrapError> {
ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
}
pub async fn ensure_singleton_with_config(
config: BootstrapConfig,
) -> Result<Outcome, BootstrapError> {
ensure_singleton_inner(config, true).await
}
async fn ensure_singleton_inner(
config: BootstrapConfig,
publish: bool,
) -> Result<Outcome, BootstrapError> {
#[cfg(windows)]
let copy_path = windows_connection_copy::resolve(&config);
let path = config.connection_file_path;
if stable_peer(&path).await? {
return Ok(Outcome::AlreadyRunning);
}
let lock = StartLock::acquire(&path).await?;
if stable_peer(&path).await? {
return Ok(Outcome::AlreadyRunning);
}
let run_dir_lock = config
.live_children_path
.as_deref()
.map(crate::run_dir_lock::RunDirLock::acquire)
.transpose()?;
remove_stale_connection_file_if_present(&path)?;
#[cfg(windows)]
let copy_path = windows_connection_copy::reclaim(&path, copy_path).await;
let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
None => (None, lock, run_dir_lock),
Some(machine_id_path) => {
let task_path = machine_id_path.clone();
match run_blocking_holding((lock, run_dir_lock), move || {
crate::machine_id::load_or_mint(&task_path)
})
.await
{
Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
Err(join_error) => {
return Err(BootstrapError::MachineId(
crate::machine_id::MachineIdFileError::Write {
path: machine_id_path,
source: io::Error::other(describe_blocking_task_failure(
"loading or minting the machine id",
&join_error,
)),
},
))
}
}
}
};
let (listeners, endpoints) = bind_loopback(config.port).await?;
let (local_listener, local_endpoint, local_endpoint_unavailable) =
if config.local_listener.configured_dir().is_some() {
match local_listener::bind(&config.local_listener, &path) {
Ok(local_listener) => {
let endpoint = local_listener.endpoint();
(Some(local_listener), Some(endpoint), None)
}
Err(error) => {
let reason = error.to_string();
warn!(
error = %reason,
"the local endpoint is unavailable; serving loopback TCP only"
);
(None, None, Some(reason))
}
}
} else {
(None, None, None)
};
let connection_info = ConnectionInfo {
schema: SCHEMA_VERSION,
wire_version: Some(PROTOCOL_VERSION),
endpoints,
key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
pid: process::id(),
daemon_ver: config.daemon_ver,
};
let publication_lock = if publish {
let (published, lock) = publish_connection_file(
&path,
#[cfg(windows)]
copy_path.as_deref(),
&connection_info,
local_endpoint.as_ref(),
lock,
)
.await;
if let Err(error) = published {
if let Some(local_listener) = local_listener.as_ref() {
local_listener::release(local_listener);
}
drop(listeners);
return Err(error);
}
drop(lock);
None
} else {
Some(lock)
};
Ok(Outcome::Bound(BoundDaemon {
listeners,
local_listener,
local_endpoint,
local_endpoint_unavailable,
connection_info,
connection_file_path: path,
connection_file_source: config.connection_file_source,
#[cfg(windows)]
connection_file_copy: copy_path,
machine_id,
run_dir_lock,
publication_lock,
}))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Probe {
Live,
StaleOrAbsent,
}
async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
if !matches!(probe_existing(path).await?, Probe::Live) {
return Ok(false);
}
tokio::time::sleep(Duration::from_millis(1500)).await;
Ok(matches!(probe_existing(path).await?, Probe::Live))
}
async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
let info = match connection_file::read(path) {
Ok(info) => info,
Err(source) if is_absent_or_stale_connection_file(&source) => {
return Ok(Probe::StaleOrAbsent)
}
Err(source) => {
return Err(BootstrapError::ConnectionFileRead {
path: path.to_path_buf(),
source,
})
}
};
for endpoint in &info.endpoints {
if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
return Ok(Probe::Live);
}
}
Ok(Probe::StaleOrAbsent)
}
async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
return Probe::StaleOrAbsent;
};
if !ip.is_loopback() {
return Probe::StaleOrAbsent;
}
let addr = SocketAddr::new(ip, endpoint.port);
let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => stream,
Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
};
match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
Ok(()) => Probe::Live,
Err(AuthError::DaemonIdMismatch)
| Err(AuthError::InvalidServerProof)
| Err(AuthError::UnexpectedEof { .. })
| Err(AuthError::Timeout { .. })
| Err(AuthError::JsonEncode { .. })
| Err(AuthError::JsonDecode { .. })
| Err(AuthError::Io { .. })
| Err(AuthError::MessageTooLarge { .. })
| Err(AuthError::KeyTooShort { .. })
| Err(AuthError::Random(_))
| Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
}
}
async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
if let Some(endpoint) = info.endpoints.first() {
if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
return Ok(());
}
}
Err(BootstrapError::StartupNotServing)
}
async fn run_blocking_holding<G, T>(
guards: G,
work: impl FnOnce() -> T + Send + 'static,
) -> Result<(T, G), JoinError>
where
G: Send + 'static,
T: Send + 'static,
{
tokio::task::spawn_blocking(move || {
let output = work();
(output, guards)
})
.await
}
fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
if join_error.is_panic() {
format!("the background thread {work} panicked before finishing: {join_error}")
} else {
format!(
"the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
)
}
}
async fn publish_connection_file<G: Send + 'static>(
path: &Path,
#[cfg(windows)] copy_path: Option<&Path>,
info: &ConnectionInfo,
local_endpoint: Option<&LocalEndpoint>,
guard: G,
) -> (Result<(), BootstrapError>, Option<G>) {
let task_path = path.to_path_buf();
let task_info = info.clone();
let task_endpoint = local_endpoint.cloned();
#[cfg(windows)]
let copy = windows_connection_copy::claim_for_publication(path, copy_path, info).await;
let outcome = run_blocking_holding(guard, move || {
write_connection_file(&task_path, &task_info, task_endpoint.as_ref())?;
#[cfg(windows)]
if let Some((copy_path, _lock)) = copy {
if let Err(error) =
write_connection_file(©_path, &task_info, task_endpoint.as_ref())
{
windows_connection_copy::warn_unavailable(&task_path, Some(©_path), &error);
}
}
Ok(())
})
.await;
match outcome {
Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
Ok((Err(source), guard)) => (
Err(BootstrapError::ConnectionFileWrite {
path: path.to_path_buf(),
source,
}),
Some(guard),
),
Err(join_error) => (
Err(BootstrapError::ConnectionFileWrite {
path: path.to_path_buf(),
source: ConnectionFileError::Io {
op: "write",
path: path.to_path_buf(),
source: io::Error::other(describe_blocking_task_failure(
"writing the connection file",
&join_error,
)),
},
}),
None,
),
}
}
fn write_connection_file(
path: &Path,
info: &ConnectionInfo,
local_endpoint: Option<&LocalEndpoint>,
) -> Result<(), ConnectionFileError> {
#[cfg(test)]
connection_file_write_hook::run(path);
write_atomic_with_local_endpoint(path, info, local_endpoint)
}
#[cfg(test)]
mod connection_file_write_hook {
use std::{
path::{Path, PathBuf},
sync::{Mutex, PoisonError},
};
type Hook = Box<dyn FnOnce() + Send>;
static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
HOOKS
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push((path.to_path_buf(), Box::new(hook)));
}
pub(super) fn run(path: &Path) {
let hook = {
let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
hooks
.iter()
.position(|(hooked, _)| hooked == path)
.map(|index| hooks.swap_remove(index).1)
};
if let Some(hook) = hook {
hook();
}
}
}
#[cfg(test)]
mod startup_publication_hook {
use std::{
path::{Path, PathBuf},
sync::{Mutex, PoisonError},
};
use tokio::sync::oneshot;
type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
let (entered_tx, entered_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
BARRIERS
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push((path.to_path_buf(), (entered_tx, release_rx)));
(entered_rx, release_tx)
}
pub(super) async fn run(path: &Path) {
let barrier = {
let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
barriers
.iter()
.position(|(hooked, _)| hooked == path)
.map(|index| barriers.swap_remove(index).1)
};
if let Some((entered, release)) = barrier {
let _ = entered.send(());
let _ = release.await;
}
}
}
fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
match err {
ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
ConnectionFileError::JsonRead { .. }
| ConnectionFileError::UnsupportedSchema { .. }
| ConnectionFileError::Invalid { .. }
| ConnectionFileError::KeyTooShort { .. }
| ConnectionFileError::InsecurePermissions { .. } => true,
ConnectionFileError::MissingParent { .. }
| ConnectionFileError::MissingFileName { .. }
| ConnectionFileError::InsecureParentDirectory { .. }
| ConnectionFileError::Io { .. }
| ConnectionFileError::JsonWrite { .. }
| ConnectionFileError::Random(_)
| ConnectionFileError::WireVersionMismatch { .. } => false,
}
}
fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
match fs::remove_file(path) {
Ok(()) => Ok(()),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
Err(source) => Err(BootstrapError::RemoveStale {
path: path.to_path_buf(),
source,
}),
}
}
async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
}
async fn bind_loopback_with<F, Fut>(
port: u16,
mut bind_v6: F,
) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
where
F: FnMut(u16) -> Fut,
Fut: std::future::Future<Output = io::Result<TcpListener>>,
{
let v4_host = Ipv4Addr::LOCALHOST;
let v4 = TcpListener::bind((v4_host, port))
.await
.map_err(|source| BootstrapError::Bind {
host: v4_host.to_string(),
port,
source,
})?;
let actual_port = v4
.local_addr()
.map_err(|source| BootstrapError::LocalAddr {
host: v4_host.to_string(),
source,
})?
.port();
let mut listeners = vec![v4];
let mut endpoints = vec![Endpoint {
host: v4_host.to_string(),
port: actual_port,
}];
let v6_host = Ipv6Addr::LOCALHOST;
match bind_v6(actual_port).await {
Ok(v6) => {
listeners.push(v6);
endpoints.push(Endpoint {
host: v6_host.to_string(),
port: actual_port,
});
}
Err(err)
if ipv6_loopback_unavailable(&err)
|| (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
{
warn!(
port = actual_port,
error = %err,
"IPv6 loopback unavailable; serving only IPv4 loopback"
);
}
Err(source) => {
drop(listeners);
return Err(BootstrapError::Bind {
host: v6_host.to_string(),
port: actual_port,
source,
});
}
}
Ok((listeners, endpoints))
}
fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
matches!(
err.kind(),
io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
}
struct AbortOnDrop<T> {
handle: JoinHandle<T>,
}
impl<T> AbortOnDrop<T> {
fn new(handle: JoinHandle<T>) -> Self {
Self { handle }
}
async fn join(&mut self) -> Result<T, JoinError> {
(&mut self.handle).await
}
}
impl<T> Drop for AbortOnDrop<T> {
fn drop(&mut self) {
if !self.handle.is_finished() {
self.handle.abort();
}
}
}
#[derive(Debug)]
struct StartLock {
_file: fs::File,
}
impl Drop for StartLock {
fn drop(&mut self) {
if let Err(error) = FileExt::unlock(&self._file) {
tracing::warn!(%error, "failed to release the singleton start lock");
}
}
}
impl StartLock {
async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
let path = start_lock_path(connection_file_path);
for _ in 0..START_LOCK_RETRIES {
let file = match open_owner_only_lock(&path) {
Ok(file) => file,
Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
};
match FileExt::try_lock(&file) {
Ok(()) => return Ok(Self { _file: file }),
Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
Err(TryLockError::Error(source)) => {
return Err(BootstrapError::StartLockCreate { path, source });
}
}
}
Err(BootstrapError::StartLockBusy {
path,
attempts: START_LOCK_RETRIES,
})
}
}
pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
let mut options = fs::OpenOptions::new();
options.read(true).write(true).create(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
options.open(path)
}
fn start_lock_path(connection_file_path: &Path) -> PathBuf {
let file_name = connection_file_path
.file_name()
.map(|name| name.to_string_lossy())
.unwrap_or_else(|| CONNECTION_FILE_NAME.into());
let lock_name = format!("{file_name}.start-lock");
connection_file_path
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."))
.join(lock_name)
}
fn non_empty_os_var(key: &str) -> Option<OsString> {
let value = env::var_os(key)?;
if value.is_empty() {
None
} else {
Some(value)
}
}
#[derive(Debug)]
pub enum BootstrapError {
StartupNotServing,
#[cfg(unix)]
Signal(io::Error),
InvalidPort {
raw: String,
source: std::num::ParseIntError,
},
ConnectionFileRead {
path: PathBuf,
source: ConnectionFileError,
},
ConnectionFileWrite {
path: PathBuf,
source: ConnectionFileError,
},
GenerateConnectionFile(ConnectionFileError),
StartLockCreate {
path: PathBuf,
source: io::Error,
},
StartLockBusy {
path: PathBuf,
attempts: usize,
},
RunDirLockCreate {
path: PathBuf,
source: io::Error,
},
RunDirBusy {
path: PathBuf,
holder_pid: Option<u32>,
},
RemoveStale {
path: PathBuf,
source: io::Error,
},
Bind {
host: String,
port: u16,
source: io::Error,
},
LocalAddr {
host: String,
source: io::Error,
},
DaemonConfig(DaemonConfigError),
MachineId(crate::machine_id::MachineIdFileError),
RunDir(daemon_config::DaemonRunDirError),
Serve(ServerError),
ServeJoin(tokio::task::JoinError),
}
impl From<ServerError> for BootstrapError {
fn from(error: ServerError) -> Self {
Self::Serve(error)
}
}
impl fmt::Display for BootstrapError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
#[cfg(unix)]
Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
Self::InvalidPort { raw, source } => {
write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
}
Self::ConnectionFileRead { path, source } => write!(
f,
"failed to read connection file {}: {source}",
path.display()
),
Self::ConnectionFileWrite { path, source } => write!(
f,
"failed to publish connection file {}: {source}",
path.display()
),
Self::GenerateConnectionFile(err) => {
write!(f, "failed to generate connection-file auth material: {err}")
}
Self::StartLockCreate { path, source } => {
write!(
f,
"failed to create start lock {}: {source}",
path.display()
)
}
Self::StartLockBusy { path, attempts } => write!(
f,
"start lock {} remained busy after {attempts} attempts",
path.display()
),
Self::RunDirLockCreate { path, source } => write!(
f,
"refusing to start: failed to lock run directory via {}: {source}",
path.display()
),
Self::RunDirBusy { path, holder_pid } => {
write!(
f,
"refusing to start: run directory lock {} is held by another daemon",
path.display()
)?;
if let Some(pid) = holder_pid {
write!(f, " (pid {pid})")?;
}
write!(
f,
"; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
)
}
Self::RemoveStale { path, source } => write!(
f,
"failed to remove stale connection file {}: {source}",
path.display()
),
Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
write!(
f,
"port {port} in use on loopback {host}: {source}; set the port in config"
)
}
Self::Bind { host, port, source } => {
write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
}
Self::LocalAddr { host, source } => {
write!(f, "failed to read local address for {host}: {source}")
}
Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
Self::MachineId(err) => write!(f, "refusing to start: {err}"),
Self::RunDir(err) => write!(f, "refusing to start: {err}"),
Self::Serve(err) => write!(f, "daemon server failed: {err}"),
Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
}
}
}
impl Error for BootstrapError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::StartupNotServing => None,
#[cfg(unix)]
Self::Signal(source) => Some(source),
Self::InvalidPort { source, .. } => Some(source),
Self::ConnectionFileRead { source, .. }
| Self::ConnectionFileWrite { source, .. }
| Self::GenerateConnectionFile(source) => Some(source),
Self::StartLockCreate { source, .. }
| Self::RunDirLockCreate { source, .. }
| Self::RemoveStale { source, .. }
| Self::Bind { source, .. }
| Self::LocalAddr { source, .. } => Some(source),
Self::DaemonConfig(err) => Some(err),
Self::MachineId(err) => Some(err),
Self::RunDir(err) => Some(err),
Self::Serve(err) => Some(err),
Self::ServeJoin(err) => Some(err),
Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
}
}
}
#[cfg(test)]
mod tests {
mod activity {
include!("activity_bootstrap_tests.rs");
}
mod startup_order {
include!("startup_order_tests.rs");
}
use super::*;
use crate::server::ServerAuth;
use cortexkit_test_support::ScratchDir;
#[cfg(target_os = "linux")]
use std::collections::BTreeSet;
use std::sync::Mutex;
#[cfg(target_os = "linux")]
use subc_control::ModuleProtocol;
use subc_transport::{write_atomic, MIN_KEY_LEN};
use tokio::io::AsyncReadExt;
use tokio::task::JoinHandle;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
static ENV_LOCK: Mutex<()> = Mutex::new(());
#[test]
fn normalized_build_provenance_preserves_real_values() {
assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
}
#[test]
fn normalized_build_provenance_omits_unavailable_and_empty_values() {
assert_eq!(normalized_build_provenance("unavailable"), None);
assert_eq!(normalized_build_provenance(""), None);
}
#[cfg(target_os = "linux")]
fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
let cgroups = fs::read_to_string("/proc/self/cgroup")?;
let relative = cgroups
.lines()
.find_map(|line| line.strip_prefix("0::"))
.ok_or_else(|| {
io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
})?;
Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
}
#[cfg(target_os = "linux")]
fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
let modules = current_cgroup_path_for_test()?.join("subc-modules");
let entries = match fs::read_dir(modules) {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
Err(error) => return Err(error),
};
let mut directories = BTreeSet::new();
for entry in entries {
let entry = entry?;
if entry.file_type()?.is_dir() {
directories.insert(entry.file_name());
}
}
Ok(directories)
}
#[cfg(target_os = "linux")]
async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
while !path.exists() && tokio::time::Instant::now() < deadline {
assert!(
!task.is_finished(),
"daemon exited before creating {}",
path.display()
);
sleep(Duration::from_millis(10)).await;
}
assert!(path.exists(), "daemon did not create {}", path.display());
}
#[cfg(target_os = "linux")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
let module_id = format!("cgroup-isolation-probe-{}", process::id());
let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
assert!(
!before.contains(&OsString::from(&module_id)),
"isolation probe cgroup already exists before this daemon starts"
);
let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
let module = ConfiguredModule {
module_id,
program: PathBuf::from("sh"),
args: vec!["-c".to_string(), "sleep 30".to_string()],
env: Vec::new(),
log: None,
enabled: true,
reserved: false,
reserved_prefixes: Vec::new(),
protocol: ModuleProtocol::None,
overlap: Default::default(),
health: HealthConfig::default(),
drain_timeout_ms: None,
register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
route_bind_relay_timeout_ms: None,
restart: RestartPolicy::default(),
};
let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
.with_configured_modules([module])
.with_capture_logs_dir(temp.join("logs"))
.with_terminal_journal_path(temp.join("terminals.jsonl"));
let task = tokio::spawn(run_with_config(config));
wait_for_path(&capture, &task).await;
let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
task.abort();
assert!(task
.await
.expect_err("aborted daemon task must cancel")
.is_cancelled());
assert_eq!(
before, after,
"an in-process daemon must not create or reconcile ambient module cgroups"
);
}
#[cfg(target_os = "linux")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
let cgroup_root = temp.join("cgroup");
fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
let modules = cgroup_root.join("subc-modules");
let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
.with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
.with_terminal_journal_path(temp.join("terminals.jsonl"));
let task = tokio::spawn(run_with_config(config));
wait_for_path(&modules, &task).await;
task.abort();
assert!(task
.await
.expect_err("aborted daemon task must cancel")
.is_cancelled());
assert!(
fs::read_dir(&modules)
.expect("read prepared modules directory")
.next()
.is_none(),
"the delegation probe must clean up after itself"
);
}
struct EnvGuard {
key: &'static str,
previous: Option<OsString>,
}
impl EnvGuard {
fn set(key: &'static str, value: &Path) -> Self {
let previous = env::var_os(key);
env::set_var(key, value);
Self { key, previous }
}
fn set_str(key: &'static str, value: &str) -> Self {
let previous = env::var_os(key);
env::set_var(key, value);
Self { key, previous }
}
fn unset(key: &'static str) -> Self {
let previous = env::var_os(key);
env::remove_var(key);
Self { key, previous }
}
}
impl Drop for EnvGuard {
fn drop(&mut self) {
match &self.previous {
Some(value) => env::set_var(self.key, value),
None => env::remove_var(self.key),
}
}
}
fn unique_temp_dir(name: &str) -> ScratchDir {
ScratchDir::new(name)
}
fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
let dir = unique_temp_dir(name);
let path = dir.join("conn.json");
(dir, path)
}
fn auth_for(info: &ConnectionInfo) -> ServerAuth {
ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
}
fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
let auth = auth_for(&bound.connection_info);
tokio::spawn(serve_listeners(
bound.listeners,
Arc::new(Router::with_default_self_handler()),
auth,
))
}
fn expect_bound(outcome: Outcome) -> BoundDaemon {
match outcome {
Outcome::Bound(bound) => bound,
Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
}
}
async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
let endpoint = conn
.endpoints
.first()
.expect("test connection file should have an endpoint");
let ip: IpAddr = endpoint.host.parse().unwrap();
TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
}
fn make_connection_info(port: u16) -> ConnectionInfo {
ConnectionInfo {
schema: SCHEMA_VERSION,
wire_version: Some(PROTOCOL_VERSION),
endpoints: vec![Endpoint {
host: "127.0.0.1".to_owned(),
port,
}],
key: generate_key().unwrap(),
daemon_id: generate_daemon_id().unwrap(),
pid: process::id(),
daemon_ver: "test-subc".to_owned(),
}
}
fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
fs::write(path, contents).unwrap();
#[cfg(unix)]
fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
}
fn assert_owner_only_connection_file(path: &Path) {
#[cfg(unix)]
{
let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
assert_eq!(mode, 0o600);
}
#[cfg(not(unix))]
let _ = path;
}
#[test]
fn connection_file_path_uses_xdg_runtime_dir_when_set() {
let _env_lock = ENV_LOCK.lock().unwrap();
let runtime_dir = unique_temp_dir("xdg-runtime");
let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
assert_eq!(
connection_file_path(),
runtime_dir.join(CONNECTION_FILE_NAME)
);
}
#[test]
fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
let runtime_dir = OsString::from("/run/user/1000");
let (path, source) = connection_file_path_with_source(Some(runtime_dir));
assert_eq!(
path,
PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
);
assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
}
#[test]
fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
let _env_lock = ENV_LOCK.lock().unwrap();
let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
assert_eq!(
connection_file_path(),
env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
);
}
#[test]
fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
let (path, source) = connection_file_path_with_source(None);
assert_eq!(
path,
env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
);
assert_eq!(source, ConnectionFileSource::TempDirFallback);
}
#[test]
fn user_connection_token_is_stable_under_concurrent_callers() {
let expected = user_connection_token();
let workers: Vec<_> = (0..32)
.map(|_| {
std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
})
.collect();
for worker in workers {
for token in worker.join().expect("probe thread") {
assert_eq!(token, expected, "token diverged under concurrent probes");
}
}
}
#[test]
fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
let (path, source) = connection_file_path_with_source(Some(OsString::new()));
assert_eq!(
path,
env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
);
assert_eq!(source, ConnectionFileSource::TempDirFallback);
}
#[test]
fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
let _env_lock = ENV_LOCK.lock().unwrap();
let root = unique_temp_dir("daemon-binary-config");
let data_home = root.join("data");
let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
let _port = EnvGuard::unset(SUBC_PORT_ENV);
let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
let run_dir = data_home.join("cortexkit").join("run");
assert_eq!(
config.terminal_journal_path,
Some(run_dir.join("terminals.jsonl"))
);
assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
assert_eq!(
BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
None,
"an in-process config must not journal anywhere unless asked to"
);
}
#[test]
fn daemon_binary_config_refuses_a_relative_data_home() {
let _env_lock = ENV_LOCK.lock().unwrap();
let root = unique_temp_dir("daemon-binary-relative-data");
let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
let _port = EnvGuard::unset(SUBC_PORT_ENV);
let error = BootstrapConfig::from_env_for_daemon_binary()
.expect_err("a relative data home must refuse the daemon binary's config");
assert!(
matches!(error, BootstrapError::RunDir(_)),
"expected a run-directory refusal, got {error}"
);
assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
}
#[test]
fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
let _env_lock = ENV_LOCK.lock().unwrap();
let _port = EnvGuard::unset(SUBC_PORT_ENV);
let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
let config_path = conn_path.with_file_name("subc.jsonc");
for (doc, expected) in [
(None, daemon_config::PrivilegedOperatorPrompt::Log),
(
Some(r#"{"version":1}"#),
daemon_config::PrivilegedOperatorPrompt::Log,
),
(
Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
daemon_config::PrivilegedOperatorPrompt::Log,
),
(
Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
daemon_config::PrivilegedOperatorPrompt::Enforce,
),
] {
if let Some(doc) = doc {
fs::write(&config_path, doc).unwrap();
}
assert_eq!(
BootstrapConfig::from_env_with_daemon_config_path(&config_path)
.unwrap()
.privileged_operator_prompt,
expected
);
assert_eq!(
BootstrapConfig::new(&conn_path, 0)
.with_daemon_config_path(&config_path)
.unwrap()
.privileged_operator_prompt,
expected
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
let connection_path = dir.join("connection.json");
let config_path = dir.join("subc.jsonc");
fs::write(
&config_path,
r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
)
.unwrap();
let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
.with_daemon_config_path(&config_path)
.unwrap();
let connection_path = config.connection_file_path.clone();
let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
tokio::time::timeout(Duration::from_secs(5), async {
while !connection_path.exists() {
assert!(
!daemon.handle.is_finished(),
"daemon exited before publishing"
);
tokio::task::yield_now().await;
}
})
.await
.unwrap();
let info = connection_file::read(&connection_path).unwrap();
let mut client = connect_from_info(&info).await.unwrap();
authenticate_client(&mut client, &info, Duration::from_secs(5))
.await
.unwrap();
for (correlation, mode, expected) in [
(1, "enforce", Vec::new()),
(2, "log", vec!["privileged_operator_prompt"]),
(3, "log", vec!["privileged_operator_prompt"]),
] {
fs::write(
&config_path,
serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
)
.unwrap();
let frame = crate::Frame::build(
subc_protocol::FrameType::Request,
subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
0,
0,
correlation,
serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
preview: true,
})
.unwrap(),
)
.unwrap();
crate::write_frame(&mut client, &frame).await.unwrap();
let reply =
tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
.await
.unwrap()
.unwrap()
.unwrap();
assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
assert_eq!(reply.header.corr, correlation);
let subc_control::ClientControlResponse::SupervisorRescan { result } =
serde_json::from_slice(&reply.body).unwrap()
else {
panic!("expected a rescan response");
};
assert_eq!(result.restart_required, expected);
}
}
#[test]
fn configured_port_uses_default_config_and_env_override() {
let _env_lock = ENV_LOCK.lock().unwrap();
let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
let config_path = conn_path.with_file_name("subc.jsonc");
let _port = EnvGuard::unset(SUBC_PORT_ENV);
assert_eq!(
BootstrapConfig::from_env_with_daemon_config_path(&config_path)
.unwrap()
.port,
DEFAULT_SUBC_PORT
);
fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
assert_eq!(
BootstrapConfig::from_env_with_daemon_config_path(&config_path)
.unwrap()
.port,
8123
);
let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
assert_eq!(
BootstrapConfig::from_env_with_daemon_config_path(&config_path)
.unwrap()
.port,
9012
);
}
#[tokio::test]
async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
let (_dir, path) = temp_connection_file_path("departing-peer");
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let info = make_connection_info(listener.local_addr().unwrap().port());
write_connection_file(&path, &info, None).unwrap();
let peer = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
subc_transport::authenticate_server(
&mut stream,
&info.key,
&info.daemon_id,
&info.daemon_ver,
PROBE_AUTH_DEADLINE,
)
.await
.unwrap();
});
let outcome = ensure_singleton(&path, 0).await.unwrap();
peer.await.unwrap();
assert!(
matches!(outcome, Outcome::Bound(_)),
"a departing peer is not AlreadyRunning"
);
}
#[tokio::test]
async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
let (_dir, path) = temp_connection_file_path("already-running");
let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
let server = start_server(bound);
let second = ensure_singleton(&path, 0).await.unwrap();
assert!(matches!(second, Outcome::AlreadyRunning));
server.abort();
let _ = server.await;
}
#[tokio::test]
async fn daemon_connection_file_publishes_protocol_wire_version() {
let (_dir, path) = temp_connection_file_path("wire-version");
let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
assert_eq!(
connection_file::read(&path).unwrap().wire_version,
Some(PROTOCOL_VERSION)
);
drop(bound.listeners);
}
#[tokio::test]
async fn stale_unbound_connection_file_is_reclaimed() {
let (_dir, path) = temp_connection_file_path("stale-reclaim");
let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
let stale_port = stale.local_addr().unwrap().port();
drop(stale);
let stale_info = make_connection_info(stale_port);
write_atomic(&path, &stale_info).unwrap();
let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
assert_ne!(bound.connection_info.key, stale_info.key);
drop(bound.listeners);
}
#[cfg(unix)]
#[tokio::test]
async fn ensure_singleton_reclaims_insecure_connection_file() {
let (_dir, path) = temp_connection_file_path("insecure-reclaim");
let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
let stale_port = stale.local_addr().unwrap().port();
drop(stale);
let stale_info = make_connection_info(stale_port);
write_atomic(&path, &stale_info).unwrap();
fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
assert_ne!(bound.connection_info.key, stale_info.key);
assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
assert_owner_only_connection_file(&path);
drop(bound.listeners);
}
#[tokio::test]
async fn ensure_singleton_reclaims_non_loopback_connection_file() {
let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
let mut stale_info = make_connection_info(8757);
stale_info.endpoints = vec![Endpoint {
host: "192.0.2.10".to_owned(),
port: 8757,
}];
write_atomic(&path, &stale_info).unwrap();
let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
assert_ne!(bound.connection_info.key, stale_info.key);
assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
assert!(bound
.connection_info
.endpoints
.iter()
.all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
assert_owner_only_connection_file(&path);
drop(bound.listeners);
}
#[tokio::test]
async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
let mut unsupported_schema = make_connection_info(8757);
unsupported_schema.schema = SCHEMA_VERSION + 1;
let mut empty_endpoints = make_connection_info(8757);
empty_endpoints.endpoints.clear();
let mut short_key = make_connection_info(8757);
short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
let cases = vec![
(
"unsupported-schema",
serde_json::to_vec(&unsupported_schema).unwrap(),
Some(unsupported_schema),
),
(
"empty-endpoints",
serde_json::to_vec(&empty_endpoints).unwrap(),
Some(empty_endpoints),
),
(
"short-key",
serde_json::to_vec(&short_key).unwrap(),
Some(short_key),
),
("invalid-json", b"{not valid connection json".to_vec(), None),
];
for (label, contents, old_info) in cases {
let (_dir, path) = temp_connection_file_path(label);
write_raw_owner_only_connection_file(&path, &contents);
let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
if let Some(old_info) = old_info {
assert_ne!(bound.connection_info.key, old_info.key, "{label}");
assert_ne!(
bound.connection_info.daemon_id, old_info.daemon_id,
"{label}"
);
}
assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
assert_owner_only_connection_file(&path);
drop(bound.listeners);
}
}
#[tokio::test]
async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
let (_dir, path) = temp_connection_file_path("foreign-reclaim");
let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
let foreign_port = foreign.local_addr().unwrap().port();
write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
let foreign_task = tokio::spawn(async move {
if let Ok((mut stream, _)) = foreign.accept().await {
let mut buf = [0u8; 64];
let _ = stream.read(&mut buf).await;
}
});
let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
assert!(bound
.connection_info
.endpoints
.iter()
.all(|endpoint| endpoint.port != foreign_port));
drop(bound.listeners);
let _ = foreign_task.await;
}
#[tokio::test]
async fn stale_start_lock_file_is_reclaimable() {
let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
let lock_path = start_lock_path(&path);
drop(open_owner_only_lock(&lock_path).unwrap());
assert!(lock_path.is_file());
let lock = StartLock::acquire(&path).await.unwrap();
assert!(lock_path.is_file());
drop(lock);
assert!(lock_path.is_file());
}
#[tokio::test]
async fn held_start_lock_blocks_second_acquire_until_release() {
let (_dir, path) = temp_connection_file_path("start-lock-held");
let lock_path = start_lock_path(&path);
let first = StartLock::acquire(&path).await.unwrap();
let err = match StartLock::acquire(&path).await {
Ok(_) => panic!("second acquire while held must stay busy"),
Err(err) => err,
};
assert!(matches!(
err,
BootstrapError::StartLockBusy {
ref path,
attempts: START_LOCK_RETRIES,
} if path == &lock_path
));
drop(first);
let second = StartLock::acquire(&path)
.await
.expect("released advisory lock should be reclaimable");
drop(second);
}
#[tokio::test]
async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
let (_dir, path) = temp_connection_file_path("bind-conflict");
let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
let occupied_port = occupied.local_addr().unwrap().port();
let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
assert!(matches!(
err,
BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
));
assert!(err.to_string().contains("set the port in config"));
drop(occupied);
}
#[tokio::test]
async fn ephemeral_ipv6_collision_still_serves_ipv4() {
let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
drop(occupied);
result
})
.await
.expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
assert_eq!(endpoints.len(), 1);
assert_eq!(endpoints[0].host, "127.0.0.1");
TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
.await
.unwrap();
drop(listeners);
}
#[tokio::test]
async fn unpublished_boot_holds_singleton_lock_until_publication() {
let dir = ScratchDir::new("ul");
let path = dir.join("c");
let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
assert!(!path.exists());
let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
assert!(
matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
"an unpublished boot must keep its singleton claim"
);
drop(bound);
FileExt::try_lock(&contender).unwrap();
}
#[tokio::test]
async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
let dir = ScratchDir::new("sl");
let path = dir.join("c");
let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
let inherited = bound
.publication_lock
.as_ref()
.unwrap()
._file
.try_clone()
.unwrap();
let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
assert!(matches!(
FileExt::try_lock(&contender),
Err(TryLockError::WouldBlock)
));
drop(bound);
FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
drop(inherited);
}
#[tokio::test]
async fn readiness_probe_requires_an_authenticating_server() {
let dir = ScratchDir::new("unserved-readiness");
let bound = expect_bound(
ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
.await
.unwrap(),
);
assert!(matches!(
verify_serving(&bound.connection_info).await,
Err(BootstrapError::StartupNotServing)
));
}
#[cfg(unix)]
#[tokio::test]
async fn orphan_sweep_finishes_before_connection_file_publication() {
use crate::live_children::{ExecutableIdentity, LiveChild};
let dir = ScratchDir::new("pas");
let path = dir.join("c");
let record = dir.join("live-children.json");
let ready = dir.join("ready");
let mut child = std::process::Command::new("/bin/sh")
.args([
"-c",
"trap '' TERM; : > \"$0\"; read -r _",
ready.to_str().unwrap(),
])
.stdin(std::process::Stdio::piped())
.env("XDG_DATA_HOME", dir.path())
.env("XDG_RUNTIME_DIR", dir.path())
.env("XDG_CONFIG_HOME", dir.path())
.spawn()
.unwrap();
timeout(Duration::from_secs(5), async {
while !ready.exists() {
sleep(Duration::from_millis(5)).await;
}
})
.await
.unwrap();
let observed = subc_os::Process::open(child.id())
.unwrap()
.unwrap()
.observe()
.unwrap();
crate::live_children::write_record(
&record,
&[LiveChild {
module_id: "old-sleep".into(),
pid: child.id(),
protocol: subc_control::ModuleProtocol::None,
start_time: Some(observed.start_time),
executable: observed.executable.map(ExecutableIdentity::from),
cgroup_name: None,
}],
)
.unwrap();
let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
.with_live_children_record(&record);
let daemon = tokio::spawn(run_with_config(config));
timeout(Duration::from_secs(5), async {
while !path.exists() {
sleep(Duration::from_millis(5)).await;
}
})
.await
.unwrap();
assert!(path.exists());
let ended = child.try_wait().unwrap().is_some();
if !ended {
child.kill().unwrap();
}
child.wait().unwrap();
if ended {
assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
}
daemon.abort();
let _ = daemon.await;
assert!(
ended,
"the published daemon must not still owe its orphan sweep"
);
}
#[tokio::test(flavor = "current_thread")]
async fn connection_file_publication_leaves_the_async_runtime_free() {
use std::sync::atomic::{AtomicBool, Ordering};
let (_dir, path) = temp_connection_file_path("publish-off-runtime");
let write_started = Arc::new(AtomicBool::new(false));
let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
let hook_started = Arc::clone(&write_started);
connection_file_write_hook::install(&path, move || {
hook_started.store(true, Ordering::SeqCst);
let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
let _ = verdict_tx.send(progressed);
});
let ticker = tokio::spawn(async move {
loop {
if write_started.load(Ordering::SeqCst) {
let _ = progress_tx.send(());
return;
}
sleep(Duration::from_millis(5)).await;
}
});
let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
let published = timeout(Duration::from_secs(15), async {
while !path.exists() && !daemon.is_finished() {
sleep(Duration::from_millis(5)).await;
}
})
.await;
let daemon_exited = daemon.is_finished();
daemon.abort();
let daemon_result = daemon.await;
ticker.abort();
assert!(
published.is_ok() && !daemon_exited,
"the daemon must start and publish its connection file: {daemon_result:?}"
);
match verdict_rx.try_recv() {
Ok(true) => {}
Ok(false) => panic!(
"no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
),
Err(_) => panic!("startup published the connection file without the hooked write"),
}
}
#[tokio::test]
async fn a_panicking_connection_file_write_fails_startup() {
let (_dir, path) = temp_connection_file_path("publish-panics");
connection_file_write_hook::install(&path, || {
panic!("deliberate panic inside the connection-file write")
});
let error = timeout(
Duration::from_secs(15),
run_with_config(BootstrapConfig::new(&path, 0)),
)
.await
.expect("startup must end once the write task fails")
.expect_err("startup must fail when the connection-file write panics");
assert!(
matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
"expected a connection-file write error, got {error:?}"
);
let message = error.to_string();
assert!(message.contains(&path.display().to_string()), "{message}");
assert!(message.contains("panicked"), "{message}");
assert!(!path.exists(), "no connection file may be published");
}
#[cfg(target_os = "macos")]
#[test]
fn nofile_raise_worker() {
let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
return;
};
let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
rlimit::Resource::NOFILE.set(256, hard).unwrap();
raise_nofile_limit_with_ceiling(Some(10240));
std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
}
#[cfg(target_os = "macos")]
#[test]
fn macos_nofile_raise_clamps_to_kernel_ceiling() {
let dir = ScratchDir::new("nofile-ceiling");
let result = dir.join("soft");
let status = std::process::Command::new(std::env::current_exe().unwrap())
.args(["--exact", "bootstrap::tests::nofile_raise_worker"])
.env("SUBC_NOFILE_TEST_FILE", &result)
.env("XDG_DATA_HOME", dir.path())
.env("XDG_RUNTIME_DIR", dir.path())
.env("XDG_CONFIG_HOME", dir.path())
.status()
.unwrap();
assert!(status.success());
let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
assert!(
soft > 256 && soft <= 10240,
"the inherited limit must be raised within the kernel ceiling; got {soft}"
);
}
#[tokio::test]
async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
let (_dir, path) = temp_connection_file_path("key-rotation");
let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
let old_info = first.connection_info.clone();
let fixed_port = old_info.endpoints[0].port;
drop(first.listeners);
let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
let new_info = second.connection_info.clone();
assert_ne!(old_info.key, new_info.key);
assert_ne!(old_info.daemon_id, new_info.daemon_id);
let server = start_server(second);
let mut old_stream = connect_from_info(&old_info).await.unwrap();
let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
assert!(
old_auth.is_err(),
"old key must not authenticate after restart"
);
let reread = connection_file::read(&path).unwrap();
let mut new_stream = connect_from_info(&reread).await.unwrap();
authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
.await
.unwrap();
server.abort();
let _ = server.await;
}
#[cfg(unix)]
#[tokio::test]
async fn published_connection_file_permissions_are_owner_only() {
let (_dir, path) = temp_connection_file_path("permissions");
let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
assert_eq!(mode, 0o600);
drop(bound.listeners);
}
fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
let dir = config
.connection_file_path
.parent()
.map(Path::to_path_buf)
.unwrap_or_else(|| PathBuf::from("."));
#[cfg(unix)]
let config = BootstrapConfig {
connection_file_path: dir.join("c"),
local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
..config
};
#[cfg(not(unix))]
let _ = dir;
config
}
#[cfg(unix)]
#[test]
fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
let _env_lock = ENV_LOCK.lock().unwrap();
let root = ScratchDir::new("binary-local-socket");
let data_home = root.join("data");
let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
let _port = EnvGuard::unset(SUBC_PORT_ENV);
let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
let run_dir = data_home.join("cortexkit").join("run");
assert_eq!(
config.local_listener.configured_dir(),
Some(run_dir.as_path()),
"the binary serves from its run directory"
);
let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
.expect("the binary config names a directory");
assert_eq!(socket.parent().unwrap(), run_dir);
assert_eq!(
socket.file_name().unwrap().to_string_lossy(),
"subc.sock",
"a fixed name: the run directory belongs to one data home"
);
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
let root = ScratchDir::new("in-process-no-local-socket");
let data_home = root.join("data");
fs::create_dir_all(&data_home).unwrap();
let connection_path = root.join("c");
let _data;
let _config;
let _port;
{
let _env_lock = ENV_LOCK.lock().unwrap();
_data = EnvGuard::set("XDG_DATA_HOME", &data_home);
_config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
_port = EnvGuard::unset(SUBC_PORT_ENV);
}
let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
assert!(bound.local_endpoint.is_none());
assert!(bound.local_endpoint_unavailable.is_none());
let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
assert!(published.local_endpoint.is_none());
assert!(
local_listener::socket_path_for(None, &connection_path).is_none(),
"no configured directory, no socket"
);
let mut leftovers = Vec::new();
let mut pending = vec![data_home.clone()];
while let Some(dir) = pending.pop() {
for entry in fs::read_dir(&dir).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if entry.file_type().unwrap().is_dir() {
pending.push(path.clone());
}
if entry.file_name().to_string_lossy().ends_with(".sock") {
leftovers.push(path);
}
}
}
assert!(
leftovers.is_empty(),
"an in-process daemon must not create a socket in the data home: {leftovers:?}"
);
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
let dir = ScratchDir::new("tcp-only");
let canonical = std::fs::canonicalize(&dir).unwrap();
let connection_path = canonical.join("c");
let deep = canonical.join("x".repeat(80));
fs::create_dir(&deep).unwrap();
fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
let config =
BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
assert!(
bound.local_endpoint.is_none(),
"a socket that cannot be bound is not published"
);
assert!(
bound
.local_endpoint_unavailable
.as_deref()
.is_some_and(|reason| reason.contains("too long")),
"{:?}",
bound.local_endpoint_unavailable
);
let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
assert!(published.local_endpoint.is_none());
assert!(
!published.info.endpoints.is_empty(),
"TCP is still published"
);
let server = start_server(bound);
let info = connection_file::read(&connection_path).unwrap();
let mut client = connect_from_info(&info).await.unwrap();
authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
.await
.expect("a TCP client still completes the handshake");
server.abort();
let _ = server.await;
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn published_connection_file_names_the_served_local_endpoint() {
let (_dir, path) = temp_connection_file_path("local-endpoint");
let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
let config =
BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
let endpoint = connection_file::read_with_local_endpoint(&path)
.unwrap()
.local_endpoint
.expect("the daemon serves a local endpoint, so the file names it");
assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
panic!("a Unix daemon publishes a socket path");
};
assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
assert_eq!(
fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
0o600
);
let router = Arc::new(Router::with_default_self_handler());
let auth = auth_for(&bound.connection_info);
let server = tokio::spawn(serve_local_listener(
bound.local_listener.expect("the local listener is bound"),
router,
auth,
));
let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
.await
.expect("a socket client completes the same handshake as a TCP client");
server.abort();
drop(bound.listeners);
}
#[cfg(unix)]
#[test]
fn a_socket_directory_that_is_a_file_refuses() {
let dir = ScratchDir::new("local-listener-file-as-dir");
let file = dir.join("not-a-directory");
fs::write(&file, b"x").unwrap();
let error = local_listener::bind(
&LocalListenerConfig::in_socket_dir(file),
&dir.join("connection.json"),
)
.expect_err("a path that is not a directory cannot hold the socket");
assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
}
#[cfg(unix)]
#[test]
fn the_socket_path_stays_within_the_unix_limit() {
let run_dir = PathBuf::from("/tmp/short-run");
assert!(
local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
"no configured directory means no socket"
);
let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
.expect("a configured directory gets a socket");
assert_eq!(socket, run_dir.join("subc.sock"));
assert_eq!(
socket,
local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
"the name does not come from the connection file"
);
let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
.expect("a configured directory gets a socket");
assert!(
long.as_os_str().len() < 104,
"{} is {} bytes",
long.display(),
long.as_os_str().len()
);
}
}