Skip to main content

subc_daemon/
bootstrap.rs

1use std::{
2    collections::BTreeMap,
3    env,
4    error::Error,
5    ffi::OsString,
6    fmt, fs, io,
7    net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8    path::{Path, PathBuf},
9    process,
10    time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17    authenticate_client, connection_file, generate_daemon_id, generate_key,
18    write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
19    LocalEndpoint, SCHEMA_VERSION,
20};
21use tokio::{
22    net::{TcpListener, TcpStream},
23    task::{JoinError, JoinHandle},
24    time::{sleep, timeout},
25};
26use tracing::{error, info, warn};
27
28use crate::{
29    daemon_config::{self, ConfiguredModule, DaemonConfigError},
30    local_listener::{self, BoundLocalListener, LocalListenerConfig},
31    server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
32    supervise::HealthConfig,
33    ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
34    ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
35    SupervisorProcessLiveness,
36};
37use std::sync::Arc;
38
39#[cfg(windows)]
40mod windows_connection_copy;
41
42pub const DEFAULT_SUBC_PORT: u16 = 8757;
43pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
44use subc_transport::CONNECTION_FILE_NAME;
45const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
46const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
47const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
48const START_LOCK_RETRIES: usize = 40;
49const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum ConnectionFileSource {
53    XdgRuntimeDir,
54    TempDirFallback,
55    Explicit,
56}
57
58impl ConnectionFileSource {
59    fn reason(self) -> &'static str {
60        match self {
61            Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
62            Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
63            Self::Explicit => "configured path",
64        }
65    }
66}
67
68impl fmt::Display for ConnectionFileSource {
69    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
70        formatter.write_str(match self {
71            Self::XdgRuntimeDir => "xdg_runtime_dir",
72            Self::TempDirFallback => "temp_dir_fallback",
73            Self::Explicit => "explicit",
74        })
75    }
76}
77
78/// Runtime bootstrap configuration. Production uses the default fixed port and
79/// optional daemon-config override; tests pass port 0 to let the OS assign a free
80/// loopback port and discover it from the connection file.
81#[derive(Debug, Clone, Default)]
82struct AdmissionFactsConfig {
83    carrier_module_id: Option<String>,
84    targets: Option<Vec<String>>,
85}
86
87/// Controls where module cgroups are prepared.
88///
89/// In-process daemons default to [`Self::Disabled`] so they never derive a
90/// production location from the host process. The shipped daemon explicitly uses
91/// [`Self::Current`]. The `ck-subc` binary accepts
92/// `SUBC_CGROUP_PLACEMENT=disabled` for isolated test processes; an unset
93/// variable retains `Current`, and every other value is rejected at startup.
94/// Tests that exercise placement can own a [`Self::Root`].
95#[derive(Debug, Clone, Default, PartialEq, Eq)]
96pub enum CgroupPlacementConfig {
97    #[default]
98    Disabled,
99    Current,
100    Root(PathBuf),
101}
102
103#[derive(Debug, Clone)]
104pub struct BootstrapConfig {
105    #[cfg(feature = "test-support")]
106    shutdown_intent_window: Duration,
107    pub operator_queue_wait: Duration,
108    pub operator_timeout: Duration,
109    pub operator_stuck_grace: Duration,
110    #[cfg(feature = "test-support")]
111    pub operator_script: Option<PathBuf>,
112    #[cfg(feature = "test-support")]
113    pub operator_events: Option<PathBuf>,
114    pub connection_file_path: PathBuf,
115    pub port: u16,
116    pub daemon_ver: String,
117    configured_modules: Vec<ConfiguredModule>,
118    storage_config: Option<daemon_config::StorageConfig>,
119    admission_facts: AdmissionFactsConfig,
120    /// Module ids whose scopes may carry the attributes that grant authority.
121    scope_authority_owners: Vec<String>,
122    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
123    activity_config: daemon_config::ActivityConfig,
124    daemon_config_path: Option<PathBuf>,
125    configured_port: Option<u16>,
126    /// Daemon-wide route.bind relay budget in milliseconds (the fallback for
127    /// any module without a per-module override). `None` = built-in default
128    /// (12s — see `control::DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`).
129    route_bind_relay_default_ms: Option<u64>,
130    reserved_capabilities: BTreeMap<String, String>,
131    watchdog_config: DaemonSelfWatchdogConfig,
132    connection_file_source: ConnectionFileSource,
133    #[cfg(all(test, windows))]
134    profile_override: Option<PathBuf>,
135    /// Where module cgroups are prepared. Disabled unless a caller explicitly
136    /// opts in, because `Current` derives a host location from `/proc/self/cgroup`.
137    cgroup_placement: CgroupPlacementConfig,
138    /// Directory the supervisor writes per-module stdout/stderr capture files
139    /// into. `None` disables capture; the shipped binary supplies its real run
140    /// directory explicitly.
141    capture_logs_dir: Option<PathBuf>,
142    /// File the supervisor appends every module's terminal exits to, so exit
143    /// history survives a daemon restart. `None` keeps terminal history in
144    /// memory only (each module's ring). Absent by default for the same reason
145    /// as `capture_logs_dir`: an in-process daemon booted by a test must not
146    /// append its exits to the operator's real `terminals.jsonl`, where they
147    /// would show up in `ck module terminals`. The shipped binary supplies
148    /// `<run dir>/terminals.jsonl` explicitly.
149    terminal_journal_path: Option<PathBuf>,
150    /// Where the machine id is read from, or minted into when absent. `None`
151    /// serves no machine id. Absent by default for the same reason as
152    /// `capture_logs_dir`: an in-process daemon booted by a test must never
153    /// derive the operator's real data home and mint into it. The shipped binary
154    /// supplies `<data home>/cortexkit/machine-id` explicitly.
155    machine_id_path: Option<PathBuf>,
156
157    /// Where this daemon's same-user local listener lives (a Unix-domain socket
158    /// on macOS and Linux; nothing on Windows, where the pipe name is minted per
159    /// start). The default serves no local endpoint, so an in-process daemon
160    /// never touches the operator's run directory. The shipped binary names its
161    /// run directory; [`BootstrapConfig::with_local_listener_socket_dir`] names
162    /// any other, and an in-process daemon that wants the endpoint names one.
163    local_listener: LocalListenerConfig,
164    /// The live-children record: every supervised process this daemon has
165    /// running, kept so the next daemon can end the ones a crash left behind.
166    /// At startup, before any module is spawned, the previous daemon's record
167    /// here is swept. `None` keeps no record and sweeps nothing, for the same
168    /// reason as `capture_logs_dir`: an in-process daemon booted by a test
169    /// must never signal processes listed in the operator's real run
170    /// directory. The shipped binary supplies `<run dir>/live-children.json`.
171    live_children_path: Option<PathBuf>,
172    privacy_trampoline: Option<PathBuf>,
173}
174
175impl BootstrapConfig {
176    /// Shorten restart intent validity in daemon integration fixtures.
177    #[cfg(feature = "test-support")]
178    pub fn with_shutdown_intent_window(mut self, window: Duration) -> Self {
179        self.shutdown_intent_window = window;
180        self
181    }
182
183    pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
184        Self {
185            #[cfg(feature = "test-support")]
186            shutdown_intent_window: Duration::from_secs(120),
187            connection_file_path: connection_file_path.into(),
188            operator_queue_wait: Duration::from_secs(150),
189            operator_timeout: Duration::from_secs(120),
190            operator_stuck_grace: Duration::from_secs(10),
191            #[cfg(feature = "test-support")]
192            operator_script: None,
193            #[cfg(feature = "test-support")]
194            operator_events: None,
195            port,
196            daemon_ver: DAEMON_VERSION.to_owned(),
197            configured_modules: Vec::new(),
198            storage_config: None,
199            admission_facts: AdmissionFactsConfig::default(),
200            scope_authority_owners: daemon_config::default_scope_authority_owners(),
201            privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
202            activity_config: daemon_config::ActivityConfig::default(),
203            daemon_config_path: None,
204            configured_port: None,
205            route_bind_relay_default_ms: None,
206            reserved_capabilities: BTreeMap::new(),
207            watchdog_config: DaemonSelfWatchdogConfig::default(),
208            connection_file_source: ConnectionFileSource::Explicit,
209            #[cfg(all(test, windows))]
210            profile_override: None,
211            cgroup_placement: CgroupPlacementConfig::default(),
212            capture_logs_dir: None,
213            terminal_journal_path: None,
214            machine_id_path: None,
215            local_listener: LocalListenerConfig { socket_dir: None },
216            live_children_path: None,
217            privacy_trampoline: None,
218        }
219    }
220
221    /// Keep the local listener's socket in `dir`. Embedding daemons and tests
222    /// pass a path inside their own fixture tree.
223    pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
224        self.local_listener = LocalListenerConfig::in_socket_dir(dir);
225        self
226    }
227
228    /// Set where the local listener lives, as [`LocalListenerConfig`] describes.
229    pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
230        self.local_listener = config;
231        self
232    }
233
234    /// Keep the live-children record at `path`, and at startup end the
235    /// processes a previous daemon recorded there that are still running.
236    /// Embedding daemons and tests pass a path inside their own fixture tree.
237    pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
238        self.live_children_path = Some(path.into());
239        self
240    }
241
242    /// macOS module launches require an explicit executable implementing the
243    /// subc-os trampoline entry point. Other systems ignore this setting.
244    pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
245        self.privacy_trampoline = Some(path.into());
246        self
247    }
248
249    /// Serve the machine id stored at `path`, minting it there at startup when
250    /// the file is absent. Embedding daemons and tests pass a path inside their
251    /// own fixture tree.
252    pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
253        self.machine_id_path = Some(path.into());
254        self
255    }
256
257    /// Selects module cgroup placement. The default is disabled.
258    pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
259        self.cgroup_placement = placement;
260        self
261    }
262
263    /// Redirects per-module stdout/stderr capture files out of the real run
264    /// directory. Tests that start an in-process daemon must call this with a
265    /// path inside their fixture tree.
266    pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
267        self.capture_logs_dir = Some(dir.into());
268        self
269    }
270
271    /// Redirects the daemon-private journal, allowing embedded daemons and tests
272    /// to keep their observations out of the operator's live run directory.
273    pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
274        self.terminal_journal_path = Some(path.into());
275        self
276    }
277
278    pub fn from_env() -> Result<Self, BootstrapError> {
279        Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
280    }
281
282    /// The SHIPPED BINARY's config, which is the only caller that should capture
283    /// child output into the operator's real run directory.
284    ///
285    /// Kept separate from `from_env` deliberately: see `capture_logs_dir` and
286    /// `terminal_journal_path` on this struct for why an absent value must mean
287    /// NO CAPTURE and NO JOURNAL rather than the operator's real run directory.
288    /// A run directory that cannot be resolved (a relative data home) refuses
289    /// startup instead of landing under the working directory.
290    pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
291        let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
292        let machine_id_path =
293            crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
294        let config = Self::from_env()?
295            .with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
296            .with_capture_logs_dir(run_dir.join("logs"))
297            .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
298            .with_live_children_record(crate::live_children::record_path(&run_dir))
299            .with_machine_id_path(machine_id_path);
300        #[cfg(feature = "test-support")]
301        let config = {
302            let mut config = config;
303            config.operator_script =
304                std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
305            config.operator_events =
306                std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
307            for (name, value) in [
308                (
309                    "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
310                    &mut config.operator_queue_wait,
311                ),
312                (
313                    "SUBC_TEST_OPERATOR_TIMEOUT_MS",
314                    &mut config.operator_timeout,
315                ),
316                (
317                    "SUBC_TEST_OPERATOR_STUCK_MS",
318                    &mut config.operator_stuck_grace,
319                ),
320            ] {
321                if let Ok(ms) = std::env::var(name) {
322                    if let Ok(ms) = ms.parse::<u64>() {
323                        *value = Duration::from_millis(ms);
324                    }
325                }
326            }
327            config
328        };
329        Ok(config)
330    }
331
332    pub fn from_env_with_daemon_config_path(
333        daemon_config_path: impl AsRef<Path>,
334    ) -> Result<Self, BootstrapError> {
335        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
336        let (daemon_config, activity_config) =
337            daemon_config::load_with_activity(&daemon_config_path)
338                .map_err(BootstrapError::DaemonConfig)?
339                .map(|(config, activity)| (Some(config), activity))
340                .unwrap_or_default();
341        let config_port = daemon_config.as_ref().and_then(|config| config.port);
342        let storage_config = daemon_config
343            .as_ref()
344            .and_then(|config| config.storage.clone());
345        let admission_facts_carrier_module_id = daemon_config
346            .as_ref()
347            .and_then(|config| config.admission_facts_carrier_module_id.clone());
348        let admission_facts_targets = daemon_config
349            .as_ref()
350            .and_then(|config| config.admission_facts_targets.clone());
351        let scope_authority_owners = daemon_config
352            .as_ref()
353            .map(|config| config.scope_authority_owners.clone())
354            .unwrap_or_else(daemon_config::default_scope_authority_owners);
355        let privileged_operator_prompt = daemon_config
356            .as_ref()
357            .map(|config| config.privileged_operator_prompt)
358            .unwrap_or_default();
359        let route_bind_relay_default_ms = daemon_config
360            .as_ref()
361            .and_then(|config| config.route_bind_relay_timeout_ms);
362        let reserved_capabilities = daemon_config
363            .as_ref()
364            .map(|config| config.reserved_capabilities.clone())
365            .unwrap_or_default();
366        let configured_modules = daemon_config
367            .map(|config| config.modules)
368            .unwrap_or_default();
369
370        let port = match env::var(SUBC_PORT_ENV) {
371            Ok(raw) if !raw.trim().is_empty() => {
372                let port = raw
373                    .parse::<u16>()
374                    .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
375                if let Some(config_port) = config_port {
376                    info!(
377                        env = SUBC_PORT_ENV,
378                        env_port = port,
379                        config_port,
380                        "SUBC_PORT overrides daemon config port"
381                    );
382                }
383                port
384            }
385            Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
386        };
387
388        let (connection_file_path, connection_file_source) =
389            connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
390        Ok(Self::new(connection_file_path, port)
391            .with_configured_modules(configured_modules)
392            .with_storage_config(storage_config)
393            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
394            .with_scope_authority_owners(scope_authority_owners)
395            .with_privileged_operator_prompt(privileged_operator_prompt)
396            .with_activity_config(activity_config)
397            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
398            .with_reserved_capabilities(reserved_capabilities)
399            .with_daemon_config_source(daemon_config_path, config_port)
400            .with_connection_file_source(connection_file_source))
401    }
402
403    pub fn with_daemon_config_path(
404        self,
405        daemon_config_path: impl AsRef<Path>,
406    ) -> Result<Self, BootstrapError> {
407        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
408        let (daemon_config, activity_config) =
409            daemon_config::load_with_activity(&daemon_config_path)
410                .map_err(BootstrapError::DaemonConfig)?
411                .map(|(config, activity)| (Some(config), activity))
412                .unwrap_or_default();
413        let configured_port = daemon_config.as_ref().and_then(|config| config.port);
414        let storage_config = daemon_config
415            .as_ref()
416            .and_then(|config| config.storage.clone());
417        let admission_facts_carrier_module_id = daemon_config
418            .as_ref()
419            .and_then(|config| config.admission_facts_carrier_module_id.clone());
420        let admission_facts_targets = daemon_config
421            .as_ref()
422            .and_then(|config| config.admission_facts_targets.clone());
423        let scope_authority_owners = daemon_config
424            .as_ref()
425            .map(|config| config.scope_authority_owners.clone())
426            .unwrap_or_else(daemon_config::default_scope_authority_owners);
427        let privileged_operator_prompt = daemon_config
428            .as_ref()
429            .map(|config| config.privileged_operator_prompt)
430            .unwrap_or_default();
431        let route_bind_relay_default_ms = daemon_config
432            .as_ref()
433            .and_then(|config| config.route_bind_relay_timeout_ms);
434        let reserved_capabilities = daemon_config
435            .as_ref()
436            .map(|config| config.reserved_capabilities.clone())
437            .unwrap_or_default();
438        let configured_modules = daemon_config
439            .map(|config| config.modules)
440            .unwrap_or_default();
441        Ok(self
442            .with_configured_modules(configured_modules)
443            .with_storage_config(storage_config)
444            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
445            .with_scope_authority_owners(scope_authority_owners)
446            .with_privileged_operator_prompt(privileged_operator_prompt)
447            .with_activity_config(activity_config)
448            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
449            .with_reserved_capabilities(reserved_capabilities)
450            .with_daemon_config_source(daemon_config_path, configured_port))
451    }
452
453    pub fn with_configured_modules(
454        mut self,
455        modules: impl IntoIterator<Item = ConfiguredModule>,
456    ) -> Self {
457        self.configured_modules = modules.into_iter().collect();
458        self.configured_modules
459            .sort_by(|left, right| left.module_id.cmp(&right.module_id));
460        self
461    }
462
463    pub fn with_storage_config(
464        mut self,
465        storage_config: Option<daemon_config::StorageConfig>,
466    ) -> Self {
467        self.storage_config = storage_config;
468        self
469    }
470
471    pub fn with_admission_facts_config(
472        mut self,
473        carrier_module_id: Option<String>,
474        targets: Option<Vec<String>>,
475    ) -> Self {
476        self.admission_facts = AdmissionFactsConfig {
477            carrier_module_id,
478            targets,
479        };
480        self
481    }
482
483    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
484    /// Absent from a config, this is `daemon_config::default_scope_authority_owners`.
485    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
486        self.scope_authority_owners = owners;
487        self
488    }
489
490    /// Choose the privileged-operation policy for the daemon's entire lifetime.
491    pub fn with_privileged_operator_prompt(
492        mut self,
493        mode: daemon_config::PrivilegedOperatorPrompt,
494    ) -> Self {
495        self.privileged_operator_prompt = mode;
496        self
497    }
498
499    pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
500        self.activity_config = config;
501        self
502    }
503
504    /// Set the daemon-wide route.bind relay default (the fallback for any
505    /// module without a per-module override). `None` preserves the built-in
506    /// default (12s). `serve_bound_daemon` reads this at startup and threads
507    /// it into the control handler's daemon-wide field.
508    pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
509        self.route_bind_relay_default_ms = ms;
510        self
511    }
512
513    pub fn with_reserved_capabilities(
514        mut self,
515        reserved_capabilities: BTreeMap<String, String>,
516    ) -> Self {
517        self.reserved_capabilities = reserved_capabilities;
518        self
519    }
520
521    fn with_daemon_config_source(
522        mut self,
523        daemon_config_path: PathBuf,
524        configured_port: Option<u16>,
525    ) -> Self {
526        self.daemon_config_path = Some(daemon_config_path);
527        self.configured_port = configured_port;
528        self
529    }
530
531    fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
532        self.connection_file_source = source;
533        self
534    }
535
536    pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
537        self.watchdog_config = watchdog_config;
538        self
539    }
540}
541
542/// Result of singleton discovery.
543// Built once per daemon start and immediately matched, so the size gap between
544// the variants costs nothing worth a box on the public shape.
545#[allow(clippy::large_enum_variant)]
546#[derive(Debug)]
547pub enum Outcome {
548    /// A live daemon authenticated from the connection file; this invocation should exit 0.
549    AlreadyRunning,
550    /// This process won the singleton race, owns bound loopback listener(s), and
551    /// has published a fresh connection file.
552    Bound(BoundDaemon),
553}
554
555#[derive(Debug)]
556pub struct BoundDaemon {
557    pub listeners: Vec<TcpListener>,
558    /// The same-user local listener, bound and not yet served. `None` when
559    /// binding it failed, in which case `local_endpoint` is `None` and
560    /// `local_endpoint_unavailable` says why.
561    pub local_listener: Option<BoundLocalListener>,
562    /// Why the daemon serves no local endpoint, when binding it failed. Reported
563    /// by `server.describe` so an operator can see it.
564    pub local_endpoint_unavailable: Option<String>,
565    /// What the connection file publishes under `local_endpoint`, so a client can
566    /// find the local listener without knowing how it was bound.
567    pub local_endpoint: Option<LocalEndpoint>,
568    pub connection_info: ConnectionInfo,
569    pub connection_file_path: PathBuf,
570    pub connection_file_source: ConnectionFileSource,
571    #[cfg(windows)]
572    connection_file_copy: Option<PathBuf>,
573    /// The machine id this daemon serves, established before any connection is
574    /// accepted. `None` when the config named no machine id path.
575    pub machine_id: Option<crate::machine_id::MachineId>,
576    /// Ownership of the run directory, held until the daemon exits so no other
577    /// daemon sweeps or writes this one's run state. `None` when the config
578    /// keeps no live-children record, and so has no run directory to own.
579    run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
580    /// The singleton start lock, kept when binding deferred publishing the
581    /// connection file. It stays held until the daemon has verified it is serving
582    /// and written the connection file, so a second daemon starting meanwhile
583    /// cannot claim the singleton too. `None` when the connection
584    /// file was already written at bind time.
585    publication_lock: Option<StartLock>,
586}
587
588/// Resolve subc's per-user TCP connection-file path.
589///
590/// `$XDG_RUNTIME_DIR/subc-connection.json` is preferred because the runtime
591/// directory is already per-user on Unix desktops. Without it, subc falls back
592/// to the system temp dir with a per-user token in the filename so different OS
593/// users do not collide on shared temp directories.
594pub fn connection_file_path() -> PathBuf {
595    connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
596}
597
598fn connection_file_path_with_source(
599    runtime_dir: Option<OsString>,
600) -> (PathBuf, ConnectionFileSource) {
601    if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
602        return (
603            PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
604            ConnectionFileSource::XdgRuntimeDir,
605        );
606    }
607
608    (
609        env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
610        ConnectionFileSource::TempDirFallback,
611    )
612}
613
614/// Resolve, claim, and serve the per-user daemon singleton.
615///
616/// A second invocation is successful: if a live daemon authenticates from the
617/// existing connection file, this returns `Ok(())` after logging and the caller
618/// exits with status 0.
619pub async fn run() -> Result<(), BootstrapError> {
620    // `run` is a binary entry point, so it opts into the production cgroup and
621    // child-log locations. In-process callers keep both features disabled by default.
622    run_with_config(
623        BootstrapConfig::from_env_for_daemon_binary()?
624            .with_cgroup_placement(CgroupPlacementConfig::Current),
625    )
626    .await
627}
628
629/// Serve a daemon from an explicit config. This is the entry point the twelve
630/// sibling repos use to boot an in-process daemon in their integration tests.
631///
632/// # THIS INSTALLS NO TRACING SUBSCRIBER, SO THE DAEMON IS SILENT BY DEFAULT
633///
634/// The daemon's own diagnostics go through `tracing`, and `tracing` DISCARDS
635/// every event when no subscriber is installed. The shipped binary installs one
636/// in `main` (`init_tracing`); this function deliberately does not, because a
637/// library that installs a global subscriber fights with whatever the host
638/// process already set up.
639///
640/// The consequence for a test harness is not "less verbose": it is that the
641/// daemon has NOTHING TO SAY about any failure, and a missing instrument reads
642/// exactly like a clean one. PLEX found this on 2026-09-18 while trying to
643/// capture daemon logs beside an intermittent bind failure, and discovered the
644/// daemon had been silent in every conformance run that repo had ever done --
645/// so the one client-side error string was all the evidence that could exist,
646/// and they had spent a real investigation on a failure whose second source was
647/// never being recorded.
648///
649/// Install one in the harness before calling this, and assert it did something
650/// (they measured 236 daemon lines with the subscriber installed, 0 with the
651/// call commented out) -- otherwise the fix is itself unverified.
652pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
653    #[cfg(feature = "test-support")]
654    let shutdown_intent_window = config.shutdown_intent_window;
655    let forwarding = Arc::new(ForwardingTable::default());
656    forwarding.operator_confirms().configure_bootstrap(&config);
657    let configured_modules = config.configured_modules.clone();
658    let storage_config = config.storage_config.clone();
659    let admission_facts = config.admission_facts.clone();
660    let scope_authority_owners = config.scope_authority_owners.clone();
661    let privileged_operator_prompt = config.privileged_operator_prompt;
662    let activity_config = config.activity_config;
663    let daemon_config_path = config.daemon_config_path.clone();
664    let configured_port = config.configured_port;
665    let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
666    let reserved_capabilities = config.reserved_capabilities.clone();
667    let watchdog_config = config.watchdog_config.clone();
668    let cgroup_placement_config = config.cgroup_placement.clone();
669    let capture_logs_dir = config.capture_logs_dir.clone();
670    let terminal_journal_path = config.terminal_journal_path.clone();
671    let live_children_path = config.live_children_path.clone();
672    let privacy_trampoline = config.privacy_trampoline.clone();
673    match ensure_singleton_inner(config, false).await? {
674        Outcome::AlreadyRunning => {
675            info!("subc daemon already running");
676            Ok(())
677        }
678        Outcome::Bound(bound) => {
679            #[cfg(target_os = "linux")]
680            let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
681            #[cfg(not(target_os = "linux"))]
682            let _ = cgroup_placement_config;
683            serve_bound_daemon(
684                bound,
685                forwarding,
686                configured_modules,
687                storage_config,
688                admission_facts,
689                scope_authority_owners,
690                privileged_operator_prompt,
691                activity_config,
692                daemon_config_path,
693                configured_port,
694                route_bind_relay_default_ms,
695                reserved_capabilities,
696                watchdog_config,
697                capture_logs_dir,
698                terminal_journal_path,
699                live_children_path,
700                privacy_trampoline,
701                #[cfg(feature = "test-support")]
702                shutdown_intent_window,
703                #[cfg(target_os = "linux")]
704                cgroup_placement,
705            )
706            .await
707        }
708    }
709}
710
711#[cfg(target_os = "linux")]
712fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
713    let result = match config {
714        CgroupPlacementConfig::Disabled => return None,
715        CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
716        CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
717    };
718
719    match result {
720        Ok(Some(placement)) => Some(placement),
721        Ok(None) => {
722            warn!(
723                placement = ?config,
724                "module cgroup placement is disabled: configured cgroup root is not delegated"
725            );
726            None
727        }
728        Err(error) => {
729            warn!(
730                placement = ?config,
731                error = %error,
732                "module cgroup placement is disabled by an unexpected cgroup probe error"
733            );
734            None
735        }
736    }
737}
738
739/// Target soft limit for open file descriptors, applied to the daemon before any
740/// module is spawned so children inherit it. Multi-root modules (one process
741/// aggregating every project root's sqlite stores, index caches, watchers, and
742/// LSP pipes) trivially exceed the macOS default soft limit of 256; a launchd
743/// user agent does not pass login-shell ulimits through, so the raise must
744/// happen in-process.
745#[cfg(unix)]
746const NOFILE_TARGET: u64 = 65536;
747
748/// Raise RLIMIT_NOFILE to `NOFILE_TARGET` (clamped to the hard limit).
749/// Best-effort: failure is logged and never fatal, since the daemon can run
750/// under the inherited limit — modules with few roots just have less headroom.
751#[cfg(unix)]
752fn raise_nofile_limit() {
753    #[cfg(target_os = "macos")]
754    let ceiling = std::process::Command::new("/usr/sbin/sysctl")
755        .args(["-n", "kern.maxfilesperproc"])
756        .output()
757        .ok()
758        .filter(|output| output.status.success())
759        .and_then(|output| String::from_utf8(output.stdout).ok())
760        .and_then(|value| value.trim().parse::<u64>().ok())
761        .filter(|value| *value > 0);
762    #[cfg(not(target_os = "macos"))]
763    let ceiling = None;
764    raise_nofile_limit_with_ceiling(ceiling);
765}
766
767#[cfg(unix)]
768fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
769    match rlimit::Resource::NOFILE.get() {
770        Ok((soft, hard)) => {
771            // On XNU an infinite hard limit does not remove the separate
772            // per-process kernel ceiling. Exceeding it makes setrlimit fail
773            // without raising the inherited (often 256) soft limit at all.
774            let target = NOFILE_TARGET
775                .min(hard)
776                .min(kernel_ceiling.unwrap_or(u64::MAX));
777            if soft >= target {
778                return;
779            }
780            match rlimit::Resource::NOFILE.set(target, hard) {
781                Ok(()) => info!(
782                    previous_soft = soft,
783                    new_soft = target,
784                    hard,
785                    "raised open-file soft limit for daemon and module children"
786                ),
787                Err(err) => warn!(
788                    soft,
789                    hard,
790                    error = %err,
791                    "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
792                ),
793            }
794        }
795        Err(err) => warn!(error = %err, "could not read open-file limit"),
796    }
797}
798
799/// CRT stdio-stream target on Windows (the `_setmaxstdio` maximum). Win32
800/// HANDLEs — what Rust `File`, tokio sockets, and SQLite's Win32 VFS actually
801/// consume — have a per-process quota in the millions and need no raise; the
802/// C-runtime stream table (default 512) is the only low ceiling, and it is
803/// per-process rather than inherited, so supervised modules linking the CRT
804/// must raise their own. Raising it here covers the daemon itself.
805#[cfg(windows)]
806fn raise_nofile_limit() {
807    const MAXSTDIO_TARGET: u32 = 8192;
808    let current = rlimit::getmaxstdio();
809    if current >= MAXSTDIO_TARGET {
810        return;
811    }
812    match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
813        Ok(new_max) => info!(
814            previous = current,
815            new_max, "raised CRT stdio-stream limit for daemon"
816        ),
817        Err(err) => warn!(
818            current,
819            error = %err,
820            "could not raise CRT stdio-stream limit"
821        ),
822    }
823}
824
825#[cfg(not(any(unix, windows)))]
826fn raise_nofile_limit() {}
827
828pub async fn run_with_daemon_config_path(
829    config: BootstrapConfig,
830    daemon_config_path: impl AsRef<Path>,
831) -> Result<(), BootstrapError> {
832    run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
833}
834
835#[allow(clippy::too_many_arguments)]
836async fn serve_bound_daemon(
837    bound: BoundDaemon,
838    forwarding: Arc<ForwardingTable>,
839    configured_modules: Vec<ConfiguredModule>,
840    storage_config: Option<daemon_config::StorageConfig>,
841    admission_facts: AdmissionFactsConfig,
842    scope_authority_owners: Vec<String>,
843    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
844    activity_config: daemon_config::ActivityConfig,
845    daemon_config_path: Option<PathBuf>,
846    configured_port: Option<u16>,
847    route_bind_relay_default_ms: Option<u64>,
848    reserved_capabilities: BTreeMap<String, String>,
849    watchdog_config: DaemonSelfWatchdogConfig,
850    capture_logs_dir: Option<PathBuf>,
851    terminal_journal_path: Option<PathBuf>,
852    live_children_path: Option<PathBuf>,
853    privacy_trampoline: Option<PathBuf>,
854    #[cfg(feature = "test-support")] shutdown_intent_window: Duration,
855    #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
856) -> Result<(), BootstrapError> {
857    #[cfg(unix)]
858    let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
859        .map_err(BootstrapError::Signal)?;
860    // Windows has no SIGTERM. Ctrl-C is a different event, not an equivalent
861    // service-stop contract, so this shutdown handler is intentionally Unix-only.
862    raise_nofile_limit();
863    #[cfg(windows)]
864    if let Err(error) = subc_jobobject::ensure_root_job() {
865        // Keep the control plane available. The root-job API caches setup
866        // failures, so later module spawns refuse instead of running unprotected.
867        warn!(%error, "module launches disabled: Windows root job setup failed");
868    }
869
870    info!(
871        connection_file = %bound.connection_file_path.display(),
872        connection_file_source = %bound.connection_file_source,
873        connection_file_source_reason = bound.connection_file_source.reason(),
874        endpoints = ?bound.connection_info.endpoints,
875        configured_modules = configured_modules.len(),
876        machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
877        "subc daemon starting"
878    );
879
880    // Before anything can spawn a module (the configured modules below, or a
881    // client's start request once the listeners are served): a previous daemon
882    // that died without its shutdown stop may have left children running, and
883    // a fresh copy beside one would fight it for its port and stores. The
884    // record is not a running daemon's because this daemon holds the lock on
885    // the run directory the record lives in, and a running daemon holds that
886    // lock for its whole life. (Claiming the singleton is not enough: it is
887    // keyed on the connection file, which can live in a different runtime
888    // directory from the run directory.) The lock stays held until this
889    // function returns, which is when the daemon exits.
890    let run_dir_lock = bound.run_dir_lock;
891    let publication_lock = bound.publication_lock;
892    if let Some(owner) = &run_dir_lock {
893        crate::live_children::sweep_orphans(
894            owner,
895            &crate::live_children::AdoptedPids::none(),
896            crate::live_children::SweepBounds::default(),
897        )
898        .await;
899    }
900
901    let registry = Arc::new(Registry::default());
902    let process_liveness = Arc::new(SupervisorProcessLiveness::new());
903    let supervisor_handle = SupervisorHandle::new();
904    #[cfg(feature = "test-support")]
905    supervisor_handle.set_shutdown_intent_window(shutdown_intent_window);
906    let connected_clients = ConnectedClients::new();
907    let daemon_incarnation = format!(
908        "{:032x}",
909        u128::from_be_bytes(bound.connection_info.daemon_id)
910    );
911    let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
912        .with_process_liveness(process_liveness.clone())
913        .with_forwarding(Arc::clone(&forwarding))
914        .with_handle(supervisor_handle.clone())
915        .with_connection_file_path(bound.connection_file_path.clone())
916        .with_daemon_incarnation(daemon_incarnation.clone());
917    let supervisor = match privacy_trampoline {
918        Some(path) => supervisor.with_privacy_trampoline(path),
919        None => supervisor,
920    };
921    // ABSENT MEANS NO CAPTURE AND NO JOURNAL, NOT "THE REAL RUN DIRECTORY", and
922    // the difference is a production-corruption hazard rather than a preference.
923    // Both fields below follow the same rule: `None` for `capture_logs_dir`
924    // means supervised output is not captured, and `None` for
925    // `terminal_journal_path` means terminal history lives only in each
926    // module's in-memory ring (`supervisor.terminals` still answers, with the
927    // journal counters at zero).
928    //
929    // The terminal journal used to fall back to
930    // `daemon_run_dir().join("terminals.jsonl")`, so an in-process test daemon
931    // appended its fixture exits to the operator's journal, where they then
932    // appeared in `ck module terminals`.
933    //
934    // The capture line used to be `unwrap_or_else(|| daemon_run_dir().join("logs"))`,
935    // so ANY caller that did not set the field captured supervised children into
936    // the operator's live `~/.local/share/cortexkit/run/logs/`. That is twelve
937    // sibling repos whose integration tests boot an in-process daemon through
938    // `run_with_config` -- none of which asked for it, and none of which can see
939    // it from their side.
940    //
941    // Harmless while fixture module ids are fixture-shaped: this host carries 20
942    // zero-byte files from subc's own tests (good-aft, missing-aft,
943    // preview-consumer...). THE HAZARD IS A COLLISION. A fixture named "broca"
944    // or "aft" appends to a PRODUCTION capture file that operators read
945    // forensically and that placement gates count lines in -- with no residue to
946    // notice, because the file legitimately exists and legitimately grows.
947    //
948    // Found by BROCA (2026-09-19) from the other side: their rigs spawn the
949    // SHIPPED ck-subc and set XDG_CONFIG_HOME + XDG_RUNTIME_DIR but not
950    // XDG_DATA_HOME, so every local rig run supervised a module named "broca"
951    // and captured it into production's broca.stderr.log -- the same file I
952    // count seal lines in before and after placing their binaries.
953    //
954    // The supervisor already treats `None` as no-capture and no-journal, so
955    // this only removes invented defaults. The binary keeps capturing and
956    // journaling via `BootstrapConfig::from_env_for_daemon_binary`.
957
958    // Keep both ledgers in the explicitly selected run directory. An embedded
959    // daemon without a journal path must not write to the operator's live ledger.
960    let activity_run_dir = terminal_journal_path
961        .as_ref()
962        .and_then(|path| path.parent())
963        .map(Path::to_path_buf);
964    let supervisor = match terminal_journal_path {
965        Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
966        None => supervisor,
967    };
968    let supervisor = match capture_logs_dir {
969        Some(dir) => supervisor.with_capture_logs_dir(dir),
970        None => supervisor,
971    };
972    let supervisor = match live_children_path {
973        Some(path) => supervisor.with_live_children_record(path),
974        None => supervisor,
975    };
976    #[cfg(target_os = "linux")]
977    let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
978    // Collect per-module route.bind relay overrides BEFORE handing the
979    // `configured_modules` vector to the supervisor (which only needs each
980    // module's `drain_timeout_ms`). Each entry was filled in by parse-time
981    // resolution (per-module > daemon-wide > absent), so modules with no
982    // override are absent from this map and the daemon-wide default applies.
983    let route_bind_relay_timeouts = configured_modules
984        .iter()
985        .filter_map(|module| {
986            module
987                .route_bind_relay_timeout_ms
988                .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
989        })
990        .collect::<std::collections::BTreeMap<_, _>>();
991    let control_start_clock = crate::clock::StartClock::capture();
992    let mut control = ControlHandler::with_forwarding(Arc::clone(&registry), forwarding)
993        .with_process_liveness(process_liveness)
994        .with_supervisor(supervisor_handle)
995        // The same flag `begin_daemon_shutdown` sets below, so requirement
996        // changes while modules exit during the shutdown log at INFO.
997        .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
998        .with_connected_clients(connected_clients.clone())
999        .with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
1000        .with_storage_config(storage_config)
1001        .with_machine_id(bound.machine_id.clone())
1002        .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
1003        .with_scope_authority_owners(scope_authority_owners)
1004        .with_privileged_operator_prompt(privileged_operator_prompt)
1005        .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
1006        .with_daemon_provenance(
1007            bound.connection_info.pid,
1008            control_start_clock.started_at_ms(),
1009            std::env::current_exe().ok(),
1010            normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
1011            normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
1012        )
1013        .with_daemon_start_clock(control_start_clock)
1014        .with_capability_config(
1015            configured_modules
1016                .iter()
1017                .map(|module| (module.module_id.clone(), module.enabled)),
1018            reserved_capabilities,
1019        );
1020    if let Some(run_dir) = activity_run_dir {
1021        control =
1022            control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
1023    }
1024    if let Some(ms) = route_bind_relay_default_ms {
1025        // A daemon-wide config value overrides the built-in default; a
1026        // `None` here leaves the ControlHandler's 12s default in place.
1027        control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
1028    }
1029    if let Some(config_path) = daemon_config_path {
1030        control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
1031    }
1032    let control = Arc::new(control);
1033    let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
1034    let auth = ServerAuth::new(
1035        bound.connection_info.key.clone(),
1036        bound.connection_info.daemon_id,
1037        bound.connection_info.daemon_ver.clone(),
1038    )
1039    .with_connected_clients(connected_clients);
1040
1041    // Discovery must expose every accepted configured module, including disabled
1042    // ones. Registration creates status handles but neither starts monitors
1043    // nor launches processes: children need the published file for `--subc`.
1044    let mut registered_modules = Vec::with_capacity(configured_modules.len());
1045    for configured in configured_modules {
1046        let enabled = configured.enabled;
1047        let health = configured.health.clone();
1048        let module_id = configured.module_id.clone();
1049        let registered = match supervisor.register_configured_with_health(
1050            configured.module_spec(),
1051            enabled,
1052            health.clone(),
1053            configured.drain_timeout_ms,
1054            configured.restart,
1055            configured.register_timeout,
1056        ) {
1057            Ok(registered) => registered,
1058            Err(err) => {
1059                error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
1060                continue;
1061            }
1062        };
1063        registered_modules.push(registered);
1064        // A relaxed threshold can silently outlive the temporary allowance
1065        // that required it, so report the extra tolerance on every boot.
1066        let default_threshold = HealthConfig::default().failure_threshold;
1067        if enabled && health.failure_threshold > default_threshold {
1068            warn!(
1069                module_id = %module_id,
1070                failure_threshold = health.failure_threshold,
1071                default_threshold,
1072                tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1073                "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1074            );
1075        }
1076        info!(module_id = %module_id, enabled, "configured module registered");
1077    }
1078    control.refresh_capability_requirements();
1079
1080    // The local endpoint is served beside TCP and answers the same handshake.
1081    // It serves clients only, which the server enforces per connection.
1082    let local_serve_task = bound.local_listener.map(|local_listener| {
1083        AbortOnDrop::new(tokio::spawn(serve_local_listener(
1084            local_listener,
1085            Arc::clone(&router),
1086            auth.clone(),
1087        )))
1088    });
1089    // On Unix the task is dropped below to stop accepting at shutdown. Windows
1090    // leaves it to the process exit; naming it keeps the binding uniform.
1091    #[cfg(windows)]
1092    let _local_serve_task = local_serve_task;
1093    let mut serve_task =
1094        AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1095    verify_serving(&bound.connection_info).await?;
1096    // A published endpoint promises that authentication can complete now.
1097    // Orphan cleanup and control-plane construction can take longer than an
1098    // existing-daemon probe's deadline, so finish both before discovery.
1099    // The write fsyncs, so it runs on tokio's blocking pool (see
1100    // `publish_connection_file`). The singleton lock travels with it and comes
1101    // back only once the file is in place.
1102    let (published, publication_lock) = publish_connection_file(
1103        &bound.connection_file_path,
1104        #[cfg(windows)]
1105        bound.connection_file_copy.as_deref(),
1106        &bound.connection_info,
1107        bound.local_endpoint.as_ref(),
1108        publication_lock,
1109    )
1110    .await;
1111    published?;
1112    #[cfg(test)]
1113    startup_publication_hook::run(&bound.connection_file_path).await;
1114    drop(publication_lock);
1115    let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1116    // Off the startup path: it runs `systemctl`, and only ever logs.
1117    #[cfg(target_os = "linux")]
1118    let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1119        crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1120    ));
1121    let _watchdog_task = AbortOnDrop::new(
1122        DaemonSelfWatchdog::new(
1123            bound.connection_info.clone(),
1124            bound.connection_file_path.clone(),
1125        )
1126        .with_config(watchdog_config)
1127        .spawn(),
1128    );
1129
1130    for registered in registered_modules {
1131        match supervisor.start_registered_configured(registered, None, true) {
1132            Ok(module) => {
1133                info!(module_id = %module.module_id(), "configured module supervised");
1134            }
1135            Err(err) => {
1136                error!(error = %err, "failed to start configured module; continuing daemon startup");
1137            }
1138        }
1139    }
1140
1141    control.refresh_capability_requirements();
1142    Arc::clone(&control).spawn_capability_deadline_loop();
1143    Arc::clone(&control).spawn_scope_expiry_loop();
1144
1145    #[cfg(unix)]
1146    {
1147        tokio::select! {
1148            result = serve_task.join() => {
1149                return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1150            }
1151            _ = terminate.recv() => {}
1152        }
1153        // First, before the notice, the drain, or any connection close: from
1154        // here on a module exit is recorded as `daemon_shutdown` and never
1155        // respawned. Also before allowing a second signal to cut the bounded
1156        // wait short, so the journal marker is always written.
1157        supervisor.begin_daemon_shutdown();
1158        // Stop the self-watchdog before closing the listener. Its next tick would
1159        // connect to that listener, fail, and log an ERROR indistinguishable from
1160        // a wedged daemon, once for every interval a planned stop lasts.
1161        drop(_watchdog_task);
1162        // Dropping the listener stops new accepts, not established connections:
1163        // their detached tasks must remain live throughout notice and drain. The
1164        // local endpoint stops accepting the same way.
1165        drop(serve_task);
1166        drop(local_serve_task);
1167        let second_sigterm = async {
1168            terminate.recv().await;
1169            std::time::Instant::now()
1170        };
1171        tokio::pin!(second_sigterm);
1172        let mut escalation_started = tokio::select! {
1173            biased;
1174            at = second_sigterm.as_mut() => {
1175                info!("second SIGTERM: abandoning daemon shutdown wait");
1176                Some(at)
1177            }
1178            result = supervisor.drain_for_daemon_shutdown() => {
1179                if let Err(error) = result {
1180                    warn!(%error, "daemon shutdown drain failed; exiting anyway");
1181                }
1182                None
1183            }
1184        };
1185        // Supervised modules lead their own process groups, so a service
1186        // manager's kill of this process's group does not reach them. (A
1187        // systemd unit with KillMode=control-group kills by cgroup instead and
1188        // does reach them; see `systemd_kill_mode`.) The daemon ends them
1189        // itself: EOF (or SIGTERM for a protocol none child) first, then
1190        // signals at each child's own drain deadline.
1191        let already_escalated = escalation_started.is_some();
1192        supervisor
1193            .end_children_for_daemon_shutdown(already_escalated, async {
1194                if already_escalated {
1195                    std::future::pending::<()>().await;
1196                } else {
1197                    escalation_started = Some(second_sigterm.await);
1198                }
1199            })
1200            .await;
1201        if let Some(at) = escalation_started {
1202            info!(
1203                second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1204                "daemon shutdown wait ended after second SIGTERM"
1205            );
1206        }
1207        info!("daemon exiting: SIGTERM shutdown complete");
1208        Ok(())
1209    }
1210    #[cfg(not(unix))]
1211    serve_task
1212        .join()
1213        .await
1214        .map_err(BootstrapError::ServeJoin)?
1215        .map_err(BootstrapError::Serve)
1216}
1217
1218fn normalized_build_provenance(value: &str) -> Option<String> {
1219    match value.trim() {
1220        "" | "unavailable" => None,
1221        value => Some(value.to_string()),
1222    }
1223}
1224
1225/// Find an existing daemon or atomically bind loopback TCP for this daemon.
1226///
1227/// The algorithm is intentionally connect-first: an endpoint from the connection
1228/// file is treated as live only after the TCP+key server-proof authenticates for
1229/// that file's key and daemon_id. Stale or foreign connection files are reclaimed
1230/// only while holding the per-user start lock; the TCP port is never the
1231/// singleton primitive.
1232pub async fn ensure_singleton(
1233    connection_file_path: impl AsRef<Path>,
1234    port: u16,
1235) -> Result<Outcome, BootstrapError> {
1236    ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1237}
1238
1239pub async fn ensure_singleton_with_config(
1240    config: BootstrapConfig,
1241) -> Result<Outcome, BootstrapError> {
1242    ensure_singleton_inner(config, true).await
1243}
1244
1245async fn ensure_singleton_inner(
1246    config: BootstrapConfig,
1247    publish: bool,
1248) -> Result<Outcome, BootstrapError> {
1249    #[cfg(windows)]
1250    let copy_path = windows_connection_copy::resolve(&config);
1251    let path = config.connection_file_path;
1252
1253    if stable_peer(&path).await? {
1254        return Ok(Outcome::AlreadyRunning);
1255    }
1256
1257    let lock = StartLock::acquire(&path).await?;
1258
1259    // Re-probe after acquiring the start lock so a peer that won the race between
1260    // our first failed probe and the lock acquisition is observed instead of
1261    // overwritten.
1262    if stable_peer(&path).await? {
1263        return Ok(Outcome::AlreadyRunning);
1264    }
1265
1266    // The start lock and the probe above only rule out a daemon using this
1267    // connection file. A daemon started with another runtime directory but
1268    // the same data home shares the run directory while passing both, so the
1269    // run directory needs its own owner: refuse at once if a live daemon holds
1270    // it, before anything below reads or writes run or data-home state.
1271    let run_dir_lock = config
1272        .live_children_path
1273        .as_deref()
1274        .map(crate::run_dir_lock::RunDirLock::acquire)
1275        .transpose()?;
1276
1277    remove_stale_connection_file_if_present(&path)?;
1278    #[cfg(windows)]
1279    let copy_path = windows_connection_copy::reclaim(&path, copy_path).await;
1280
1281    // Established under the start lock, and under the run-directory lock when
1282    // there is one, and before binding: a corrupt file stops boot before
1283    // anything is published, and two daemons racing to start cannot both mint.
1284    //
1285    // Minting a new machine id fsyncs its file, so loading or minting runs on
1286    // tokio's blocking pool. Both locks travel with that work, so neither is
1287    // released while a mint is in flight.
1288    let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1289        None => (None, lock, run_dir_lock),
1290        Some(machine_id_path) => {
1291            let task_path = machine_id_path.clone();
1292            match run_blocking_holding((lock, run_dir_lock), move || {
1293                crate::machine_id::load_or_mint(&task_path)
1294            })
1295            .await
1296            {
1297                Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1298                Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1299                Err(join_error) => {
1300                    return Err(BootstrapError::MachineId(
1301                        crate::machine_id::MachineIdFileError::Write {
1302                            path: machine_id_path,
1303                            source: io::Error::other(describe_blocking_task_failure(
1304                                "loading or minting the machine id",
1305                                &join_error,
1306                            )),
1307                        },
1308                    ))
1309                }
1310            }
1311        }
1312    };
1313
1314    let (listeners, endpoints) = bind_loopback(config.port).await?;
1315    // The local endpoint is an addition to the daemon, not a condition for it:
1316    // a bind failure leaves clients on TCP only, with the connection file naming
1317    // no local endpoint, and the reason reported by `server.describe`. A daemon
1318    // that refused to start would take every module down for a feature nothing
1319    // has to use yet.
1320    let (local_listener, local_endpoint, local_endpoint_unavailable) =
1321        if config.local_listener.configured_dir().is_some() {
1322            match local_listener::bind(&config.local_listener, &path) {
1323                Ok(local_listener) => {
1324                    let endpoint = local_listener.endpoint();
1325                    (Some(local_listener), Some(endpoint), None)
1326                }
1327                Err(error) => {
1328                    let reason = error.to_string();
1329                    warn!(
1330                        error = %reason,
1331                        "the local endpoint is unavailable; serving loopback TCP only"
1332                    );
1333                    (None, None, Some(reason))
1334                }
1335            }
1336        } else {
1337            (None, None, None)
1338        };
1339    let connection_info = ConnectionInfo {
1340        schema: SCHEMA_VERSION,
1341        wire_version: Some(PROTOCOL_VERSION),
1342        endpoints,
1343        key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1344        daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1345        pid: process::id(),
1346        daemon_ver: config.daemon_ver,
1347    };
1348
1349    // The low-level binding API historically publishes for callers that serve
1350    // these listeners themselves. The daemon entry point defers publication
1351    // until its own authentication server is running.
1352    let publication_lock = if publish {
1353        let (published, lock) = publish_connection_file(
1354            &path,
1355            #[cfg(windows)]
1356            copy_path.as_deref(),
1357            &connection_info,
1358            local_endpoint.as_ref(),
1359            lock,
1360        )
1361        .await;
1362        if let Err(error) = published {
1363            if let Some(local_listener) = local_listener.as_ref() {
1364                local_listener::release(local_listener);
1365            }
1366            drop(listeners);
1367            return Err(error);
1368        }
1369        drop(lock);
1370        None
1371    } else {
1372        Some(lock)
1373    };
1374
1375    Ok(Outcome::Bound(BoundDaemon {
1376        listeners,
1377        local_listener,
1378        local_endpoint,
1379        local_endpoint_unavailable,
1380        connection_info,
1381        connection_file_path: path,
1382        connection_file_source: config.connection_file_source,
1383        #[cfg(windows)]
1384        connection_file_copy: copy_path,
1385        machine_id,
1386        run_dir_lock,
1387        publication_lock,
1388    }))
1389}
1390
1391#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1392enum Probe {
1393    Live,
1394    StaleOrAbsent,
1395}
1396
1397async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1398    if !matches!(probe_existing(path).await?, Probe::Live) {
1399        return Ok(false);
1400    }
1401    // A terminating peer can still authenticate while its listeners drain.
1402    // Require another proof after a short settling interval before reporting
1403    // a successful duplicate start. If the peer left, proceed with startup;
1404    // any remaining bind/ownership conflict is an error, not exit 0 (which
1405    // would suppress launchd's restart-on-failure policy).
1406    tokio::time::sleep(Duration::from_millis(1500)).await;
1407    Ok(matches!(probe_existing(path).await?, Probe::Live))
1408}
1409
1410async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1411    let info = match connection_file::read(path) {
1412        Ok(info) => info,
1413        Err(source) if is_absent_or_stale_connection_file(&source) => {
1414            return Ok(Probe::StaleOrAbsent)
1415        }
1416        Err(source) => {
1417            return Err(BootstrapError::ConnectionFileRead {
1418                path: path.to_path_buf(),
1419                source,
1420            })
1421        }
1422    };
1423
1424    for endpoint in &info.endpoints {
1425        if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1426            return Ok(Probe::Live);
1427        }
1428    }
1429
1430    Ok(Probe::StaleOrAbsent)
1431}
1432
1433async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1434    let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1435        return Probe::StaleOrAbsent;
1436    };
1437    if !ip.is_loopback() {
1438        return Probe::StaleOrAbsent;
1439    }
1440    let addr = SocketAddr::new(ip, endpoint.port);
1441
1442    let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1443        Ok(Ok(stream)) => stream,
1444        Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1445    };
1446
1447    match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1448        Ok(()) => Probe::Live,
1449        Err(AuthError::DaemonIdMismatch)
1450        | Err(AuthError::InvalidServerProof)
1451        | Err(AuthError::UnexpectedEof { .. })
1452        | Err(AuthError::Timeout { .. })
1453        | Err(AuthError::JsonEncode { .. })
1454        | Err(AuthError::JsonDecode { .. })
1455        | Err(AuthError::Io { .. })
1456        | Err(AuthError::MessageTooLarge { .. })
1457        | Err(AuthError::KeyTooShort { .. })
1458        | Err(AuthError::Random(_))
1459        | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1460    }
1461}
1462
1463async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1464    // Scheduling a server task is not a readiness barrier. Authenticate once
1465    // against the mandatory IPv4 listener before exposing discovery to peers.
1466    // This uses the normal probe deadline, with no retries or widened budget.
1467    if let Some(endpoint) = info.endpoints.first() {
1468        if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1469            return Ok(());
1470        }
1471    }
1472    Err(BootstrapError::StartupNotServing)
1473}
1474
1475/// Run blocking filesystem work (an fsync, a rename) on tokio's blocking-thread
1476/// pool rather than on the async worker thread that awaits it. On a slow or
1477/// busy disk an fsync can take seconds, and a runtime has few worker threads
1478/// (an in-process test daemon may share two with everything else), so doing it
1479/// inline stalls unrelated tasks for that long.
1480///
1481/// `guards` (lock handles) move into the blocking task and are handed back
1482/// with the work's result. If the awaiting future is dropped first, the work
1483/// still runs to completion on its thread and the guards are released only
1484/// after it ends, so a lock is never given up while the work it protects is
1485/// still writing.
1486///
1487/// `Err` means the task panicked, or was cancelled before it started because
1488/// the runtime is shutting down; the guards have then been released already.
1489async fn run_blocking_holding<G, T>(
1490    guards: G,
1491    work: impl FnOnce() -> T + Send + 'static,
1492) -> Result<(T, G), JoinError>
1493where
1494    G: Send + 'static,
1495    T: Send + 'static,
1496{
1497    tokio::task::spawn_blocking(move || {
1498        let output = work();
1499        (output, guards)
1500    })
1501    .await
1502}
1503
1504/// Explain, for a startup error, why blocking work produced no result.
1505fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1506    if join_error.is_panic() {
1507        format!("the background thread {work} panicked before finishing: {join_error}")
1508    } else {
1509        format!(
1510            "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1511        )
1512    }
1513}
1514
1515/// Publish the connection file: owner-only permissions, written to a temporary
1516/// file, fsynced and renamed over `path` (all done by `write_atomic`). The
1517/// write runs on tokio's blocking pool through [`run_blocking_holding`], so the
1518/// fsync never holds an async worker thread.
1519///
1520/// `guard` is the singleton start lock (or whatever the caller must keep held
1521/// across publication); it is returned with the result. It comes back `None`
1522/// only when the write task panicked or was cancelled, in which case the
1523/// result is an error and the file must be treated as not published.
1524///
1525/// On Windows, `copy_path` asks for the profile copy as well. Its start lock is
1526/// taken before the primary write. Once `path` is written, an error returned
1527/// while preparing or writing the copy only logs a warning, and the result is
1528/// still success. The two writes share one blocking task but aren't one atomic
1529/// step, and a panic or cancellation of that task is still a publication
1530/// error.
1531async fn publish_connection_file<G: Send + 'static>(
1532    path: &Path,
1533    #[cfg(windows)] copy_path: Option<&Path>,
1534    info: &ConnectionInfo,
1535    local_endpoint: Option<&LocalEndpoint>,
1536    guard: G,
1537) -> (Result<(), BootstrapError>, Option<G>) {
1538    let task_path = path.to_path_buf();
1539    let task_info = info.clone();
1540    let task_endpoint = local_endpoint.cloned();
1541    #[cfg(windows)]
1542    let copy = windows_connection_copy::claim_for_publication(path, copy_path, info).await;
1543    let outcome = run_blocking_holding(guard, move || {
1544        write_connection_file(&task_path, &task_info, task_endpoint.as_ref())?;
1545        #[cfg(windows)]
1546        if let Some((copy_path, _lock)) = copy {
1547            if let Err(error) =
1548                write_connection_file(&copy_path, &task_info, task_endpoint.as_ref())
1549            {
1550                windows_connection_copy::warn_unavailable(&task_path, Some(&copy_path), &error);
1551            }
1552        }
1553        Ok(())
1554    })
1555    .await;
1556    match outcome {
1557        Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1558        Ok((Err(source), guard)) => (
1559            Err(BootstrapError::ConnectionFileWrite {
1560                path: path.to_path_buf(),
1561                source,
1562            }),
1563            Some(guard),
1564        ),
1565        Err(join_error) => (
1566            Err(BootstrapError::ConnectionFileWrite {
1567                path: path.to_path_buf(),
1568                source: ConnectionFileError::Io {
1569                    op: "write",
1570                    path: path.to_path_buf(),
1571                    source: io::Error::other(describe_blocking_task_failure(
1572                        "writing the connection file",
1573                        &join_error,
1574                    )),
1575                },
1576            }),
1577            None,
1578        ),
1579    }
1580}
1581
1582/// The synchronous connection-file write, including its fsync. Call it only
1583/// through [`publish_connection_file`], which keeps it off the async workers.
1584///
1585/// `local_endpoint` is written through subc-transport's endpoint-aware writer,
1586/// so the file carries `local_endpoint` when there is a local listener and is
1587/// byte-identical to before when there is none.
1588fn write_connection_file(
1589    path: &Path,
1590    info: &ConnectionInfo,
1591    local_endpoint: Option<&LocalEndpoint>,
1592) -> Result<(), ConnectionFileError> {
1593    #[cfg(test)]
1594    connection_file_write_hook::run(path);
1595    write_atomic_with_local_endpoint(path, info, local_endpoint)
1596}
1597
1598/// Lets a test run code inside the connection-file write, on whichever thread
1599/// performs it, for example to block there and check what else can still run.
1600#[cfg(test)]
1601mod connection_file_write_hook {
1602    use std::{
1603        path::{Path, PathBuf},
1604        sync::{Mutex, PoisonError},
1605    };
1606
1607    type Hook = Box<dyn FnOnce() + Send>;
1608
1609    /// Keyed by connection-file path, so a hook fires only for the test that
1610    /// installed it while other tests publish their own files in parallel.
1611    static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
1612
1613    /// Run `hook` once, at the start of the next write of `path`.
1614    pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
1615        HOOKS
1616            .lock()
1617            .unwrap_or_else(PoisonError::into_inner)
1618            .push((path.to_path_buf(), Box::new(hook)));
1619    }
1620
1621    pub(super) fn run(path: &Path) {
1622        let hook = {
1623            let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
1624            hooks
1625                .iter()
1626                .position(|(hooked, _)| hooked == path)
1627                .map(|index| hooks.swap_remove(index).1)
1628        };
1629        if let Some(hook) = hook {
1630            hook();
1631        }
1632    }
1633}
1634
1635/// A path-scoped barrier after publication and before any configured activation.
1636/// The async wait leaves the listeners free to answer discovery-time requests.
1637#[cfg(test)]
1638mod startup_publication_hook {
1639    use std::{
1640        path::{Path, PathBuf},
1641        sync::{Mutex, PoisonError},
1642    };
1643    use tokio::sync::oneshot;
1644
1645    type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
1646    static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
1647
1648    pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
1649        let (entered_tx, entered_rx) = oneshot::channel();
1650        let (release_tx, release_rx) = oneshot::channel();
1651        BARRIERS
1652            .lock()
1653            .unwrap_or_else(PoisonError::into_inner)
1654            .push((path.to_path_buf(), (entered_tx, release_rx)));
1655        (entered_rx, release_tx)
1656    }
1657
1658    pub(super) async fn run(path: &Path) {
1659        let barrier = {
1660            let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
1661            barriers
1662                .iter()
1663                .position(|(hooked, _)| hooked == path)
1664                .map(|index| barriers.swap_remove(index).1)
1665        };
1666        if let Some((entered, release)) = barrier {
1667            let _ = entered.send(());
1668            let _ = release.await;
1669        }
1670    }
1671}
1672
1673fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
1674    match err {
1675        ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
1676        ConnectionFileError::JsonRead { .. }
1677        | ConnectionFileError::UnsupportedSchema { .. }
1678        | ConnectionFileError::Invalid { .. }
1679        | ConnectionFileError::KeyTooShort { .. }
1680        // A live daemon always publishes the file owner-only (0600), so a file
1681        // with insecure permissions is never a daemon we should defer to: treat it
1682        // as stale and take over (which republishes a correct 0600 file).
1683        | ConnectionFileError::InsecurePermissions { .. } => true,
1684        ConnectionFileError::MissingParent { .. }
1685        | ConnectionFileError::MissingFileName { .. }
1686        // A writable ancestor is an operator misconfiguration, never evidence
1687        // about whether a daemon is live. Reclaiming the file would republish key
1688        // material into the same directory the refusal is about.
1689        | ConnectionFileError::InsecureParentDirectory { .. }
1690        | ConnectionFileError::Io { .. }
1691        | ConnectionFileError::JsonWrite { .. }
1692        | ConnectionFileError::Random(_)
1693        // A wire mismatch may identify a newer live daemon, so never reclaim its
1694        // connection file merely because this binary cannot speak its envelope.
1695        | ConnectionFileError::WireVersionMismatch { .. } => false,
1696    }
1697}
1698
1699fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
1700    match fs::remove_file(path) {
1701        Ok(()) => Ok(()),
1702        Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
1703        Err(source) => Err(BootstrapError::RemoveStale {
1704            path: path.to_path_buf(),
1705            source,
1706        }),
1707    }
1708}
1709
1710async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
1711    bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
1712}
1713
1714async fn bind_loopback_with<F, Fut>(
1715    port: u16,
1716    mut bind_v6: F,
1717) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
1718where
1719    F: FnMut(u16) -> Fut,
1720    Fut: std::future::Future<Output = io::Result<TcpListener>>,
1721{
1722    let v4_host = Ipv4Addr::LOCALHOST;
1723    let v4 = TcpListener::bind((v4_host, port))
1724        .await
1725        .map_err(|source| BootstrapError::Bind {
1726            host: v4_host.to_string(),
1727            port,
1728            source,
1729        })?;
1730    let actual_port = v4
1731        .local_addr()
1732        .map_err(|source| BootstrapError::LocalAddr {
1733            host: v4_host.to_string(),
1734            source,
1735        })?
1736        .port();
1737
1738    let mut listeners = vec![v4];
1739    let mut endpoints = vec![Endpoint {
1740        host: v4_host.to_string(),
1741        port: actual_port,
1742    }];
1743
1744    let v6_host = Ipv6Addr::LOCALHOST;
1745    match bind_v6(actual_port).await {
1746        Ok(v6) => {
1747            listeners.push(v6);
1748            endpoints.push(Endpoint {
1749                host: v6_host.to_string(),
1750                port: actual_port,
1751            });
1752        }
1753        Err(err)
1754            if ipv6_loopback_unavailable(&err)
1755                || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
1756        {
1757            warn!(
1758                port = actual_port,
1759                error = %err,
1760                "IPv6 loopback unavailable; serving only IPv4 loopback"
1761            );
1762        }
1763        Err(source) => {
1764            drop(listeners);
1765            return Err(BootstrapError::Bind {
1766                host: v6_host.to_string(),
1767                port: actual_port,
1768                source,
1769            });
1770        }
1771    }
1772
1773    Ok((listeners, endpoints))
1774}
1775
1776fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
1777    matches!(
1778        err.kind(),
1779        io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
1780    ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
1781}
1782
1783struct AbortOnDrop<T> {
1784    handle: JoinHandle<T>,
1785}
1786
1787impl<T> AbortOnDrop<T> {
1788    fn new(handle: JoinHandle<T>) -> Self {
1789        Self { handle }
1790    }
1791
1792    async fn join(&mut self) -> Result<T, JoinError> {
1793        (&mut self.handle).await
1794    }
1795}
1796
1797impl<T> Drop for AbortOnDrop<T> {
1798    fn drop(&mut self) {
1799        if !self.handle.is_finished() {
1800            self.handle.abort();
1801        }
1802    }
1803}
1804
1805#[derive(Debug)]
1806struct StartLock {
1807    // Keep the locked file handle alive for the duration of bootstrap.
1808    _file: fs::File,
1809}
1810
1811impl Drop for StartLock {
1812    fn drop(&mut self) {
1813        // Closing alone leaves the lock held if a concurrent fork inherited
1814        // the open file description and has not exec'd yet. Unlock explicitly
1815        // so a restart can reclaim the singleton as soon as bootstrap ends.
1816        if let Err(error) = FileExt::unlock(&self._file) {
1817            tracing::warn!(%error, "failed to release the singleton start lock");
1818        }
1819    }
1820}
1821
1822impl StartLock {
1823    async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
1824        let path = start_lock_path(connection_file_path);
1825        for _ in 0..START_LOCK_RETRIES {
1826            let file = match open_owner_only_lock(&path) {
1827                Ok(file) => file,
1828                Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
1829            };
1830            match FileExt::try_lock(&file) {
1831                Ok(()) => return Ok(Self { _file: file }),
1832                Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
1833                Err(TryLockError::Error(source)) => {
1834                    return Err(BootstrapError::StartLockCreate { path, source });
1835                }
1836            }
1837        }
1838
1839        Err(BootstrapError::StartLockBusy {
1840            path,
1841            attempts: START_LOCK_RETRIES,
1842        })
1843    }
1844}
1845
1846pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
1847    let mut options = fs::OpenOptions::new();
1848    options.read(true).write(true).create(true);
1849    #[cfg(unix)]
1850    {
1851        use std::os::unix::fs::OpenOptionsExt;
1852        options.mode(0o600);
1853    }
1854    options.open(path)
1855}
1856
1857fn start_lock_path(connection_file_path: &Path) -> PathBuf {
1858    let file_name = connection_file_path
1859        .file_name()
1860        .map(|name| name.to_string_lossy())
1861        .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
1862    let lock_name = format!("{file_name}.start-lock");
1863    connection_file_path
1864        .parent()
1865        .filter(|parent| !parent.as_os_str().is_empty())
1866        .unwrap_or_else(|| Path::new("."))
1867        .join(lock_name)
1868}
1869
1870fn non_empty_os_var(key: &str) -> Option<OsString> {
1871    let value = env::var_os(key)?;
1872    if value.is_empty() {
1873        None
1874    } else {
1875        Some(value)
1876    }
1877}
1878
1879/// Bootstrap-layer errors are deliberately typed so startup never panics for
1880/// ordinary daemon-discovery races or stale filesystem state.
1881#[derive(Debug)]
1882pub enum BootstrapError {
1883    /// The bound server could not authenticate its own readiness probe.
1884    StartupNotServing,
1885    #[cfg(unix)]
1886    Signal(io::Error),
1887    InvalidPort {
1888        raw: String,
1889        source: std::num::ParseIntError,
1890    },
1891    ConnectionFileRead {
1892        path: PathBuf,
1893        source: ConnectionFileError,
1894    },
1895    ConnectionFileWrite {
1896        path: PathBuf,
1897        source: ConnectionFileError,
1898    },
1899    GenerateConnectionFile(ConnectionFileError),
1900    StartLockCreate {
1901        path: PathBuf,
1902        source: io::Error,
1903    },
1904    StartLockBusy {
1905        path: PathBuf,
1906        attempts: usize,
1907    },
1908    /// The run-directory lock file could not be created, opened or locked.
1909    RunDirLockCreate {
1910        path: PathBuf,
1911        source: io::Error,
1912    },
1913    /// Another live process, almost certainly a daemon started with a
1914    /// different runtime directory over the same data home, owns the run
1915    /// directory. The daemon does not start. `holder_pid` is read from the
1916    /// lock file when possible and is informational only.
1917    RunDirBusy {
1918        path: PathBuf,
1919        holder_pid: Option<u32>,
1920    },
1921    RemoveStale {
1922        path: PathBuf,
1923        source: io::Error,
1924    },
1925    Bind {
1926        host: String,
1927        port: u16,
1928        source: io::Error,
1929    },
1930    LocalAddr {
1931        host: String,
1932        source: io::Error,
1933    },
1934    DaemonConfig(DaemonConfigError),
1935    /// The machine id could not be established: its file is corrupt, unreadable
1936    /// or unwritable, or the data home is relative. The daemon does not start.
1937    MachineId(crate::machine_id::MachineIdFileError),
1938    /// The daemon run directory could not be resolved because the data home is
1939    /// relative. The daemon does not start.
1940    RunDir(daemon_config::DaemonRunDirError),
1941    Serve(ServerError),
1942    ServeJoin(tokio::task::JoinError),
1943}
1944
1945impl From<ServerError> for BootstrapError {
1946    fn from(error: ServerError) -> Self {
1947        Self::Serve(error)
1948    }
1949}
1950
1951impl fmt::Display for BootstrapError {
1952    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1953        match self {
1954            Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
1955            #[cfg(unix)]
1956            Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
1957            Self::InvalidPort { raw, source } => {
1958                write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
1959            }
1960            Self::ConnectionFileRead { path, source } => write!(
1961                f,
1962                "failed to read connection file {}: {source}",
1963                path.display()
1964            ),
1965            Self::ConnectionFileWrite { path, source } => write!(
1966                f,
1967                "failed to publish connection file {}: {source}",
1968                path.display()
1969            ),
1970            Self::GenerateConnectionFile(err) => {
1971                write!(f, "failed to generate connection-file auth material: {err}")
1972            }
1973            Self::StartLockCreate { path, source } => {
1974                write!(
1975                    f,
1976                    "failed to create start lock {}: {source}",
1977                    path.display()
1978                )
1979            }
1980            Self::StartLockBusy { path, attempts } => write!(
1981                f,
1982                "start lock {} remained busy after {attempts} attempts",
1983                path.display()
1984            ),
1985            Self::RunDirLockCreate { path, source } => write!(
1986                f,
1987                "refusing to start: failed to lock run directory via {}: {source}",
1988                path.display()
1989            ),
1990            Self::RunDirBusy { path, holder_pid } => {
1991                write!(
1992                    f,
1993                    "refusing to start: run directory lock {} is held by another daemon",
1994                    path.display()
1995                )?;
1996                if let Some(pid) = holder_pid {
1997                    write!(f, " (pid {pid})")?;
1998                }
1999                write!(
2000                    f,
2001                    "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
2002                )
2003            }
2004            Self::RemoveStale { path, source } => write!(
2005                f,
2006                "failed to remove stale connection file {}: {source}",
2007                path.display()
2008            ),
2009            Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
2010                write!(
2011                    f,
2012                    "port {port} in use on loopback {host}: {source}; set the port in config"
2013                )
2014            }
2015            Self::Bind { host, port, source } => {
2016                write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
2017            }
2018            Self::LocalAddr { host, source } => {
2019                write!(f, "failed to read local address for {host}: {source}")
2020            }
2021            Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
2022            Self::MachineId(err) => write!(f, "refusing to start: {err}"),
2023            Self::RunDir(err) => write!(f, "refusing to start: {err}"),
2024            Self::Serve(err) => write!(f, "daemon server failed: {err}"),
2025            Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
2026        }
2027    }
2028}
2029
2030impl Error for BootstrapError {
2031    fn source(&self) -> Option<&(dyn Error + 'static)> {
2032        match self {
2033            Self::StartupNotServing => None,
2034            #[cfg(unix)]
2035            Self::Signal(source) => Some(source),
2036            Self::InvalidPort { source, .. } => Some(source),
2037            Self::ConnectionFileRead { source, .. }
2038            | Self::ConnectionFileWrite { source, .. }
2039            | Self::GenerateConnectionFile(source) => Some(source),
2040            Self::StartLockCreate { source, .. }
2041            | Self::RunDirLockCreate { source, .. }
2042            | Self::RemoveStale { source, .. }
2043            | Self::Bind { source, .. }
2044            | Self::LocalAddr { source, .. } => Some(source),
2045            Self::DaemonConfig(err) => Some(err),
2046            Self::MachineId(err) => Some(err),
2047            Self::RunDir(err) => Some(err),
2048            Self::Serve(err) => Some(err),
2049            Self::ServeJoin(err) => Some(err),
2050            Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
2051        }
2052    }
2053}
2054
2055#[cfg(test)]
2056mod tests {
2057    mod activity {
2058        include!("activity_bootstrap_tests.rs");
2059    }
2060    mod startup_order {
2061        include!("startup_order_tests.rs");
2062    }
2063    use super::*;
2064    use crate::server::ServerAuth;
2065    use cortexkit_test_support::ScratchDir;
2066    #[cfg(target_os = "linux")]
2067    use std::collections::BTreeSet;
2068    use std::sync::Mutex;
2069    #[cfg(target_os = "linux")]
2070    use subc_control::ModuleProtocol;
2071    use subc_transport::{write_atomic, MIN_KEY_LEN};
2072    use tokio::io::AsyncReadExt;
2073    use tokio::task::JoinHandle;
2074
2075    #[cfg(unix)]
2076    use std::os::unix::fs::PermissionsExt;
2077
2078    static ENV_LOCK: Mutex<()> = Mutex::new(());
2079
2080    #[test]
2081    fn normalized_build_provenance_preserves_real_values() {
2082        assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
2083    }
2084
2085    #[test]
2086    fn normalized_build_provenance_omits_unavailable_and_empty_values() {
2087        assert_eq!(normalized_build_provenance("unavailable"), None);
2088        assert_eq!(normalized_build_provenance(""), None);
2089    }
2090
2091    #[cfg(target_os = "linux")]
2092    fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
2093        let cgroups = fs::read_to_string("/proc/self/cgroup")?;
2094        let relative = cgroups
2095            .lines()
2096            .find_map(|line| line.strip_prefix("0::"))
2097            .ok_or_else(|| {
2098                io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
2099            })?;
2100        Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
2101    }
2102
2103    #[cfg(target_os = "linux")]
2104    fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
2105        let modules = current_cgroup_path_for_test()?.join("subc-modules");
2106        let entries = match fs::read_dir(modules) {
2107            Ok(entries) => entries,
2108            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
2109            Err(error) => return Err(error),
2110        };
2111        let mut directories = BTreeSet::new();
2112        for entry in entries {
2113            let entry = entry?;
2114            if entry.file_type()?.is_dir() {
2115                directories.insert(entry.file_name());
2116            }
2117        }
2118        Ok(directories)
2119    }
2120
2121    #[cfg(target_os = "linux")]
2122    async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
2123        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2124        while !path.exists() && tokio::time::Instant::now() < deadline {
2125            assert!(
2126                !task.is_finished(),
2127                "daemon exited before creating {}",
2128                path.display()
2129            );
2130            sleep(Duration::from_millis(10)).await;
2131        }
2132        assert!(path.exists(), "daemon did not create {}", path.display());
2133    }
2134
2135    /// An in-process daemon with the default (disabled) cgroup placement must
2136    /// leave the host's live module cgroup tree exactly as it found it.
2137    /// Red-checking this test (making it fail) creates a
2138    /// `cgroup-isolation-probe-*` directory in the live cgroup tree, which must be removed by hand.
2139    #[cfg(target_os = "linux")]
2140    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2141    async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
2142        let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
2143        let module_id = format!("cgroup-isolation-probe-{}", process::id());
2144        let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
2145        assert!(
2146            !before.contains(&OsString::from(&module_id)),
2147            "isolation probe cgroup already exists before this daemon starts"
2148        );
2149        let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
2150        let module = ConfiguredModule {
2151            module_id,
2152            program: PathBuf::from("sh"),
2153            args: vec!["-c".to_string(), "sleep 30".to_string()],
2154            env: Vec::new(),
2155            log: None,
2156            enabled: true,
2157            reserved: false,
2158            reserved_prefixes: Vec::new(),
2159            protocol: ModuleProtocol::None,
2160            overlap: Default::default(),
2161            health: HealthConfig::default(),
2162            drain_timeout_ms: None,
2163            register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2164            route_bind_relay_timeout_ms: None,
2165            restart: RestartPolicy::default(),
2166        };
2167        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2168            .with_configured_modules([module])
2169            .with_capture_logs_dir(temp.join("logs"))
2170            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2171        let task = tokio::spawn(run_with_config(config));
2172
2173        wait_for_path(&capture, &task).await;
2174        let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2175
2176        task.abort();
2177        assert!(task
2178            .await
2179            .expect_err("aborted daemon task must cancel")
2180            .is_cancelled());
2181        assert_eq!(
2182            before, after,
2183            "an in-process daemon must not create or reconcile ambient module cgroups"
2184        );
2185    }
2186
2187    #[cfg(target_os = "linux")]
2188    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2189    async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2190        let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2191        let cgroup_root = temp.join("cgroup");
2192        fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2193        fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2194        let modules = cgroup_root.join("subc-modules");
2195        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2196            .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2197            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2198        let task = tokio::spawn(run_with_config(config));
2199
2200        wait_for_path(&modules, &task).await;
2201
2202        task.abort();
2203        assert!(task
2204            .await
2205            .expect_err("aborted daemon task must cancel")
2206            .is_cancelled());
2207        assert!(
2208            fs::read_dir(&modules)
2209                .expect("read prepared modules directory")
2210                .next()
2211                .is_none(),
2212            "the delegation probe must clean up after itself"
2213        );
2214    }
2215
2216    struct EnvGuard {
2217        key: &'static str,
2218        previous: Option<OsString>,
2219    }
2220
2221    impl EnvGuard {
2222        fn set(key: &'static str, value: &Path) -> Self {
2223            let previous = env::var_os(key);
2224            env::set_var(key, value);
2225            Self { key, previous }
2226        }
2227
2228        fn set_str(key: &'static str, value: &str) -> Self {
2229            let previous = env::var_os(key);
2230            env::set_var(key, value);
2231            Self { key, previous }
2232        }
2233
2234        fn unset(key: &'static str) -> Self {
2235            let previous = env::var_os(key);
2236            env::remove_var(key);
2237            Self { key, previous }
2238        }
2239    }
2240
2241    impl Drop for EnvGuard {
2242        fn drop(&mut self) {
2243            match &self.previous {
2244                Some(value) => env::set_var(self.key, value),
2245                None => env::remove_var(self.key),
2246            }
2247        }
2248    }
2249
2250    fn unique_temp_dir(name: &str) -> ScratchDir {
2251        ScratchDir::new(name)
2252    }
2253
2254    fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2255        let dir = unique_temp_dir(name);
2256        let path = dir.join("conn.json");
2257        (dir, path)
2258    }
2259
2260    fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2261        ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2262    }
2263
2264    fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2265        let auth = auth_for(&bound.connection_info);
2266        tokio::spawn(serve_listeners(
2267            bound.listeners,
2268            Arc::new(Router::with_default_self_handler()),
2269            auth,
2270        ))
2271    }
2272
2273    fn expect_bound(outcome: Outcome) -> BoundDaemon {
2274        match outcome {
2275            Outcome::Bound(bound) => bound,
2276            Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2277        }
2278    }
2279
2280    async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2281        let endpoint = conn
2282            .endpoints
2283            .first()
2284            .expect("test connection file should have an endpoint");
2285        let ip: IpAddr = endpoint.host.parse().unwrap();
2286        TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2287    }
2288
2289    fn make_connection_info(port: u16) -> ConnectionInfo {
2290        ConnectionInfo {
2291            schema: SCHEMA_VERSION,
2292            wire_version: Some(PROTOCOL_VERSION),
2293            endpoints: vec![Endpoint {
2294                host: "127.0.0.1".to_owned(),
2295                port,
2296            }],
2297            key: generate_key().unwrap(),
2298            daemon_id: generate_daemon_id().unwrap(),
2299            pid: process::id(),
2300            daemon_ver: "test-subc".to_owned(),
2301        }
2302    }
2303
2304    fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2305        fs::write(path, contents).unwrap();
2306        #[cfg(unix)]
2307        fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2308    }
2309
2310    fn assert_owner_only_connection_file(path: &Path) {
2311        // `path` is only inspected on Unix (mode bits); on Windows the owner-only
2312        // guarantee comes from the inherited %TEMP% ACL, nothing to assert here.
2313        #[cfg(unix)]
2314        {
2315            let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2316            assert_eq!(mode, 0o600);
2317        }
2318        #[cfg(not(unix))]
2319        let _ = path;
2320    }
2321
2322    #[test]
2323    fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2324        let _env_lock = ENV_LOCK.lock().unwrap();
2325        let runtime_dir = unique_temp_dir("xdg-runtime");
2326        let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2327
2328        assert_eq!(
2329            connection_file_path(),
2330            runtime_dir.join(CONNECTION_FILE_NAME)
2331        );
2332    }
2333
2334    #[test]
2335    fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2336        let runtime_dir = OsString::from("/run/user/1000");
2337
2338        let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2339
2340        assert_eq!(
2341            path,
2342            PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2343        );
2344        assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2345    }
2346
2347    #[test]
2348    fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2349        let _env_lock = ENV_LOCK.lock().unwrap();
2350        let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2351
2352        assert_eq!(
2353            connection_file_path(),
2354            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2355        );
2356    }
2357
2358    #[test]
2359    fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2360        let (path, source) = connection_file_path_with_source(None);
2361
2362        assert_eq!(
2363            path,
2364            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2365        );
2366        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2367    }
2368
2369    /// Concurrent callers must derive one token. The former temp-file uid probe
2370    /// could fail transiently (same-tick name collision, fd exhaustion) and send
2371    /// the loser down the env-derived fallback with a different identity for the
2372    /// same user; this fence keeps identity independent of filesystem luck.
2373    #[test]
2374    fn user_connection_token_is_stable_under_concurrent_callers() {
2375        let expected = user_connection_token();
2376        let workers: Vec<_> = (0..32)
2377            .map(|_| {
2378                std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2379            })
2380            .collect();
2381        for worker in workers {
2382            for token in worker.join().expect("probe thread") {
2383                assert_eq!(token, expected, "token diverged under concurrent probes");
2384            }
2385        }
2386    }
2387
2388    #[test]
2389    fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
2390        let (path, source) = connection_file_path_with_source(Some(OsString::new()));
2391
2392        assert_eq!(
2393            path,
2394            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2395        );
2396        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2397    }
2398
2399    /// The shipped binary is the only caller that journals terminal exits into
2400    /// the real run directory, so its constructor must supply that path itself;
2401    /// the in-process default leaves it unset.
2402    #[test]
2403    fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
2404        let _env_lock = ENV_LOCK.lock().unwrap();
2405        let root = unique_temp_dir("daemon-binary-config");
2406        let data_home = root.join("data");
2407        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
2408        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2409        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2410
2411        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
2412
2413        let run_dir = data_home.join("cortexkit").join("run");
2414        assert_eq!(
2415            config.terminal_journal_path,
2416            Some(run_dir.join("terminals.jsonl"))
2417        );
2418        assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
2419        assert_eq!(
2420            BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
2421            None,
2422            "an in-process config must not journal anywhere unless asked to"
2423        );
2424    }
2425
2426    #[test]
2427    fn daemon_binary_config_refuses_a_relative_data_home() {
2428        let _env_lock = ENV_LOCK.lock().unwrap();
2429        let root = unique_temp_dir("daemon-binary-relative-data");
2430        let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
2431        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2432        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2433
2434        let error = BootstrapConfig::from_env_for_daemon_binary()
2435            .expect_err("a relative data home must refuse the daemon binary's config");
2436        assert!(
2437            matches!(error, BootstrapError::RunDir(_)),
2438            "expected a run-directory refusal, got {error}"
2439        );
2440        assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
2441    }
2442
2443    #[test]
2444    fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
2445        let _env_lock = ENV_LOCK.lock().unwrap();
2446        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2447        let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
2448        let config_path = conn_path.with_file_name("subc.jsonc");
2449        for (doc, expected) in [
2450            (None, daemon_config::PrivilegedOperatorPrompt::Log),
2451            (
2452                Some(r#"{"version":1}"#),
2453                daemon_config::PrivilegedOperatorPrompt::Log,
2454            ),
2455            (
2456                Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
2457                daemon_config::PrivilegedOperatorPrompt::Log,
2458            ),
2459            (
2460                Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
2461                daemon_config::PrivilegedOperatorPrompt::Enforce,
2462            ),
2463        ] {
2464            if let Some(doc) = doc {
2465                fs::write(&config_path, doc).unwrap();
2466            }
2467            assert_eq!(
2468                BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2469                    .unwrap()
2470                    .privileged_operator_prompt,
2471                expected
2472            );
2473            assert_eq!(
2474                BootstrapConfig::new(&conn_path, 0)
2475                    .with_daemon_config_path(&config_path)
2476                    .unwrap()
2477                    .privileged_operator_prompt,
2478                expected
2479            );
2480        }
2481    }
2482
2483    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2484    async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
2485        let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
2486        let connection_path = dir.join("connection.json");
2487        let config_path = dir.join("subc.jsonc");
2488        fs::write(
2489            &config_path,
2490            r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
2491        )
2492        .unwrap();
2493        let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
2494            .with_daemon_config_path(&config_path)
2495            .unwrap();
2496        let connection_path = config.connection_file_path.clone();
2497        let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
2498        tokio::time::timeout(Duration::from_secs(5), async {
2499            while !connection_path.exists() {
2500                assert!(
2501                    !daemon.handle.is_finished(),
2502                    "daemon exited before publishing"
2503                );
2504                tokio::task::yield_now().await;
2505            }
2506        })
2507        .await
2508        .unwrap();
2509        let info = connection_file::read(&connection_path).unwrap();
2510        let mut client = connect_from_info(&info).await.unwrap();
2511        authenticate_client(&mut client, &info, Duration::from_secs(5))
2512            .await
2513            .unwrap();
2514        for (correlation, mode, expected) in [
2515            (1, "enforce", Vec::new()),
2516            (2, "log", vec!["privileged_operator_prompt"]),
2517            (3, "log", vec!["privileged_operator_prompt"]),
2518        ] {
2519            fs::write(
2520                &config_path,
2521                serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
2522            )
2523            .unwrap();
2524            let frame = crate::Frame::build(
2525                subc_protocol::FrameType::Request,
2526                subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
2527                0,
2528                0,
2529                correlation,
2530                serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
2531                    preview: true,
2532                })
2533                .unwrap(),
2534            )
2535            .unwrap();
2536            crate::write_frame(&mut client, &frame).await.unwrap();
2537            let reply =
2538                tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
2539                    .await
2540                    .unwrap()
2541                    .unwrap()
2542                    .unwrap();
2543            assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
2544            assert_eq!(reply.header.corr, correlation);
2545            let subc_control::ClientControlResponse::SupervisorRescan { result } =
2546                serde_json::from_slice(&reply.body).unwrap()
2547            else {
2548                panic!("expected a rescan response");
2549            };
2550            assert_eq!(result.restart_required, expected);
2551        }
2552    }
2553
2554    #[test]
2555    fn configured_port_uses_default_config_and_env_override() {
2556        let _env_lock = ENV_LOCK.lock().unwrap();
2557        let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
2558        let config_path = conn_path.with_file_name("subc.jsonc");
2559
2560        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2561        assert_eq!(
2562            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2563                .unwrap()
2564                .port,
2565            DEFAULT_SUBC_PORT
2566        );
2567
2568        fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
2569        assert_eq!(
2570            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2571                .unwrap()
2572                .port,
2573            8123
2574        );
2575
2576        let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
2577        assert_eq!(
2578            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2579                .unwrap()
2580                .port,
2581            9012
2582        );
2583    }
2584
2585    #[tokio::test]
2586    async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
2587        let (_dir, path) = temp_connection_file_path("departing-peer");
2588        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2589        let info = make_connection_info(listener.local_addr().unwrap().port());
2590        write_connection_file(&path, &info, None).unwrap();
2591        let peer = tokio::spawn(async move {
2592            let (mut stream, _) = listener.accept().await.unwrap();
2593            subc_transport::authenticate_server(
2594                &mut stream,
2595                &info.key,
2596                &info.daemon_id,
2597                &info.daemon_ver,
2598                PROBE_AUTH_DEADLINE,
2599            )
2600            .await
2601            .unwrap();
2602        });
2603        let outcome = ensure_singleton(&path, 0).await.unwrap();
2604        peer.await.unwrap();
2605        assert!(
2606            matches!(outcome, Outcome::Bound(_)),
2607            "a departing peer is not AlreadyRunning"
2608        );
2609    }
2610
2611    #[tokio::test]
2612    async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
2613        let (_dir, path) = temp_connection_file_path("already-running");
2614
2615        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2616        let server = start_server(bound);
2617
2618        let second = ensure_singleton(&path, 0).await.unwrap();
2619        assert!(matches!(second, Outcome::AlreadyRunning));
2620
2621        server.abort();
2622        let _ = server.await;
2623    }
2624
2625    #[tokio::test]
2626    async fn daemon_connection_file_publishes_protocol_wire_version() {
2627        let (_dir, path) = temp_connection_file_path("wire-version");
2628        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2629        assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
2630        assert_eq!(
2631            connection_file::read(&path).unwrap().wire_version,
2632            Some(PROTOCOL_VERSION)
2633        );
2634
2635        drop(bound.listeners);
2636    }
2637
2638    #[tokio::test]
2639    async fn stale_unbound_connection_file_is_reclaimed() {
2640        let (_dir, path) = temp_connection_file_path("stale-reclaim");
2641        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2642        let stale_port = stale.local_addr().unwrap().port();
2643        drop(stale);
2644        let stale_info = make_connection_info(stale_port);
2645        write_atomic(&path, &stale_info).unwrap();
2646
2647        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2648        assert_ne!(bound.connection_info.key, stale_info.key);
2649        drop(bound.listeners);
2650    }
2651
2652    #[cfg(unix)]
2653    #[tokio::test]
2654    async fn ensure_singleton_reclaims_insecure_connection_file() {
2655        let (_dir, path) = temp_connection_file_path("insecure-reclaim");
2656        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2657        let stale_port = stale.local_addr().unwrap().port();
2658        drop(stale);
2659        let stale_info = make_connection_info(stale_port);
2660        write_atomic(&path, &stale_info).unwrap();
2661        fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
2662
2663        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2664        assert_ne!(bound.connection_info.key, stale_info.key);
2665        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2666        assert_owner_only_connection_file(&path);
2667
2668        drop(bound.listeners);
2669    }
2670
2671    #[tokio::test]
2672    async fn ensure_singleton_reclaims_non_loopback_connection_file() {
2673        let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
2674        let mut stale_info = make_connection_info(8757);
2675        stale_info.endpoints = vec![Endpoint {
2676            host: "192.0.2.10".to_owned(),
2677            port: 8757,
2678        }];
2679        write_atomic(&path, &stale_info).unwrap();
2680
2681        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2682        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
2683        assert_ne!(bound.connection_info.key, stale_info.key);
2684        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2685        assert!(bound
2686            .connection_info
2687            .endpoints
2688            .iter()
2689            .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
2690        assert_owner_only_connection_file(&path);
2691
2692        drop(bound.listeners);
2693    }
2694
2695    #[tokio::test]
2696    async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
2697        let mut unsupported_schema = make_connection_info(8757);
2698        unsupported_schema.schema = SCHEMA_VERSION + 1;
2699
2700        let mut empty_endpoints = make_connection_info(8757);
2701        empty_endpoints.endpoints.clear();
2702
2703        let mut short_key = make_connection_info(8757);
2704        short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
2705
2706        let cases = vec![
2707            (
2708                "unsupported-schema",
2709                serde_json::to_vec(&unsupported_schema).unwrap(),
2710                Some(unsupported_schema),
2711            ),
2712            (
2713                "empty-endpoints",
2714                serde_json::to_vec(&empty_endpoints).unwrap(),
2715                Some(empty_endpoints),
2716            ),
2717            (
2718                "short-key",
2719                serde_json::to_vec(&short_key).unwrap(),
2720                Some(short_key),
2721            ),
2722            ("invalid-json", b"{not valid connection json".to_vec(), None),
2723        ];
2724
2725        for (label, contents, old_info) in cases {
2726            let (_dir, path) = temp_connection_file_path(label);
2727            write_raw_owner_only_connection_file(&path, &contents);
2728
2729            let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2730            if let Some(old_info) = old_info {
2731                assert_ne!(bound.connection_info.key, old_info.key, "{label}");
2732                assert_ne!(
2733                    bound.connection_info.daemon_id, old_info.daemon_id,
2734                    "{label}"
2735                );
2736            }
2737            assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
2738            assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
2739            assert_owner_only_connection_file(&path);
2740
2741            drop(bound.listeners);
2742        }
2743    }
2744
2745    #[tokio::test]
2746    async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
2747        let (_dir, path) = temp_connection_file_path("foreign-reclaim");
2748        let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2749        let foreign_port = foreign.local_addr().unwrap().port();
2750        write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
2751        let foreign_task = tokio::spawn(async move {
2752            if let Ok((mut stream, _)) = foreign.accept().await {
2753                let mut buf = [0u8; 64];
2754                let _ = stream.read(&mut buf).await;
2755            }
2756        });
2757
2758        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2759        assert!(bound
2760            .connection_info
2761            .endpoints
2762            .iter()
2763            .all(|endpoint| endpoint.port != foreign_port));
2764
2765        drop(bound.listeners);
2766        let _ = foreign_task.await;
2767    }
2768
2769    #[tokio::test]
2770    async fn stale_start_lock_file_is_reclaimable() {
2771        let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
2772        let lock_path = start_lock_path(&path);
2773        drop(open_owner_only_lock(&lock_path).unwrap());
2774        assert!(lock_path.is_file());
2775
2776        let lock = StartLock::acquire(&path).await.unwrap();
2777        assert!(lock_path.is_file());
2778
2779        drop(lock);
2780        assert!(lock_path.is_file());
2781    }
2782
2783    #[tokio::test]
2784    async fn held_start_lock_blocks_second_acquire_until_release() {
2785        let (_dir, path) = temp_connection_file_path("start-lock-held");
2786        let lock_path = start_lock_path(&path);
2787        let first = StartLock::acquire(&path).await.unwrap();
2788
2789        let err = match StartLock::acquire(&path).await {
2790            Ok(_) => panic!("second acquire while held must stay busy"),
2791            Err(err) => err,
2792        };
2793        assert!(matches!(
2794            err,
2795            BootstrapError::StartLockBusy {
2796                ref path,
2797                attempts: START_LOCK_RETRIES,
2798            } if path == &lock_path
2799        ));
2800
2801        drop(first);
2802
2803        let second = StartLock::acquire(&path)
2804            .await
2805            .expect("released advisory lock should be reclaimable");
2806        drop(second);
2807    }
2808
2809    #[tokio::test]
2810    async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
2811        let (_dir, path) = temp_connection_file_path("bind-conflict");
2812        let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2813        let occupied_port = occupied.local_addr().unwrap().port();
2814
2815        let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
2816        assert!(matches!(
2817            err,
2818            BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
2819        ));
2820        assert!(err.to_string().contains("set the port in config"));
2821
2822        drop(occupied);
2823    }
2824
2825    #[tokio::test]
2826    async fn ephemeral_ipv6_collision_still_serves_ipv4() {
2827        let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
2828            let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
2829            let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
2830            drop(occupied);
2831            result
2832        })
2833        .await
2834        .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
2835        assert_eq!(endpoints.len(), 1);
2836        assert_eq!(endpoints[0].host, "127.0.0.1");
2837        TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
2838            .await
2839            .unwrap();
2840        drop(listeners);
2841    }
2842
2843    #[tokio::test]
2844    async fn unpublished_boot_holds_singleton_lock_until_publication() {
2845        let dir = ScratchDir::new("ul");
2846        let path = dir.join("c");
2847        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2848        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2849        assert!(!path.exists());
2850        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2851        assert!(
2852            matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
2853            "an unpublished boot must keep its singleton claim"
2854        );
2855        drop(bound);
2856        FileExt::try_lock(&contender).unwrap();
2857    }
2858
2859    #[tokio::test]
2860    async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
2861        let dir = ScratchDir::new("sl");
2862        let path = dir.join("c");
2863        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2864        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2865        // try_clone shares the open file description just as fork does,
2866        // without relying on the timing of another test's child spawn.
2867        let inherited = bound
2868            .publication_lock
2869            .as_ref()
2870            .unwrap()
2871            ._file
2872            .try_clone()
2873            .unwrap();
2874        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2875        assert!(matches!(
2876            FileExt::try_lock(&contender),
2877            Err(TryLockError::WouldBlock)
2878        ));
2879        drop(bound);
2880        FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
2881        drop(inherited);
2882    }
2883
2884    #[tokio::test]
2885    async fn readiness_probe_requires_an_authenticating_server() {
2886        let dir = ScratchDir::new("unserved-readiness");
2887        let bound = expect_bound(
2888            ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
2889                .await
2890                .unwrap(),
2891        );
2892        assert!(matches!(
2893            verify_serving(&bound.connection_info).await,
2894            Err(BootstrapError::StartupNotServing)
2895        ));
2896    }
2897
2898    #[cfg(unix)]
2899    #[tokio::test]
2900    async fn orphan_sweep_finishes_before_connection_file_publication() {
2901        use crate::live_children::{ExecutableIdentity, LiveChild};
2902        let dir = ScratchDir::new("pas");
2903        // A one-letter connection file name, and no long socket path: the sweep
2904        // this test covers runs before the local endpoint is bound, so a longer
2905        // name would only add a way for the test to fail on something else.
2906        let path = dir.join("c");
2907        let record = dir.join("live-children.json");
2908        let ready = dir.join("ready");
2909        let mut child = std::process::Command::new("/bin/sh")
2910            .args([
2911                "-c",
2912                "trap '' TERM; : > \"$0\"; read -r _",
2913                ready.to_str().unwrap(),
2914            ])
2915            .stdin(std::process::Stdio::piped())
2916            .env("XDG_DATA_HOME", dir.path())
2917            .env("XDG_RUNTIME_DIR", dir.path())
2918            .env("XDG_CONFIG_HOME", dir.path())
2919            .spawn()
2920            .unwrap();
2921        timeout(Duration::from_secs(5), async {
2922            while !ready.exists() {
2923                sleep(Duration::from_millis(5)).await;
2924            }
2925        })
2926        .await
2927        .unwrap();
2928        let observed = subc_os::Process::open(child.id())
2929            .unwrap()
2930            .unwrap()
2931            .observe()
2932            .unwrap();
2933        crate::live_children::write_record(
2934            &record,
2935            &[LiveChild {
2936                module_id: "old-sleep".into(),
2937                pid: child.id(),
2938                protocol: subc_control::ModuleProtocol::None,
2939                start_time: Some(observed.start_time),
2940                executable: observed.executable.map(ExecutableIdentity::from),
2941                cgroup_name: None,
2942            }],
2943        )
2944        .unwrap();
2945        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
2946            .with_live_children_record(&record);
2947        let daemon = tokio::spawn(run_with_config(config));
2948        timeout(Duration::from_secs(5), async {
2949            while !path.exists() {
2950                sleep(Duration::from_millis(5)).await;
2951            }
2952        })
2953        .await
2954        .unwrap();
2955        assert!(path.exists());
2956        let ended = child.try_wait().unwrap().is_some();
2957        if !ended {
2958            child.kill().unwrap();
2959        }
2960        child.wait().unwrap();
2961        if ended {
2962            assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
2963        }
2964        daemon.abort();
2965        let _ = daemon.await;
2966        assert!(
2967            ended,
2968            "the published daemon must not still owe its orphan sweep"
2969        );
2970    }
2971
2972    /// Publishing the connection file fsyncs, which can take seconds on a busy
2973    /// disk, so the daemon must do it off the async worker thread. The hook
2974    /// blocks inside the write until another task on the same single-threaded
2975    /// runtime has run. Writing on the worker thread would starve that task: the
2976    /// hook then gives up after its deadline and the test fails, as it does if
2977    /// startup publishes without going through the hooked write at all.
2978    #[tokio::test(flavor = "current_thread")]
2979    async fn connection_file_publication_leaves_the_async_runtime_free() {
2980        use std::sync::atomic::{AtomicBool, Ordering};
2981
2982        let (_dir, path) = temp_connection_file_path("publish-off-runtime");
2983        let write_started = Arc::new(AtomicBool::new(false));
2984        let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
2985        let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
2986        let hook_started = Arc::clone(&write_started);
2987        connection_file_write_hook::install(&path, move || {
2988            hook_started.store(true, Ordering::SeqCst);
2989            let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
2990            let _ = verdict_tx.send(progressed);
2991        });
2992        // Signals only after the write has started, so a message can only come
2993        // from this task running while the write is still blocked.
2994        let ticker = tokio::spawn(async move {
2995            loop {
2996                if write_started.load(Ordering::SeqCst) {
2997                    let _ = progress_tx.send(());
2998                    return;
2999                }
3000                sleep(Duration::from_millis(5)).await;
3001            }
3002        });
3003
3004        let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
3005        let published = timeout(Duration::from_secs(15), async {
3006            while !path.exists() && !daemon.is_finished() {
3007                sleep(Duration::from_millis(5)).await;
3008            }
3009        })
3010        .await;
3011        let daemon_exited = daemon.is_finished();
3012        daemon.abort();
3013        let daemon_result = daemon.await;
3014        ticker.abort();
3015
3016        assert!(
3017            published.is_ok() && !daemon_exited,
3018            "the daemon must start and publish its connection file: {daemon_result:?}"
3019        );
3020        match verdict_rx.try_recv() {
3021            Ok(true) => {}
3022            Ok(false) => panic!(
3023                "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
3024            ),
3025            Err(_) => panic!("startup published the connection file without the hooked write"),
3026        }
3027    }
3028
3029    /// A write that never reports back must stop startup with an error naming
3030    /// the connection file, never be taken as published.
3031    #[tokio::test]
3032    async fn a_panicking_connection_file_write_fails_startup() {
3033        let (_dir, path) = temp_connection_file_path("publish-panics");
3034        connection_file_write_hook::install(&path, || {
3035            panic!("deliberate panic inside the connection-file write")
3036        });
3037
3038        let error = timeout(
3039            Duration::from_secs(15),
3040            run_with_config(BootstrapConfig::new(&path, 0)),
3041        )
3042        .await
3043        .expect("startup must end once the write task fails")
3044        .expect_err("startup must fail when the connection-file write panics");
3045
3046        assert!(
3047            matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
3048            "expected a connection-file write error, got {error:?}"
3049        );
3050        let message = error.to_string();
3051        assert!(message.contains(&path.display().to_string()), "{message}");
3052        assert!(message.contains("panicked"), "{message}");
3053        assert!(!path.exists(), "no connection file may be published");
3054    }
3055
3056    #[cfg(target_os = "macos")]
3057    #[test]
3058    fn nofile_raise_worker() {
3059        let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
3060            return;
3061        };
3062        let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
3063        rlimit::Resource::NOFILE.set(256, hard).unwrap();
3064        raise_nofile_limit_with_ceiling(Some(10240));
3065        std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
3066    }
3067
3068    #[cfg(target_os = "macos")]
3069    #[test]
3070    fn macos_nofile_raise_clamps_to_kernel_ceiling() {
3071        let dir = ScratchDir::new("nofile-ceiling");
3072        let result = dir.join("soft");
3073        let status = std::process::Command::new(std::env::current_exe().unwrap())
3074            .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
3075            .env("SUBC_NOFILE_TEST_FILE", &result)
3076            .env("XDG_DATA_HOME", dir.path())
3077            .env("XDG_RUNTIME_DIR", dir.path())
3078            .env("XDG_CONFIG_HOME", dir.path())
3079            .status()
3080            .unwrap();
3081        assert!(status.success());
3082        let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
3083        assert!(
3084            soft > 256 && soft <= 10240,
3085            "the inherited limit must be raised within the kernel ceiling; got {soft}"
3086        );
3087    }
3088
3089    #[tokio::test]
3090    async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
3091        let (_dir, path) = temp_connection_file_path("key-rotation");
3092        let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3093        let old_info = first.connection_info.clone();
3094        let fixed_port = old_info.endpoints[0].port;
3095        drop(first.listeners);
3096
3097        let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
3098        let new_info = second.connection_info.clone();
3099        assert_ne!(old_info.key, new_info.key);
3100        assert_ne!(old_info.daemon_id, new_info.daemon_id);
3101        let server = start_server(second);
3102
3103        let mut old_stream = connect_from_info(&old_info).await.unwrap();
3104        let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
3105        assert!(
3106            old_auth.is_err(),
3107            "old key must not authenticate after restart"
3108        );
3109
3110        let reread = connection_file::read(&path).unwrap();
3111        let mut new_stream = connect_from_info(&reread).await.unwrap();
3112        authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
3113            .await
3114            .unwrap();
3115
3116        server.abort();
3117        let _ = server.await;
3118    }
3119
3120    #[cfg(unix)]
3121    #[tokio::test]
3122    async fn published_connection_file_permissions_are_owner_only() {
3123        let (_dir, path) = temp_connection_file_path("permissions");
3124        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3125
3126        let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
3127        assert_eq!(mode, 0o600);
3128
3129        drop(bound.listeners);
3130    }
3131
3132    /// A config with the socket in `dir`'s own short subdirectory.
3133    ///
3134    /// A Unix socket path is limited to just over a hundred bytes, and this
3135    /// harness's temporary directories use nearly all of that, so a test that
3136    /// boots a real daemon passes a short directory and a one-letter connection
3137    /// file name; anything longer makes the bind refuse. On Windows there is no
3138    /// socket path to keep short, so the config is returned unchanged.
3139    fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
3140        let dir = config
3141            .connection_file_path
3142            .parent()
3143            .map(Path::to_path_buf)
3144            .unwrap_or_else(|| PathBuf::from("."));
3145        #[cfg(unix)]
3146        let config = BootstrapConfig {
3147            connection_file_path: dir.join("c"),
3148            local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
3149            ..config
3150        };
3151        #[cfg(not(unix))]
3152        let _ = dir;
3153        config
3154    }
3155
3156    /// The shipped binary puts the socket in its own run directory. The file is
3157    /// named from the connection file's path, so a restart finds it again.
3158    #[cfg(unix)]
3159    #[test]
3160    fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
3161        let _env_lock = ENV_LOCK.lock().unwrap();
3162        let root = ScratchDir::new("binary-local-socket");
3163        let data_home = root.join("data");
3164        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3165        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3166        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3167
3168        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3169
3170        let run_dir = data_home.join("cortexkit").join("run");
3171        assert_eq!(
3172            config.local_listener.configured_dir(),
3173            Some(run_dir.as_path()),
3174            "the binary serves from its run directory"
3175        );
3176        let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
3177            .expect("the binary config names a directory");
3178        assert_eq!(socket.parent().unwrap(), run_dir);
3179        assert_eq!(
3180            socket.file_name().unwrap().to_string_lossy(),
3181            "subc.sock",
3182            "a fixed name: the run directory belongs to one data home"
3183        );
3184    }
3185
3186    /// An in-process daemon that names no directory serves TCP only: it publishes
3187    /// no local endpoint, creates nothing under the data home, and says nothing is
3188    /// unavailable, because nothing was asked for.
3189    /// The environment lock covers only the setup: the daemon under test reads
3190    /// its data home from the process environment while it boots, and holding
3191    /// the lock across that await would block every other test that sets one.
3192    #[cfg(unix)]
3193    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3194    async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
3195        let root = ScratchDir::new("in-process-no-local-socket");
3196        let data_home = root.join("data");
3197        fs::create_dir_all(&data_home).unwrap();
3198        let connection_path = root.join("c");
3199        let _data;
3200        let _config;
3201        let _port;
3202        {
3203            let _env_lock = ENV_LOCK.lock().unwrap();
3204            _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3205            _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3206            _port = EnvGuard::unset(SUBC_PORT_ENV);
3207        }
3208
3209        let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
3210
3211        assert!(bound.local_endpoint.is_none());
3212        assert!(bound.local_endpoint_unavailable.is_none());
3213        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3214        assert!(published.local_endpoint.is_none());
3215        assert!(
3216            local_listener::socket_path_for(None, &connection_path).is_none(),
3217            "no configured directory, no socket"
3218        );
3219        let mut leftovers = Vec::new();
3220        let mut pending = vec![data_home.clone()];
3221        while let Some(dir) = pending.pop() {
3222            for entry in fs::read_dir(&dir).unwrap() {
3223                let entry = entry.unwrap();
3224                let path = entry.path();
3225                if entry.file_type().unwrap().is_dir() {
3226                    pending.push(path.clone());
3227                }
3228                if entry.file_name().to_string_lossy().ends_with(".sock") {
3229                    leftovers.push(path);
3230                }
3231            }
3232        }
3233        assert!(
3234            leftovers.is_empty(),
3235            "an in-process daemon must not create a socket in the data home: {leftovers:?}"
3236        );
3237    }
3238
3239    /// A local endpoint that cannot be bound does not stop the daemon: it keeps
3240    /// serving TCP, publishes no local endpoint, and reports the reason where an
3241    /// operator reads daemon facts. Every module stays up for a feature nothing
3242    /// has to use.
3243    #[cfg(unix)]
3244    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3245    async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
3246        let dir = ScratchDir::new("tcp-only");
3247        let canonical = std::fs::canonicalize(&dir).unwrap();
3248        let connection_path = canonical.join("c");
3249        // The socket directory is deep enough that the socket's path is over the
3250        // Unix socket limit, so binding it must fail.
3251        let deep = canonical.join("x".repeat(80));
3252        fs::create_dir(&deep).unwrap();
3253        fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
3254        let config =
3255            BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
3256        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3257        assert!(
3258            bound.local_endpoint.is_none(),
3259            "a socket that cannot be bound is not published"
3260        );
3261        assert!(
3262            bound
3263                .local_endpoint_unavailable
3264                .as_deref()
3265                .is_some_and(|reason| reason.contains("too long")),
3266            "{:?}",
3267            bound.local_endpoint_unavailable
3268        );
3269        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3270        assert!(published.local_endpoint.is_none());
3271        assert!(
3272            !published.info.endpoints.is_empty(),
3273            "TCP is still published"
3274        );
3275
3276        // The daemon serves TCP as usual.
3277        let server = start_server(bound);
3278        let info = connection_file::read(&connection_path).unwrap();
3279        let mut client = connect_from_info(&info).await.unwrap();
3280        authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
3281            .await
3282            .expect("a TCP client still completes the handshake");
3283        server.abort();
3284        let _ = server.await;
3285    }
3286
3287    /// The published file names the local endpoint the daemon actually serves,
3288    /// and a client that trusts the file completes the handshake over it.
3289    #[cfg(unix)]
3290    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3291    async fn published_connection_file_names_the_served_local_endpoint() {
3292        let (_dir, path) = temp_connection_file_path("local-endpoint");
3293        // The socket goes in the system temporary directory rather than inside
3294        // the harness's own directory, whose long lockdown-sandbox path leaves
3295        // no room inside the Unix socket path limit.
3296        let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
3297        let config =
3298            BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
3299        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3300        let endpoint = connection_file::read_with_local_endpoint(&path)
3301            .unwrap()
3302            .local_endpoint
3303            .expect("the daemon serves a local endpoint, so the file names it");
3304        assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
3305        let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
3306            panic!("a Unix daemon publishes a socket path");
3307        };
3308        assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
3309        assert_eq!(
3310            fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
3311            0o600
3312        );
3313
3314        let router = Arc::new(Router::with_default_self_handler());
3315        let auth = auth_for(&bound.connection_info);
3316        let server = tokio::spawn(serve_local_listener(
3317            bound.local_listener.expect("the local listener is bound"),
3318            router,
3319            auth,
3320        ));
3321        let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
3322        authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
3323            .await
3324            .expect("a socket client completes the same handshake as a TCP client");
3325
3326        server.abort();
3327        drop(bound.listeners);
3328    }
3329
3330    /// A configured directory that is not a directory refuses at bind rather
3331    /// than publishing an endpoint nothing serves.
3332    #[cfg(unix)]
3333    #[test]
3334    fn a_socket_directory_that_is_a_file_refuses() {
3335        let dir = ScratchDir::new("local-listener-file-as-dir");
3336        let file = dir.join("not-a-directory");
3337        fs::write(&file, b"x").unwrap();
3338        let error = local_listener::bind(
3339            &LocalListenerConfig::in_socket_dir(file),
3340            &dir.join("connection.json"),
3341        )
3342        .expect_err("a path that is not a directory cannot hold the socket");
3343        assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
3344    }
3345
3346    /// The socket sits directly in the configured directory under one fixed
3347    /// name, and a directory that would put the path over the platform limit is
3348    /// refused rather than shortened around.
3349    #[cfg(unix)]
3350    #[test]
3351    fn the_socket_path_stays_within_the_unix_limit() {
3352        let run_dir = PathBuf::from("/tmp/short-run");
3353        assert!(
3354            local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
3355            "no configured directory means no socket"
3356        );
3357        let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
3358            .expect("a configured directory gets a socket");
3359        assert_eq!(socket, run_dir.join("subc.sock"));
3360        assert_eq!(
3361            socket,
3362            local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
3363            "the name does not come from the connection file"
3364        );
3365
3366        let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
3367        let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
3368            .expect("a configured directory gets a socket");
3369        assert!(
3370            long.as_os_str().len() < 104,
3371            "{} is {} bytes",
3372            long.display(),
3373            long.as_os_str().len()
3374        );
3375    }
3376}