Skip to main content

subc_daemon/
bootstrap.rs

1use std::{
2    collections::BTreeMap,
3    env,
4    error::Error,
5    ffi::OsString,
6    fmt, fs, io,
7    net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8    path::{Path, PathBuf},
9    process,
10    time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17    authenticate_client, connection_file, generate_daemon_id, generate_key, write_atomic,
18    AuthError, ConnectionFileError, ConnectionInfo, Endpoint, SCHEMA_VERSION,
19};
20use tokio::{
21    net::{TcpListener, TcpStream},
22    task::{JoinError, JoinHandle},
23    time::{sleep, timeout},
24};
25use tracing::{error, info, warn};
26
27use crate::{
28    daemon_config::{self, ConfiguredModule, DaemonConfigError},
29    server::{serve_listeners, ServerAuth, ServerError},
30    supervise::HealthConfig,
31    ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
32    ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
33    SupervisorProcessLiveness,
34};
35use std::sync::Arc;
36
37pub const DEFAULT_SUBC_PORT: u16 = 8757;
38pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
39use subc_transport::CONNECTION_FILE_NAME;
40const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
41const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
42const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
43const START_LOCK_RETRIES: usize = 40;
44const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
45
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47pub enum ConnectionFileSource {
48    XdgRuntimeDir,
49    TempDirFallback,
50    Explicit,
51}
52
53impl ConnectionFileSource {
54    fn reason(self) -> &'static str {
55        match self {
56            Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
57            Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
58            Self::Explicit => "configured path",
59        }
60    }
61}
62
63impl fmt::Display for ConnectionFileSource {
64    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
65        formatter.write_str(match self {
66            Self::XdgRuntimeDir => "xdg_runtime_dir",
67            Self::TempDirFallback => "temp_dir_fallback",
68            Self::Explicit => "explicit",
69        })
70    }
71}
72
73/// Runtime bootstrap configuration. Production uses the default fixed port and
74/// optional daemon-config override; tests pass port 0 to let the OS assign a free
75/// loopback port and discover it from the connection file.
76#[derive(Debug, Clone, Default)]
77struct AdmissionFactsConfig {
78    carrier_module_id: Option<String>,
79    targets: Option<Vec<String>>,
80}
81
82/// Controls where module cgroups are prepared.
83///
84/// In-process daemons default to [`Self::Disabled`] so they never derive a
85/// production location from the host process. The shipped daemon explicitly uses
86/// [`Self::Current`]. The `ck-subc` binary accepts
87/// `SUBC_CGROUP_PLACEMENT=disabled` for isolated test processes; an unset
88/// variable retains `Current`, and every other value is rejected at startup.
89/// Tests that exercise placement can own a [`Self::Root`].
90#[derive(Debug, Clone, Default, PartialEq, Eq)]
91pub enum CgroupPlacementConfig {
92    #[default]
93    Disabled,
94    Current,
95    Root(PathBuf),
96}
97
98#[derive(Debug, Clone)]
99pub struct BootstrapConfig {
100    pub operator_queue_wait: Duration,
101    pub operator_timeout: Duration,
102    pub operator_stuck_grace: Duration,
103    #[cfg(feature = "test-support")]
104    pub operator_script: Option<PathBuf>,
105    #[cfg(feature = "test-support")]
106    pub operator_events: Option<PathBuf>,
107    pub connection_file_path: PathBuf,
108    pub port: u16,
109    pub daemon_ver: String,
110    configured_modules: Vec<ConfiguredModule>,
111    storage_config: Option<daemon_config::StorageConfig>,
112    admission_facts: AdmissionFactsConfig,
113    /// Module ids whose scopes may carry the attributes that grant authority.
114    scope_authority_owners: Vec<String>,
115    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
116    activity_config: daemon_config::ActivityConfig,
117    daemon_config_path: Option<PathBuf>,
118    configured_port: Option<u16>,
119    /// Daemon-wide route.bind relay budget in milliseconds (the fallback for
120    /// any module without a per-module override). `None` = built-in default
121    /// (12s — see `control::DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`).
122    route_bind_relay_default_ms: Option<u64>,
123    reserved_capabilities: BTreeMap<String, String>,
124    watchdog_config: DaemonSelfWatchdogConfig,
125    connection_file_source: ConnectionFileSource,
126    /// Where module cgroups are prepared. Disabled unless a caller explicitly
127    /// opts in, because `Current` derives a host location from `/proc/self/cgroup`.
128    cgroup_placement: CgroupPlacementConfig,
129    /// Directory the supervisor writes per-module stdout/stderr capture files
130    /// into. `None` disables capture; the shipped binary supplies its real run
131    /// directory explicitly.
132    capture_logs_dir: Option<PathBuf>,
133    /// File the supervisor appends every module's terminal exits to, so exit
134    /// history survives a daemon restart. `None` keeps terminal history in
135    /// memory only (each module's ring). Absent by default for the same reason
136    /// as `capture_logs_dir`: an in-process daemon booted by a test must not
137    /// append its exits to the operator's real `terminals.jsonl`, where they
138    /// would show up in `ck module terminals`. The shipped binary supplies
139    /// `<run dir>/terminals.jsonl` explicitly.
140    terminal_journal_path: Option<PathBuf>,
141    /// Where the machine id is read from, or minted into when absent. `None`
142    /// serves no machine id. Absent by default for the same reason as
143    /// `capture_logs_dir`: an in-process daemon booted by a test must never
144    /// derive the operator's real data home and mint into it. The shipped binary
145    /// supplies `<data home>/cortexkit/machine-id` explicitly.
146    machine_id_path: Option<PathBuf>,
147    /// The live-children record: every supervised process this daemon has
148    /// running, kept so the next daemon can end the ones a crash left behind.
149    /// At startup, before any module is spawned, the previous daemon's record
150    /// here is swept. `None` keeps no record and sweeps nothing, for the same
151    /// reason as `capture_logs_dir`: an in-process daemon booted by a test
152    /// must never signal processes listed in the operator's real run
153    /// directory. The shipped binary supplies `<run dir>/live-children.json`.
154    live_children_path: Option<PathBuf>,
155    privacy_trampoline: Option<PathBuf>,
156}
157
158impl BootstrapConfig {
159    pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
160        Self {
161            connection_file_path: connection_file_path.into(),
162            operator_queue_wait: Duration::from_secs(150),
163            operator_timeout: Duration::from_secs(120),
164            operator_stuck_grace: Duration::from_secs(10),
165            #[cfg(feature = "test-support")]
166            operator_script: None,
167            #[cfg(feature = "test-support")]
168            operator_events: None,
169            port,
170            daemon_ver: DAEMON_VERSION.to_owned(),
171            configured_modules: Vec::new(),
172            storage_config: None,
173            admission_facts: AdmissionFactsConfig::default(),
174            scope_authority_owners: daemon_config::default_scope_authority_owners(),
175            privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
176            activity_config: daemon_config::ActivityConfig::default(),
177            daemon_config_path: None,
178            configured_port: None,
179            route_bind_relay_default_ms: None,
180            reserved_capabilities: BTreeMap::new(),
181            watchdog_config: DaemonSelfWatchdogConfig::default(),
182            connection_file_source: ConnectionFileSource::Explicit,
183            cgroup_placement: CgroupPlacementConfig::default(),
184            capture_logs_dir: None,
185            terminal_journal_path: None,
186            machine_id_path: None,
187            live_children_path: None,
188            privacy_trampoline: None,
189        }
190    }
191
192    /// Keep the live-children record at `path`, and at startup end the
193    /// processes a previous daemon recorded there that are still running.
194    /// Embedding daemons and tests pass a path inside their own fixture tree.
195    pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
196        self.live_children_path = Some(path.into());
197        self
198    }
199
200    /// macOS module launches require an explicit executable implementing the
201    /// subc-os trampoline entry point. Other systems ignore this setting.
202    pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
203        self.privacy_trampoline = Some(path.into());
204        self
205    }
206
207    /// Serve the machine id stored at `path`, minting it there at startup when
208    /// the file is absent. Embedding daemons and tests pass a path inside their
209    /// own fixture tree.
210    pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
211        self.machine_id_path = Some(path.into());
212        self
213    }
214
215    /// Selects module cgroup placement. The default is disabled.
216    pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
217        self.cgroup_placement = placement;
218        self
219    }
220
221    /// Redirects per-module stdout/stderr capture files out of the real run
222    /// directory. Tests that start an in-process daemon must call this with a
223    /// path inside their fixture tree.
224    pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
225        self.capture_logs_dir = Some(dir.into());
226        self
227    }
228
229    /// Redirects the daemon-private journal, allowing embedded daemons and tests
230    /// to keep their observations out of the operator's live run directory.
231    pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
232        self.terminal_journal_path = Some(path.into());
233        self
234    }
235
236    pub fn from_env() -> Result<Self, BootstrapError> {
237        Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
238    }
239
240    /// The SHIPPED BINARY's config, which is the only caller that should capture
241    /// child output into the operator's real run directory.
242    ///
243    /// Kept separate from `from_env` deliberately: see `capture_logs_dir` and
244    /// `terminal_journal_path` on this struct for why an absent value must mean
245    /// NO CAPTURE and NO JOURNAL rather than the operator's real run directory.
246    /// A run directory that cannot be resolved (a relative data home) refuses
247    /// startup instead of landing under the working directory.
248    pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
249        let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
250        let machine_id_path =
251            crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
252        let config = Self::from_env()?
253            .with_capture_logs_dir(run_dir.join("logs"))
254            .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
255            .with_live_children_record(crate::live_children::record_path(&run_dir))
256            .with_machine_id_path(machine_id_path);
257        #[cfg(feature = "test-support")]
258        let config = {
259            let mut config = config;
260            config.operator_script =
261                std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
262            config.operator_events =
263                std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
264            for (name, value) in [
265                (
266                    "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
267                    &mut config.operator_queue_wait,
268                ),
269                (
270                    "SUBC_TEST_OPERATOR_TIMEOUT_MS",
271                    &mut config.operator_timeout,
272                ),
273                (
274                    "SUBC_TEST_OPERATOR_STUCK_MS",
275                    &mut config.operator_stuck_grace,
276                ),
277            ] {
278                if let Ok(ms) = std::env::var(name) {
279                    if let Ok(ms) = ms.parse::<u64>() {
280                        *value = Duration::from_millis(ms);
281                    }
282                }
283            }
284            config
285        };
286        Ok(config)
287    }
288
289    pub fn from_env_with_daemon_config_path(
290        daemon_config_path: impl AsRef<Path>,
291    ) -> Result<Self, BootstrapError> {
292        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
293        let (daemon_config, activity_config) =
294            daemon_config::load_with_activity(&daemon_config_path)
295                .map_err(BootstrapError::DaemonConfig)?
296                .map(|(config, activity)| (Some(config), activity))
297                .unwrap_or_default();
298        let config_port = daemon_config.as_ref().and_then(|config| config.port);
299        let storage_config = daemon_config
300            .as_ref()
301            .and_then(|config| config.storage.clone());
302        let admission_facts_carrier_module_id = daemon_config
303            .as_ref()
304            .and_then(|config| config.admission_facts_carrier_module_id.clone());
305        let admission_facts_targets = daemon_config
306            .as_ref()
307            .and_then(|config| config.admission_facts_targets.clone());
308        let scope_authority_owners = daemon_config
309            .as_ref()
310            .map(|config| config.scope_authority_owners.clone())
311            .unwrap_or_else(daemon_config::default_scope_authority_owners);
312        let privileged_operator_prompt = daemon_config
313            .as_ref()
314            .map(|config| config.privileged_operator_prompt)
315            .unwrap_or_default();
316        let route_bind_relay_default_ms = daemon_config
317            .as_ref()
318            .and_then(|config| config.route_bind_relay_timeout_ms);
319        let reserved_capabilities = daemon_config
320            .as_ref()
321            .map(|config| config.reserved_capabilities.clone())
322            .unwrap_or_default();
323        let configured_modules = daemon_config
324            .map(|config| config.modules)
325            .unwrap_or_default();
326
327        let port = match env::var(SUBC_PORT_ENV) {
328            Ok(raw) if !raw.trim().is_empty() => {
329                let port = raw
330                    .parse::<u16>()
331                    .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
332                if let Some(config_port) = config_port {
333                    info!(
334                        env = SUBC_PORT_ENV,
335                        env_port = port,
336                        config_port,
337                        "SUBC_PORT overrides daemon config port"
338                    );
339                }
340                port
341            }
342            Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
343        };
344
345        let (connection_file_path, connection_file_source) =
346            connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
347        Ok(Self::new(connection_file_path, port)
348            .with_configured_modules(configured_modules)
349            .with_storage_config(storage_config)
350            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
351            .with_scope_authority_owners(scope_authority_owners)
352            .with_privileged_operator_prompt(privileged_operator_prompt)
353            .with_activity_config(activity_config)
354            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
355            .with_reserved_capabilities(reserved_capabilities)
356            .with_daemon_config_source(daemon_config_path, config_port)
357            .with_connection_file_source(connection_file_source))
358    }
359
360    pub fn with_daemon_config_path(
361        self,
362        daemon_config_path: impl AsRef<Path>,
363    ) -> Result<Self, BootstrapError> {
364        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
365        let (daemon_config, activity_config) =
366            daemon_config::load_with_activity(&daemon_config_path)
367                .map_err(BootstrapError::DaemonConfig)?
368                .map(|(config, activity)| (Some(config), activity))
369                .unwrap_or_default();
370        let configured_port = daemon_config.as_ref().and_then(|config| config.port);
371        let storage_config = daemon_config
372            .as_ref()
373            .and_then(|config| config.storage.clone());
374        let admission_facts_carrier_module_id = daemon_config
375            .as_ref()
376            .and_then(|config| config.admission_facts_carrier_module_id.clone());
377        let admission_facts_targets = daemon_config
378            .as_ref()
379            .and_then(|config| config.admission_facts_targets.clone());
380        let scope_authority_owners = daemon_config
381            .as_ref()
382            .map(|config| config.scope_authority_owners.clone())
383            .unwrap_or_else(daemon_config::default_scope_authority_owners);
384        let privileged_operator_prompt = daemon_config
385            .as_ref()
386            .map(|config| config.privileged_operator_prompt)
387            .unwrap_or_default();
388        let route_bind_relay_default_ms = daemon_config
389            .as_ref()
390            .and_then(|config| config.route_bind_relay_timeout_ms);
391        let reserved_capabilities = daemon_config
392            .as_ref()
393            .map(|config| config.reserved_capabilities.clone())
394            .unwrap_or_default();
395        let configured_modules = daemon_config
396            .map(|config| config.modules)
397            .unwrap_or_default();
398        Ok(self
399            .with_configured_modules(configured_modules)
400            .with_storage_config(storage_config)
401            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
402            .with_scope_authority_owners(scope_authority_owners)
403            .with_privileged_operator_prompt(privileged_operator_prompt)
404            .with_activity_config(activity_config)
405            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
406            .with_reserved_capabilities(reserved_capabilities)
407            .with_daemon_config_source(daemon_config_path, configured_port))
408    }
409
410    pub fn with_configured_modules(
411        mut self,
412        modules: impl IntoIterator<Item = ConfiguredModule>,
413    ) -> Self {
414        self.configured_modules = modules.into_iter().collect();
415        self.configured_modules
416            .sort_by(|left, right| left.module_id.cmp(&right.module_id));
417        self
418    }
419
420    pub fn with_storage_config(
421        mut self,
422        storage_config: Option<daemon_config::StorageConfig>,
423    ) -> Self {
424        self.storage_config = storage_config;
425        self
426    }
427
428    pub fn with_admission_facts_config(
429        mut self,
430        carrier_module_id: Option<String>,
431        targets: Option<Vec<String>>,
432    ) -> Self {
433        self.admission_facts = AdmissionFactsConfig {
434            carrier_module_id,
435            targets,
436        };
437        self
438    }
439
440    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
441    /// Absent from a config, this is `daemon_config::default_scope_authority_owners`.
442    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
443        self.scope_authority_owners = owners;
444        self
445    }
446
447    /// Choose the privileged-operation policy for the daemon's entire lifetime.
448    pub fn with_privileged_operator_prompt(
449        mut self,
450        mode: daemon_config::PrivilegedOperatorPrompt,
451    ) -> Self {
452        self.privileged_operator_prompt = mode;
453        self
454    }
455
456    pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
457        self.activity_config = config;
458        self
459    }
460
461    /// Set the daemon-wide route.bind relay default (the fallback for any
462    /// module without a per-module override). `None` preserves the built-in
463    /// default (12s). `serve_bound_daemon` reads this at startup and threads
464    /// it into the control handler's daemon-wide field.
465    pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
466        self.route_bind_relay_default_ms = ms;
467        self
468    }
469
470    pub fn with_reserved_capabilities(
471        mut self,
472        reserved_capabilities: BTreeMap<String, String>,
473    ) -> Self {
474        self.reserved_capabilities = reserved_capabilities;
475        self
476    }
477
478    fn with_daemon_config_source(
479        mut self,
480        daemon_config_path: PathBuf,
481        configured_port: Option<u16>,
482    ) -> Self {
483        self.daemon_config_path = Some(daemon_config_path);
484        self.configured_port = configured_port;
485        self
486    }
487
488    fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
489        self.connection_file_source = source;
490        self
491    }
492
493    pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
494        self.watchdog_config = watchdog_config;
495        self
496    }
497}
498
499/// Result of singleton discovery.
500// Built once per daemon start and immediately matched, so the size gap between
501// the variants costs nothing worth a box on the public shape.
502#[allow(clippy::large_enum_variant)]
503#[derive(Debug)]
504pub enum Outcome {
505    /// A live daemon authenticated from the connection file; this invocation should exit 0.
506    AlreadyRunning,
507    /// This process won the singleton race, owns bound loopback listener(s), and
508    /// has published a fresh connection file.
509    Bound(BoundDaemon),
510}
511
512#[derive(Debug)]
513pub struct BoundDaemon {
514    pub listeners: Vec<TcpListener>,
515    pub connection_info: ConnectionInfo,
516    pub connection_file_path: PathBuf,
517    pub connection_file_source: ConnectionFileSource,
518    /// The machine id this daemon serves, established before any connection is
519    /// accepted. `None` when the config named no machine id path.
520    pub machine_id: Option<crate::machine_id::MachineId>,
521    /// Ownership of the run directory, held until the daemon exits so no other
522    /// daemon sweeps or writes this one's run state. `None` when the config
523    /// keeps no live-children record, and so has no run directory to own.
524    run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
525    /// The singleton start lock, kept when binding deferred publishing the
526    /// connection file. It stays held until the daemon has verified it is serving
527    /// and written the connection file, so a second daemon starting meanwhile
528    /// cannot claim the singleton too. `None` when the connection
529    /// file was already written at bind time.
530    publication_lock: Option<StartLock>,
531}
532
533/// Resolve subc's per-user TCP connection-file path.
534///
535/// `$XDG_RUNTIME_DIR/subc-connection.json` is preferred because the runtime
536/// directory is already per-user on Unix desktops. Without it, subc falls back
537/// to the system temp dir with a per-user token in the filename so different OS
538/// users do not collide on shared temp directories.
539pub fn connection_file_path() -> PathBuf {
540    connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
541}
542
543fn connection_file_path_with_source(
544    runtime_dir: Option<OsString>,
545) -> (PathBuf, ConnectionFileSource) {
546    if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
547        return (
548            PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
549            ConnectionFileSource::XdgRuntimeDir,
550        );
551    }
552
553    (
554        env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
555        ConnectionFileSource::TempDirFallback,
556    )
557}
558
559/// Resolve, claim, and serve the per-user daemon singleton.
560///
561/// A second invocation is successful: if a live daemon authenticates from the
562/// existing connection file, this returns `Ok(())` after logging and the caller
563/// exits with status 0.
564pub async fn run() -> Result<(), BootstrapError> {
565    // `run` is a binary entry point, so it opts into the production cgroup and
566    // child-log locations. In-process callers keep both features disabled by default.
567    run_with_config(
568        BootstrapConfig::from_env_for_daemon_binary()?
569            .with_cgroup_placement(CgroupPlacementConfig::Current),
570    )
571    .await
572}
573
574/// Serve a daemon from an explicit config. This is the entry point the twelve
575/// sibling repos use to boot an in-process daemon in their integration tests.
576///
577/// # THIS INSTALLS NO TRACING SUBSCRIBER, SO THE DAEMON IS SILENT BY DEFAULT
578///
579/// The daemon's own diagnostics go through `tracing`, and `tracing` DISCARDS
580/// every event when no subscriber is installed. The shipped binary installs one
581/// in `main` (`init_tracing`); this function deliberately does not, because a
582/// library that installs a global subscriber fights with whatever the host
583/// process already set up.
584///
585/// The consequence for a test harness is not "less verbose": it is that the
586/// daemon has NOTHING TO SAY about any failure, and a missing instrument reads
587/// exactly like a clean one. PLEX found this on 2026-09-18 while trying to
588/// capture daemon logs beside an intermittent bind failure, and discovered the
589/// daemon had been silent in every conformance run that repo had ever done --
590/// so the one client-side error string was all the evidence that could exist,
591/// and they had spent a real investigation on a failure whose second source was
592/// never being recorded.
593///
594/// Install one in the harness before calling this, and assert it did something
595/// (they measured 236 daemon lines with the subscriber installed, 0 with the
596/// call commented out) -- otherwise the fix is itself unverified.
597pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
598    let forwarding = Arc::new(ForwardingTable::default());
599    forwarding.operator_confirms().configure_bootstrap(&config);
600    let configured_modules = config.configured_modules.clone();
601    let storage_config = config.storage_config.clone();
602    let admission_facts = config.admission_facts.clone();
603    let scope_authority_owners = config.scope_authority_owners.clone();
604    let privileged_operator_prompt = config.privileged_operator_prompt;
605    let activity_config = config.activity_config;
606    let daemon_config_path = config.daemon_config_path.clone();
607    let configured_port = config.configured_port;
608    let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
609    let reserved_capabilities = config.reserved_capabilities.clone();
610    let watchdog_config = config.watchdog_config.clone();
611    let cgroup_placement_config = config.cgroup_placement.clone();
612    let capture_logs_dir = config.capture_logs_dir.clone();
613    let terminal_journal_path = config.terminal_journal_path.clone();
614    let live_children_path = config.live_children_path.clone();
615    let privacy_trampoline = config.privacy_trampoline.clone();
616    match ensure_singleton_inner(config, false).await? {
617        Outcome::AlreadyRunning => {
618            info!("subc daemon already running");
619            Ok(())
620        }
621        Outcome::Bound(bound) => {
622            #[cfg(target_os = "linux")]
623            let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
624            #[cfg(not(target_os = "linux"))]
625            let _ = cgroup_placement_config;
626            serve_bound_daemon(
627                bound,
628                forwarding,
629                configured_modules,
630                storage_config,
631                admission_facts,
632                scope_authority_owners,
633                privileged_operator_prompt,
634                activity_config,
635                daemon_config_path,
636                configured_port,
637                route_bind_relay_default_ms,
638                reserved_capabilities,
639                watchdog_config,
640                capture_logs_dir,
641                terminal_journal_path,
642                live_children_path,
643                privacy_trampoline,
644                #[cfg(target_os = "linux")]
645                cgroup_placement,
646            )
647            .await
648        }
649    }
650}
651
652#[cfg(target_os = "linux")]
653fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
654    let result = match config {
655        CgroupPlacementConfig::Disabled => return None,
656        CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
657        CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
658    };
659
660    match result {
661        Ok(Some(placement)) => Some(placement),
662        Ok(None) => {
663            warn!(
664                placement = ?config,
665                "module cgroup placement is disabled: configured cgroup root is not delegated"
666            );
667            None
668        }
669        Err(error) => {
670            warn!(
671                placement = ?config,
672                error = %error,
673                "module cgroup placement is disabled by an unexpected cgroup probe error"
674            );
675            None
676        }
677    }
678}
679
680/// Target soft limit for open file descriptors, applied to the daemon before any
681/// module is spawned so children inherit it. Multi-root modules (one process
682/// aggregating every project root's sqlite stores, index caches, watchers, and
683/// LSP pipes) trivially exceed the macOS default soft limit of 256; a launchd
684/// user agent does not pass login-shell ulimits through, so the raise must
685/// happen in-process.
686#[cfg(unix)]
687const NOFILE_TARGET: u64 = 65536;
688
689/// Raise RLIMIT_NOFILE to `NOFILE_TARGET` (clamped to the hard limit).
690/// Best-effort: failure is logged and never fatal, since the daemon can run
691/// under the inherited limit — modules with few roots just have less headroom.
692#[cfg(unix)]
693fn raise_nofile_limit() {
694    #[cfg(target_os = "macos")]
695    let ceiling = std::process::Command::new("/usr/sbin/sysctl")
696        .args(["-n", "kern.maxfilesperproc"])
697        .output()
698        .ok()
699        .filter(|output| output.status.success())
700        .and_then(|output| String::from_utf8(output.stdout).ok())
701        .and_then(|value| value.trim().parse::<u64>().ok())
702        .filter(|value| *value > 0);
703    #[cfg(not(target_os = "macos"))]
704    let ceiling = None;
705    raise_nofile_limit_with_ceiling(ceiling);
706}
707
708#[cfg(unix)]
709fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
710    match rlimit::Resource::NOFILE.get() {
711        Ok((soft, hard)) => {
712            // On XNU an infinite hard limit does not remove the separate
713            // per-process kernel ceiling. Exceeding it makes setrlimit fail
714            // without raising the inherited (often 256) soft limit at all.
715            let target = NOFILE_TARGET
716                .min(hard)
717                .min(kernel_ceiling.unwrap_or(u64::MAX));
718            if soft >= target {
719                return;
720            }
721            match rlimit::Resource::NOFILE.set(target, hard) {
722                Ok(()) => info!(
723                    previous_soft = soft,
724                    new_soft = target,
725                    hard,
726                    "raised open-file soft limit for daemon and module children"
727                ),
728                Err(err) => warn!(
729                    soft,
730                    hard,
731                    error = %err,
732                    "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
733                ),
734            }
735        }
736        Err(err) => warn!(error = %err, "could not read open-file limit"),
737    }
738}
739
740/// CRT stdio-stream target on Windows (the `_setmaxstdio` maximum). Win32
741/// HANDLEs — what Rust `File`, tokio sockets, and SQLite's Win32 VFS actually
742/// consume — have a per-process quota in the millions and need no raise; the
743/// C-runtime stream table (default 512) is the only low ceiling, and it is
744/// per-process rather than inherited, so supervised modules linking the CRT
745/// must raise their own. Raising it here covers the daemon itself.
746#[cfg(windows)]
747fn raise_nofile_limit() {
748    const MAXSTDIO_TARGET: u32 = 8192;
749    let current = rlimit::getmaxstdio();
750    if current >= MAXSTDIO_TARGET {
751        return;
752    }
753    match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
754        Ok(new_max) => info!(
755            previous = current,
756            new_max, "raised CRT stdio-stream limit for daemon"
757        ),
758        Err(err) => warn!(
759            current,
760            error = %err,
761            "could not raise CRT stdio-stream limit"
762        ),
763    }
764}
765
766#[cfg(not(any(unix, windows)))]
767fn raise_nofile_limit() {}
768
769pub async fn run_with_daemon_config_path(
770    config: BootstrapConfig,
771    daemon_config_path: impl AsRef<Path>,
772) -> Result<(), BootstrapError> {
773    run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
774}
775
776#[allow(clippy::too_many_arguments)]
777async fn serve_bound_daemon(
778    bound: BoundDaemon,
779    forwarding: Arc<ForwardingTable>,
780    configured_modules: Vec<ConfiguredModule>,
781    storage_config: Option<daemon_config::StorageConfig>,
782    admission_facts: AdmissionFactsConfig,
783    scope_authority_owners: Vec<String>,
784    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
785    activity_config: daemon_config::ActivityConfig,
786    daemon_config_path: Option<PathBuf>,
787    configured_port: Option<u16>,
788    route_bind_relay_default_ms: Option<u64>,
789    reserved_capabilities: BTreeMap<String, String>,
790    watchdog_config: DaemonSelfWatchdogConfig,
791    capture_logs_dir: Option<PathBuf>,
792    terminal_journal_path: Option<PathBuf>,
793    live_children_path: Option<PathBuf>,
794    privacy_trampoline: Option<PathBuf>,
795    #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
796) -> Result<(), BootstrapError> {
797    #[cfg(unix)]
798    let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
799        .map_err(BootstrapError::Signal)?;
800    // Windows has no SIGTERM. Ctrl-C is a different event, not an equivalent
801    // service-stop contract, so this shutdown handler is intentionally Unix-only.
802    raise_nofile_limit();
803    #[cfg(windows)]
804    if let Err(error) = subc_jobobject::ensure_root_job() {
805        // Keep the control plane available. The root-job API caches setup
806        // failures, so later module spawns refuse instead of running unprotected.
807        warn!(%error, "module launches disabled: Windows root job setup failed");
808    }
809
810    info!(
811        connection_file = %bound.connection_file_path.display(),
812        connection_file_source = %bound.connection_file_source,
813        connection_file_source_reason = bound.connection_file_source.reason(),
814        endpoints = ?bound.connection_info.endpoints,
815        configured_modules = configured_modules.len(),
816        machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
817        "subc daemon starting"
818    );
819
820    // Before anything can spawn a module (the configured modules below, or a
821    // client's start request once the listeners are served): a previous daemon
822    // that died without its shutdown stop may have left children running, and
823    // a fresh copy beside one would fight it for its port and stores. The
824    // record is not a running daemon's because this daemon holds the lock on
825    // the run directory the record lives in, and a running daemon holds that
826    // lock for its whole life. (Claiming the singleton is not enough: it is
827    // keyed on the connection file, which can live in a different runtime
828    // directory from the run directory.) The lock stays held until this
829    // function returns, which is when the daemon exits.
830    let run_dir_lock = bound.run_dir_lock;
831    let publication_lock = bound.publication_lock;
832    if let Some(owner) = &run_dir_lock {
833        crate::live_children::sweep_orphans(
834            owner,
835            &crate::live_children::AdoptedPids::none(),
836            crate::live_children::SweepBounds::default(),
837        )
838        .await;
839    }
840
841    let registry = Arc::new(Registry::default());
842    let process_liveness = Arc::new(SupervisorProcessLiveness::new());
843    let supervisor_handle = SupervisorHandle::new();
844    let connected_clients = ConnectedClients::new();
845    let daemon_incarnation = format!(
846        "{:032x}",
847        u128::from_be_bytes(bound.connection_info.daemon_id)
848    );
849    let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
850        .with_process_liveness(process_liveness.clone())
851        .with_forwarding(Arc::clone(&forwarding))
852        .with_handle(supervisor_handle.clone())
853        .with_connection_file_path(bound.connection_file_path.clone())
854        .with_daemon_incarnation(daemon_incarnation.clone());
855    let supervisor = match privacy_trampoline {
856        Some(path) => supervisor.with_privacy_trampoline(path),
857        None => supervisor,
858    };
859    // ABSENT MEANS NO CAPTURE AND NO JOURNAL, NOT "THE REAL RUN DIRECTORY", and
860    // the difference is a production-corruption hazard rather than a preference.
861    // Both fields below follow the same rule: `None` for `capture_logs_dir`
862    // means supervised output is not captured, and `None` for
863    // `terminal_journal_path` means terminal history lives only in each
864    // module's in-memory ring (`supervisor.terminals` still answers, with the
865    // journal counters at zero).
866    //
867    // The terminal journal used to fall back to
868    // `daemon_run_dir().join("terminals.jsonl")`, so an in-process test daemon
869    // appended its fixture exits to the operator's journal, where they then
870    // appeared in `ck module terminals`.
871    //
872    // The capture line used to be `unwrap_or_else(|| daemon_run_dir().join("logs"))`,
873    // so ANY caller that did not set the field captured supervised children into
874    // the operator's live `~/.local/share/cortexkit/run/logs/`. That is twelve
875    // sibling repos whose integration tests boot an in-process daemon through
876    // `run_with_config` -- none of which asked for it, and none of which can see
877    // it from their side.
878    //
879    // Harmless while fixture module ids are fixture-shaped: this host carries 20
880    // zero-byte files from subc's own tests (good-aft, missing-aft,
881    // preview-consumer...). THE HAZARD IS A COLLISION. A fixture named "broca"
882    // or "aft" appends to a PRODUCTION capture file that operators read
883    // forensically and that placement gates count lines in -- with no residue to
884    // notice, because the file legitimately exists and legitimately grows.
885    //
886    // Found by BROCA (2026-09-19) from the other side: their rigs spawn the
887    // SHIPPED ck-subc and set XDG_CONFIG_HOME + XDG_RUNTIME_DIR but not
888    // XDG_DATA_HOME, so every local rig run supervised a module named "broca"
889    // and captured it into production's broca.stderr.log -- the same file I
890    // count seal lines in before and after placing their binaries.
891    //
892    // The supervisor already treats `None` as no-capture and no-journal, so
893    // this only removes invented defaults. The binary keeps capturing and
894    // journaling via `BootstrapConfig::from_env_for_daemon_binary`.
895
896    // Keep both ledgers in the explicitly selected run directory. An embedded
897    // daemon without a journal path must not write to the operator's live ledger.
898    let activity_run_dir = terminal_journal_path
899        .as_ref()
900        .and_then(|path| path.parent())
901        .map(Path::to_path_buf);
902    let supervisor = match terminal_journal_path {
903        Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
904        None => supervisor,
905    };
906    let supervisor = match capture_logs_dir {
907        Some(dir) => supervisor.with_capture_logs_dir(dir),
908        None => supervisor,
909    };
910    let supervisor = match live_children_path {
911        Some(path) => supervisor.with_live_children_record(path),
912        None => supervisor,
913    };
914    #[cfg(target_os = "linux")]
915    let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
916    // Collect per-module route.bind relay overrides BEFORE handing the
917    // `configured_modules` vector to the supervisor (which only needs each
918    // module's `drain_timeout_ms`). Each entry was filled in by parse-time
919    // resolution (per-module > daemon-wide > absent), so modules with no
920    // override are absent from this map and the daemon-wide default applies.
921    let route_bind_relay_timeouts = configured_modules
922        .iter()
923        .filter_map(|module| {
924            module
925                .route_bind_relay_timeout_ms
926                .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
927        })
928        .collect::<std::collections::BTreeMap<_, _>>();
929    let control_start_clock = crate::clock::StartClock::capture();
930    let mut control = ControlHandler::with_forwarding(Arc::clone(&registry), forwarding)
931        .with_process_liveness(process_liveness)
932        .with_supervisor(supervisor_handle)
933        // The same flag `begin_daemon_shutdown` sets below, so requirement
934        // changes while modules exit during the shutdown log at INFO.
935        .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
936        .with_connected_clients(connected_clients.clone())
937        .with_storage_config(storage_config)
938        .with_machine_id(bound.machine_id.clone())
939        .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
940        .with_scope_authority_owners(scope_authority_owners)
941        .with_privileged_operator_prompt(privileged_operator_prompt)
942        .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
943        .with_daemon_provenance(
944            bound.connection_info.pid,
945            control_start_clock.started_at_ms(),
946            std::env::current_exe().ok(),
947            normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
948            normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
949        )
950        .with_daemon_start_clock(control_start_clock)
951        .with_capability_config(
952            configured_modules
953                .iter()
954                .map(|module| (module.module_id.clone(), module.enabled)),
955            reserved_capabilities,
956        );
957    if let Some(run_dir) = activity_run_dir {
958        control =
959            control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
960    }
961    if let Some(ms) = route_bind_relay_default_ms {
962        // A daemon-wide config value overrides the built-in default; a
963        // `None` here leaves the ControlHandler's 12s default in place.
964        control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
965    }
966    if let Some(config_path) = daemon_config_path {
967        control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
968    }
969    let control = Arc::new(control);
970    let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
971    let auth = ServerAuth::new(
972        bound.connection_info.key.clone(),
973        bound.connection_info.daemon_id,
974        bound.connection_info.daemon_ver.clone(),
975    )
976    .with_connected_clients(connected_clients);
977
978    // Discovery must expose every accepted configured module, including disabled
979    // ones. Registration creates status handles but neither starts monitors
980    // nor launches processes: children need the published file for `--subc`.
981    let mut registered_modules = Vec::with_capacity(configured_modules.len());
982    for configured in configured_modules {
983        let enabled = configured.enabled;
984        let health = configured.health.clone();
985        let module_id = configured.module_id.clone();
986        let registered = match supervisor.register_configured_with_health(
987            configured.module_spec(),
988            enabled,
989            health.clone(),
990            configured.drain_timeout_ms,
991            configured.restart,
992            configured.register_timeout,
993        ) {
994            Ok(registered) => registered,
995            Err(err) => {
996                error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
997                continue;
998            }
999        };
1000        registered_modules.push(registered);
1001        // A relaxed threshold can silently outlive the temporary allowance
1002        // that required it, so report the extra tolerance on every boot.
1003        let default_threshold = HealthConfig::default().failure_threshold;
1004        if enabled && health.failure_threshold > default_threshold {
1005            warn!(
1006                module_id = %module_id,
1007                failure_threshold = health.failure_threshold,
1008                default_threshold,
1009                tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1010                "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1011            );
1012        }
1013        info!(module_id = %module_id, enabled, "configured module registered");
1014    }
1015    control.refresh_capability_requirements();
1016
1017    let mut serve_task =
1018        AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1019    verify_serving(&bound.connection_info).await?;
1020    // A published endpoint promises that authentication can complete now.
1021    // Orphan cleanup and control-plane construction can take longer than an
1022    // existing-daemon probe's deadline, so finish both before discovery.
1023    // The write fsyncs, so it runs on tokio's blocking pool (see
1024    // `publish_connection_file`). The singleton lock travels with it and comes
1025    // back only once the file is in place.
1026    let (published, publication_lock) = publish_connection_file(
1027        &bound.connection_file_path,
1028        &bound.connection_info,
1029        publication_lock,
1030    )
1031    .await;
1032    published?;
1033    #[cfg(test)]
1034    startup_publication_hook::run(&bound.connection_file_path).await;
1035    drop(publication_lock);
1036    let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1037    // Off the startup path: it runs `systemctl`, and only ever logs.
1038    #[cfg(target_os = "linux")]
1039    let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1040        crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1041    ));
1042    let _watchdog_task = AbortOnDrop::new(
1043        DaemonSelfWatchdog::new(
1044            bound.connection_info.clone(),
1045            bound.connection_file_path.clone(),
1046        )
1047        .with_config(watchdog_config)
1048        .spawn(),
1049    );
1050
1051    for registered in registered_modules {
1052        match supervisor.start_registered_configured(registered, None, true) {
1053            Ok(module) => {
1054                info!(module_id = %module.module_id(), "configured module supervised");
1055            }
1056            Err(err) => {
1057                error!(error = %err, "failed to start configured module; continuing daemon startup");
1058            }
1059        }
1060    }
1061
1062    control.refresh_capability_requirements();
1063    Arc::clone(&control).spawn_capability_deadline_loop();
1064    Arc::clone(&control).spawn_scope_expiry_loop();
1065
1066    #[cfg(unix)]
1067    {
1068        tokio::select! {
1069            result = serve_task.join() => {
1070                return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1071            }
1072            _ = terminate.recv() => {}
1073        }
1074        // First, before the notice, the drain, or any connection close: from
1075        // here on a module exit is recorded as `daemon_shutdown` and never
1076        // respawned. Also before allowing a second signal to cut the bounded
1077        // wait short, so the journal marker is always written.
1078        supervisor.begin_daemon_shutdown();
1079        // Stop the self-watchdog before closing the listener. Its next tick would
1080        // connect to that listener, fail, and log an ERROR indistinguishable from
1081        // a wedged daemon, once for every interval a planned stop lasts.
1082        drop(_watchdog_task);
1083        // Dropping the listener stops new accepts, not established connections:
1084        // their detached tasks must remain live throughout notice and drain.
1085        drop(serve_task);
1086        let second_sigterm = async {
1087            terminate.recv().await;
1088            std::time::Instant::now()
1089        };
1090        tokio::pin!(second_sigterm);
1091        let mut escalation_started = tokio::select! {
1092            biased;
1093            at = second_sigterm.as_mut() => {
1094                info!("second SIGTERM: abandoning daemon shutdown wait");
1095                Some(at)
1096            }
1097            result = supervisor.drain_for_daemon_shutdown() => {
1098                if let Err(error) = result {
1099                    warn!(%error, "daemon shutdown drain failed; exiting anyway");
1100                }
1101                None
1102            }
1103        };
1104        // Supervised modules lead their own process groups, so a service
1105        // manager's kill of this process's group does not reach them. (A
1106        // systemd unit with KillMode=control-group kills by cgroup instead and
1107        // does reach them; see `systemd_kill_mode`.) The daemon ends them
1108        // itself: EOF (or SIGTERM for a protocol none child) first, then
1109        // signals at each child's own drain deadline.
1110        let already_escalated = escalation_started.is_some();
1111        supervisor
1112            .end_children_for_daemon_shutdown(already_escalated, async {
1113                if already_escalated {
1114                    std::future::pending::<()>().await;
1115                } else {
1116                    escalation_started = Some(second_sigterm.await);
1117                }
1118            })
1119            .await;
1120        if let Some(at) = escalation_started {
1121            info!(
1122                second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1123                "daemon shutdown wait ended after second SIGTERM"
1124            );
1125        }
1126        info!("daemon exiting: SIGTERM shutdown complete");
1127        Ok(())
1128    }
1129    #[cfg(not(unix))]
1130    serve_task
1131        .join()
1132        .await
1133        .map_err(BootstrapError::ServeJoin)?
1134        .map_err(BootstrapError::Serve)
1135}
1136
1137fn normalized_build_provenance(value: &str) -> Option<String> {
1138    match value.trim() {
1139        "" | "unavailable" => None,
1140        value => Some(value.to_string()),
1141    }
1142}
1143
1144/// Find an existing daemon or atomically bind loopback TCP for this daemon.
1145///
1146/// The algorithm is intentionally connect-first: an endpoint from the connection
1147/// file is treated as live only after the TCP+key server-proof authenticates for
1148/// that file's key and daemon_id. Stale or foreign connection files are reclaimed
1149/// only while holding the per-user start lock; the TCP port is never the
1150/// singleton primitive.
1151pub async fn ensure_singleton(
1152    connection_file_path: impl AsRef<Path>,
1153    port: u16,
1154) -> Result<Outcome, BootstrapError> {
1155    ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1156}
1157
1158pub async fn ensure_singleton_with_config(
1159    config: BootstrapConfig,
1160) -> Result<Outcome, BootstrapError> {
1161    ensure_singleton_inner(config, true).await
1162}
1163
1164async fn ensure_singleton_inner(
1165    config: BootstrapConfig,
1166    publish: bool,
1167) -> Result<Outcome, BootstrapError> {
1168    let path = config.connection_file_path;
1169
1170    if stable_peer(&path).await? {
1171        return Ok(Outcome::AlreadyRunning);
1172    }
1173
1174    let lock = StartLock::acquire(&path).await?;
1175
1176    // Re-probe after acquiring the start lock so a peer that won the race between
1177    // our first failed probe and the lock acquisition is observed instead of
1178    // overwritten.
1179    if stable_peer(&path).await? {
1180        return Ok(Outcome::AlreadyRunning);
1181    }
1182
1183    // The start lock and the probe above only rule out a daemon using this
1184    // connection file. A daemon started with another runtime directory but
1185    // the same data home shares the run directory while passing both, so the
1186    // run directory needs its own owner: refuse at once if a live daemon holds
1187    // it, before anything below reads or writes run or data-home state.
1188    let run_dir_lock = config
1189        .live_children_path
1190        .as_deref()
1191        .map(crate::run_dir_lock::RunDirLock::acquire)
1192        .transpose()?;
1193
1194    remove_stale_connection_file_if_present(&path)?;
1195
1196    // Established under the start lock, and under the run-directory lock when
1197    // there is one, and before binding: a corrupt file stops boot before
1198    // anything is published, and two daemons racing to start cannot both mint.
1199    //
1200    // Minting a new machine id fsyncs its file, so loading or minting runs on
1201    // tokio's blocking pool. Both locks travel with that work, so neither is
1202    // released while a mint is in flight.
1203    let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1204        None => (None, lock, run_dir_lock),
1205        Some(machine_id_path) => {
1206            let task_path = machine_id_path.clone();
1207            match run_blocking_holding((lock, run_dir_lock), move || {
1208                crate::machine_id::load_or_mint(&task_path)
1209            })
1210            .await
1211            {
1212                Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1213                Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1214                Err(join_error) => {
1215                    return Err(BootstrapError::MachineId(
1216                        crate::machine_id::MachineIdFileError::Write {
1217                            path: machine_id_path,
1218                            source: io::Error::other(describe_blocking_task_failure(
1219                                "loading or minting the machine id",
1220                                &join_error,
1221                            )),
1222                        },
1223                    ))
1224                }
1225            }
1226        }
1227    };
1228
1229    let (listeners, endpoints) = bind_loopback(config.port).await?;
1230    let connection_info = ConnectionInfo {
1231        schema: SCHEMA_VERSION,
1232        wire_version: Some(PROTOCOL_VERSION),
1233        endpoints,
1234        key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1235        daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1236        pid: process::id(),
1237        daemon_ver: config.daemon_ver,
1238    };
1239
1240    // The low-level binding API historically publishes for callers that serve
1241    // these listeners themselves. The daemon entry point defers publication
1242    // until its own authentication server is running.
1243    let publication_lock = if publish {
1244        let (published, lock) = publish_connection_file(&path, &connection_info, lock).await;
1245        if let Err(error) = published {
1246            drop(listeners);
1247            return Err(error);
1248        }
1249        drop(lock);
1250        None
1251    } else {
1252        Some(lock)
1253    };
1254
1255    Ok(Outcome::Bound(BoundDaemon {
1256        listeners,
1257        connection_info,
1258        connection_file_path: path,
1259        connection_file_source: config.connection_file_source,
1260        machine_id,
1261        run_dir_lock,
1262        publication_lock,
1263    }))
1264}
1265
1266#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1267enum Probe {
1268    Live,
1269    StaleOrAbsent,
1270}
1271
1272async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1273    if !matches!(probe_existing(path).await?, Probe::Live) {
1274        return Ok(false);
1275    }
1276    // A terminating peer can still authenticate while its listeners drain.
1277    // Require another proof after a short settling interval before reporting
1278    // a successful duplicate start. If the peer left, proceed with startup;
1279    // any remaining bind/ownership conflict is an error, not exit 0 (which
1280    // would suppress launchd's restart-on-failure policy).
1281    tokio::time::sleep(Duration::from_millis(1500)).await;
1282    Ok(matches!(probe_existing(path).await?, Probe::Live))
1283}
1284
1285async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1286    let info = match connection_file::read(path) {
1287        Ok(info) => info,
1288        Err(source) if is_absent_or_stale_connection_file(&source) => {
1289            return Ok(Probe::StaleOrAbsent)
1290        }
1291        Err(source) => {
1292            return Err(BootstrapError::ConnectionFileRead {
1293                path: path.to_path_buf(),
1294                source,
1295            })
1296        }
1297    };
1298
1299    for endpoint in &info.endpoints {
1300        if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1301            return Ok(Probe::Live);
1302        }
1303    }
1304
1305    Ok(Probe::StaleOrAbsent)
1306}
1307
1308async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1309    let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1310        return Probe::StaleOrAbsent;
1311    };
1312    if !ip.is_loopback() {
1313        return Probe::StaleOrAbsent;
1314    }
1315    let addr = SocketAddr::new(ip, endpoint.port);
1316
1317    let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1318        Ok(Ok(stream)) => stream,
1319        Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1320    };
1321
1322    match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1323        Ok(()) => Probe::Live,
1324        Err(AuthError::DaemonIdMismatch)
1325        | Err(AuthError::InvalidServerProof)
1326        | Err(AuthError::UnexpectedEof { .. })
1327        | Err(AuthError::Timeout { .. })
1328        | Err(AuthError::JsonEncode { .. })
1329        | Err(AuthError::JsonDecode { .. })
1330        | Err(AuthError::Io { .. })
1331        | Err(AuthError::MessageTooLarge { .. })
1332        | Err(AuthError::KeyTooShort { .. })
1333        | Err(AuthError::Random(_))
1334        | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1335    }
1336}
1337
1338async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1339    // Scheduling a server task is not a readiness barrier. Authenticate once
1340    // against the mandatory IPv4 listener before exposing discovery to peers.
1341    // This uses the normal probe deadline, with no retries or widened budget.
1342    if let Some(endpoint) = info.endpoints.first() {
1343        if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1344            return Ok(());
1345        }
1346    }
1347    Err(BootstrapError::StartupNotServing)
1348}
1349
1350/// Run blocking filesystem work (an fsync, a rename) on tokio's blocking-thread
1351/// pool rather than on the async worker thread that awaits it. On a slow or
1352/// busy disk an fsync can take seconds, and a runtime has few worker threads
1353/// (an in-process test daemon may share two with everything else), so doing it
1354/// inline stalls unrelated tasks for that long.
1355///
1356/// `guards` (lock handles) move into the blocking task and are handed back
1357/// with the work's result. If the awaiting future is dropped first, the work
1358/// still runs to completion on its thread and the guards are released only
1359/// after it ends, so a lock is never given up while the work it protects is
1360/// still writing.
1361///
1362/// `Err` means the task panicked, or was cancelled before it started because
1363/// the runtime is shutting down; the guards have then been released already.
1364async fn run_blocking_holding<G, T>(
1365    guards: G,
1366    work: impl FnOnce() -> T + Send + 'static,
1367) -> Result<(T, G), JoinError>
1368where
1369    G: Send + 'static,
1370    T: Send + 'static,
1371{
1372    tokio::task::spawn_blocking(move || {
1373        let output = work();
1374        (output, guards)
1375    })
1376    .await
1377}
1378
1379/// Explain, for a startup error, why blocking work produced no result.
1380fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1381    if join_error.is_panic() {
1382        format!("the background thread {work} panicked before finishing: {join_error}")
1383    } else {
1384        format!(
1385            "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1386        )
1387    }
1388}
1389
1390/// Publish the connection file: owner-only permissions, written to a temporary
1391/// file, fsynced and renamed over `path` (all done by `write_atomic`). The
1392/// write runs on tokio's blocking pool through [`run_blocking_holding`], so the
1393/// fsync never holds an async worker thread.
1394///
1395/// `guard` is the singleton start lock (or whatever the caller must keep held
1396/// across publication); it is returned with the result. It comes back `None`
1397/// only when the write task panicked or was cancelled, in which case the
1398/// result is an error and the file must be treated as not published.
1399async fn publish_connection_file<G: Send + 'static>(
1400    path: &Path,
1401    info: &ConnectionInfo,
1402    guard: G,
1403) -> (Result<(), BootstrapError>, Option<G>) {
1404    let task_path = path.to_path_buf();
1405    let task_info = info.clone();
1406    let outcome =
1407        run_blocking_holding(guard, move || write_connection_file(&task_path, &task_info)).await;
1408    match outcome {
1409        Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1410        Ok((Err(source), guard)) => (
1411            Err(BootstrapError::ConnectionFileWrite {
1412                path: path.to_path_buf(),
1413                source,
1414            }),
1415            Some(guard),
1416        ),
1417        Err(join_error) => (
1418            Err(BootstrapError::ConnectionFileWrite {
1419                path: path.to_path_buf(),
1420                source: ConnectionFileError::Io {
1421                    op: "write",
1422                    path: path.to_path_buf(),
1423                    source: io::Error::other(describe_blocking_task_failure(
1424                        "writing the connection file",
1425                        &join_error,
1426                    )),
1427                },
1428            }),
1429            None,
1430        ),
1431    }
1432}
1433
1434/// The synchronous connection-file write, including its fsync. Call it only
1435/// through [`publish_connection_file`], which keeps it off the async workers.
1436fn write_connection_file(path: &Path, info: &ConnectionInfo) -> Result<(), ConnectionFileError> {
1437    #[cfg(test)]
1438    connection_file_write_hook::run(path);
1439    write_atomic(path, info)
1440}
1441
1442/// Lets a test run code inside the connection-file write, on whichever thread
1443/// performs it, for example to block there and check what else can still run.
1444#[cfg(test)]
1445mod connection_file_write_hook {
1446    use std::{
1447        path::{Path, PathBuf},
1448        sync::{Mutex, PoisonError},
1449    };
1450
1451    type Hook = Box<dyn FnOnce() + Send>;
1452
1453    /// Keyed by connection-file path, so a hook fires only for the test that
1454    /// installed it while other tests publish their own files in parallel.
1455    static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
1456
1457    /// Run `hook` once, at the start of the next write of `path`.
1458    pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
1459        HOOKS
1460            .lock()
1461            .unwrap_or_else(PoisonError::into_inner)
1462            .push((path.to_path_buf(), Box::new(hook)));
1463    }
1464
1465    pub(super) fn run(path: &Path) {
1466        let hook = {
1467            let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
1468            hooks
1469                .iter()
1470                .position(|(hooked, _)| hooked == path)
1471                .map(|index| hooks.swap_remove(index).1)
1472        };
1473        if let Some(hook) = hook {
1474            hook();
1475        }
1476    }
1477}
1478
1479/// A path-scoped barrier after publication and before any configured activation.
1480/// The async wait leaves the listeners free to answer discovery-time requests.
1481#[cfg(test)]
1482mod startup_publication_hook {
1483    use std::{
1484        path::{Path, PathBuf},
1485        sync::{Mutex, PoisonError},
1486    };
1487    use tokio::sync::oneshot;
1488
1489    type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
1490    static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
1491
1492    pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
1493        let (entered_tx, entered_rx) = oneshot::channel();
1494        let (release_tx, release_rx) = oneshot::channel();
1495        BARRIERS
1496            .lock()
1497            .unwrap_or_else(PoisonError::into_inner)
1498            .push((path.to_path_buf(), (entered_tx, release_rx)));
1499        (entered_rx, release_tx)
1500    }
1501
1502    pub(super) async fn run(path: &Path) {
1503        let barrier = {
1504            let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
1505            barriers
1506                .iter()
1507                .position(|(hooked, _)| hooked == path)
1508                .map(|index| barriers.swap_remove(index).1)
1509        };
1510        if let Some((entered, release)) = barrier {
1511            let _ = entered.send(());
1512            let _ = release.await;
1513        }
1514    }
1515}
1516
1517fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
1518    match err {
1519        ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
1520        ConnectionFileError::JsonRead { .. }
1521        | ConnectionFileError::UnsupportedSchema { .. }
1522        | ConnectionFileError::Invalid { .. }
1523        | ConnectionFileError::KeyTooShort { .. }
1524        // A live daemon always publishes the file owner-only (0600), so a file
1525        // with insecure permissions is never a daemon we should defer to: treat it
1526        // as stale and take over (which republishes a correct 0600 file).
1527        | ConnectionFileError::InsecurePermissions { .. } => true,
1528        ConnectionFileError::MissingParent { .. }
1529        | ConnectionFileError::MissingFileName { .. }
1530        // A writable ancestor is an operator misconfiguration, never evidence
1531        // about whether a daemon is live. Reclaiming the file would republish key
1532        // material into the same directory the refusal is about.
1533        | ConnectionFileError::InsecureParentDirectory { .. }
1534        | ConnectionFileError::Io { .. }
1535        | ConnectionFileError::JsonWrite { .. }
1536        | ConnectionFileError::Random(_)
1537        // A wire mismatch may identify a newer live daemon, so never reclaim its
1538        // connection file merely because this binary cannot speak its envelope.
1539        | ConnectionFileError::WireVersionMismatch { .. } => false,
1540    }
1541}
1542
1543fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
1544    match fs::remove_file(path) {
1545        Ok(()) => Ok(()),
1546        Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
1547        Err(source) => Err(BootstrapError::RemoveStale {
1548            path: path.to_path_buf(),
1549            source,
1550        }),
1551    }
1552}
1553
1554async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
1555    bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
1556}
1557
1558async fn bind_loopback_with<F, Fut>(
1559    port: u16,
1560    mut bind_v6: F,
1561) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
1562where
1563    F: FnMut(u16) -> Fut,
1564    Fut: std::future::Future<Output = io::Result<TcpListener>>,
1565{
1566    let v4_host = Ipv4Addr::LOCALHOST;
1567    let v4 = TcpListener::bind((v4_host, port))
1568        .await
1569        .map_err(|source| BootstrapError::Bind {
1570            host: v4_host.to_string(),
1571            port,
1572            source,
1573        })?;
1574    let actual_port = v4
1575        .local_addr()
1576        .map_err(|source| BootstrapError::LocalAddr {
1577            host: v4_host.to_string(),
1578            source,
1579        })?
1580        .port();
1581
1582    let mut listeners = vec![v4];
1583    let mut endpoints = vec![Endpoint {
1584        host: v4_host.to_string(),
1585        port: actual_port,
1586    }];
1587
1588    let v6_host = Ipv6Addr::LOCALHOST;
1589    match bind_v6(actual_port).await {
1590        Ok(v6) => {
1591            listeners.push(v6);
1592            endpoints.push(Endpoint {
1593                host: v6_host.to_string(),
1594                port: actual_port,
1595            });
1596        }
1597        Err(err)
1598            if ipv6_loopback_unavailable(&err)
1599                || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
1600        {
1601            warn!(
1602                port = actual_port,
1603                error = %err,
1604                "IPv6 loopback unavailable; serving only IPv4 loopback"
1605            );
1606        }
1607        Err(source) => {
1608            drop(listeners);
1609            return Err(BootstrapError::Bind {
1610                host: v6_host.to_string(),
1611                port: actual_port,
1612                source,
1613            });
1614        }
1615    }
1616
1617    Ok((listeners, endpoints))
1618}
1619
1620fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
1621    matches!(
1622        err.kind(),
1623        io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
1624    ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
1625}
1626
1627struct AbortOnDrop<T> {
1628    handle: JoinHandle<T>,
1629}
1630
1631impl<T> AbortOnDrop<T> {
1632    fn new(handle: JoinHandle<T>) -> Self {
1633        Self { handle }
1634    }
1635
1636    async fn join(&mut self) -> Result<T, JoinError> {
1637        (&mut self.handle).await
1638    }
1639}
1640
1641impl<T> Drop for AbortOnDrop<T> {
1642    fn drop(&mut self) {
1643        if !self.handle.is_finished() {
1644            self.handle.abort();
1645        }
1646    }
1647}
1648
1649#[derive(Debug)]
1650struct StartLock {
1651    // Keep the locked file handle alive for the duration of bootstrap.
1652    _file: fs::File,
1653}
1654
1655impl Drop for StartLock {
1656    fn drop(&mut self) {
1657        // Closing alone leaves the lock held if a concurrent fork inherited
1658        // the open file description and has not exec'd yet. Unlock explicitly
1659        // so a restart can reclaim the singleton as soon as bootstrap ends.
1660        if let Err(error) = FileExt::unlock(&self._file) {
1661            tracing::warn!(%error, "failed to release the singleton start lock");
1662        }
1663    }
1664}
1665
1666impl StartLock {
1667    async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
1668        let path = start_lock_path(connection_file_path);
1669        for _ in 0..START_LOCK_RETRIES {
1670            let file = match open_owner_only_lock(&path) {
1671                Ok(file) => file,
1672                Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
1673            };
1674            match FileExt::try_lock(&file) {
1675                Ok(()) => return Ok(Self { _file: file }),
1676                Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
1677                Err(TryLockError::Error(source)) => {
1678                    return Err(BootstrapError::StartLockCreate { path, source });
1679                }
1680            }
1681        }
1682
1683        Err(BootstrapError::StartLockBusy {
1684            path,
1685            attempts: START_LOCK_RETRIES,
1686        })
1687    }
1688}
1689
1690pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
1691    let mut options = fs::OpenOptions::new();
1692    options.read(true).write(true).create(true);
1693    #[cfg(unix)]
1694    {
1695        use std::os::unix::fs::OpenOptionsExt;
1696        options.mode(0o600);
1697    }
1698    options.open(path)
1699}
1700
1701fn start_lock_path(connection_file_path: &Path) -> PathBuf {
1702    let file_name = connection_file_path
1703        .file_name()
1704        .map(|name| name.to_string_lossy())
1705        .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
1706    let lock_name = format!("{file_name}.start-lock");
1707    connection_file_path
1708        .parent()
1709        .filter(|parent| !parent.as_os_str().is_empty())
1710        .unwrap_or_else(|| Path::new("."))
1711        .join(lock_name)
1712}
1713
1714fn non_empty_os_var(key: &str) -> Option<OsString> {
1715    let value = env::var_os(key)?;
1716    if value.is_empty() {
1717        None
1718    } else {
1719        Some(value)
1720    }
1721}
1722
1723/// Bootstrap-layer errors are deliberately typed so startup never panics for
1724/// ordinary daemon-discovery races or stale filesystem state.
1725#[derive(Debug)]
1726pub enum BootstrapError {
1727    /// The bound server could not authenticate its own readiness probe.
1728    StartupNotServing,
1729    #[cfg(unix)]
1730    Signal(io::Error),
1731    InvalidPort {
1732        raw: String,
1733        source: std::num::ParseIntError,
1734    },
1735    ConnectionFileRead {
1736        path: PathBuf,
1737        source: ConnectionFileError,
1738    },
1739    ConnectionFileWrite {
1740        path: PathBuf,
1741        source: ConnectionFileError,
1742    },
1743    GenerateConnectionFile(ConnectionFileError),
1744    StartLockCreate {
1745        path: PathBuf,
1746        source: io::Error,
1747    },
1748    StartLockBusy {
1749        path: PathBuf,
1750        attempts: usize,
1751    },
1752    /// The run-directory lock file could not be created, opened or locked.
1753    RunDirLockCreate {
1754        path: PathBuf,
1755        source: io::Error,
1756    },
1757    /// Another live process, almost certainly a daemon started with a
1758    /// different runtime directory over the same data home, owns the run
1759    /// directory. The daemon does not start. `holder_pid` is read from the
1760    /// lock file when possible and is informational only.
1761    RunDirBusy {
1762        path: PathBuf,
1763        holder_pid: Option<u32>,
1764    },
1765    RemoveStale {
1766        path: PathBuf,
1767        source: io::Error,
1768    },
1769    Bind {
1770        host: String,
1771        port: u16,
1772        source: io::Error,
1773    },
1774    LocalAddr {
1775        host: String,
1776        source: io::Error,
1777    },
1778    DaemonConfig(DaemonConfigError),
1779    /// The machine id could not be established: its file is corrupt, unreadable
1780    /// or unwritable, or the data home is relative. The daemon does not start.
1781    MachineId(crate::machine_id::MachineIdFileError),
1782    /// The daemon run directory could not be resolved because the data home is
1783    /// relative. The daemon does not start.
1784    RunDir(daemon_config::DaemonRunDirError),
1785    Serve(ServerError),
1786    ServeJoin(tokio::task::JoinError),
1787}
1788
1789impl fmt::Display for BootstrapError {
1790    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1791        match self {
1792            Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
1793            #[cfg(unix)]
1794            Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
1795            Self::InvalidPort { raw, source } => {
1796                write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
1797            }
1798            Self::ConnectionFileRead { path, source } => write!(
1799                f,
1800                "failed to read connection file {}: {source}",
1801                path.display()
1802            ),
1803            Self::ConnectionFileWrite { path, source } => write!(
1804                f,
1805                "failed to publish connection file {}: {source}",
1806                path.display()
1807            ),
1808            Self::GenerateConnectionFile(err) => {
1809                write!(f, "failed to generate connection-file auth material: {err}")
1810            }
1811            Self::StartLockCreate { path, source } => {
1812                write!(
1813                    f,
1814                    "failed to create start lock {}: {source}",
1815                    path.display()
1816                )
1817            }
1818            Self::StartLockBusy { path, attempts } => write!(
1819                f,
1820                "start lock {} remained busy after {attempts} attempts",
1821                path.display()
1822            ),
1823            Self::RunDirLockCreate { path, source } => write!(
1824                f,
1825                "refusing to start: failed to lock run directory via {}: {source}",
1826                path.display()
1827            ),
1828            Self::RunDirBusy { path, holder_pid } => {
1829                write!(
1830                    f,
1831                    "refusing to start: run directory lock {} is held by another daemon",
1832                    path.display()
1833                )?;
1834                if let Some(pid) = holder_pid {
1835                    write!(f, " (pid {pid})")?;
1836                }
1837                write!(
1838                    f,
1839                    "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
1840                )
1841            }
1842            Self::RemoveStale { path, source } => write!(
1843                f,
1844                "failed to remove stale connection file {}: {source}",
1845                path.display()
1846            ),
1847            Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
1848                write!(
1849                    f,
1850                    "port {port} in use on loopback {host}: {source}; set the port in config"
1851                )
1852            }
1853            Self::Bind { host, port, source } => {
1854                write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
1855            }
1856            Self::LocalAddr { host, source } => {
1857                write!(f, "failed to read local address for {host}: {source}")
1858            }
1859            Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
1860            Self::MachineId(err) => write!(f, "refusing to start: {err}"),
1861            Self::RunDir(err) => write!(f, "refusing to start: {err}"),
1862            Self::Serve(err) => write!(f, "daemon server failed: {err}"),
1863            Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
1864        }
1865    }
1866}
1867
1868impl Error for BootstrapError {
1869    fn source(&self) -> Option<&(dyn Error + 'static)> {
1870        match self {
1871            Self::StartupNotServing => None,
1872            #[cfg(unix)]
1873            Self::Signal(source) => Some(source),
1874            Self::InvalidPort { source, .. } => Some(source),
1875            Self::ConnectionFileRead { source, .. }
1876            | Self::ConnectionFileWrite { source, .. }
1877            | Self::GenerateConnectionFile(source) => Some(source),
1878            Self::StartLockCreate { source, .. }
1879            | Self::RunDirLockCreate { source, .. }
1880            | Self::RemoveStale { source, .. }
1881            | Self::Bind { source, .. }
1882            | Self::LocalAddr { source, .. } => Some(source),
1883            Self::DaemonConfig(err) => Some(err),
1884            Self::MachineId(err) => Some(err),
1885            Self::RunDir(err) => Some(err),
1886            Self::Serve(err) => Some(err),
1887            Self::ServeJoin(err) => Some(err),
1888            Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
1889        }
1890    }
1891}
1892
1893#[cfg(test)]
1894mod tests {
1895    mod activity {
1896        include!("activity_bootstrap_tests.rs");
1897    }
1898    mod startup_order {
1899        include!("startup_order_tests.rs");
1900    }
1901    use super::*;
1902    use crate::server::ServerAuth;
1903    use cortexkit_test_support::ScratchDir;
1904    #[cfg(target_os = "linux")]
1905    use std::collections::BTreeSet;
1906    use std::sync::Mutex;
1907    #[cfg(target_os = "linux")]
1908    use subc_control::ModuleProtocol;
1909    use subc_transport::MIN_KEY_LEN;
1910    use tokio::io::AsyncReadExt;
1911    use tokio::task::JoinHandle;
1912
1913    #[cfg(unix)]
1914    use std::os::unix::fs::PermissionsExt;
1915
1916    static ENV_LOCK: Mutex<()> = Mutex::new(());
1917
1918    #[test]
1919    fn normalized_build_provenance_preserves_real_values() {
1920        assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
1921    }
1922
1923    #[test]
1924    fn normalized_build_provenance_omits_unavailable_and_empty_values() {
1925        assert_eq!(normalized_build_provenance("unavailable"), None);
1926        assert_eq!(normalized_build_provenance(""), None);
1927    }
1928
1929    #[cfg(target_os = "linux")]
1930    fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
1931        let cgroups = fs::read_to_string("/proc/self/cgroup")?;
1932        let relative = cgroups
1933            .lines()
1934            .find_map(|line| line.strip_prefix("0::"))
1935            .ok_or_else(|| {
1936                io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
1937            })?;
1938        Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
1939    }
1940
1941    #[cfg(target_os = "linux")]
1942    fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
1943        let modules = current_cgroup_path_for_test()?.join("subc-modules");
1944        let entries = match fs::read_dir(modules) {
1945            Ok(entries) => entries,
1946            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
1947            Err(error) => return Err(error),
1948        };
1949        let mut directories = BTreeSet::new();
1950        for entry in entries {
1951            let entry = entry?;
1952            if entry.file_type()?.is_dir() {
1953                directories.insert(entry.file_name());
1954            }
1955        }
1956        Ok(directories)
1957    }
1958
1959    #[cfg(target_os = "linux")]
1960    async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
1961        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
1962        while !path.exists() && tokio::time::Instant::now() < deadline {
1963            assert!(
1964                !task.is_finished(),
1965                "daemon exited before creating {}",
1966                path.display()
1967            );
1968            sleep(Duration::from_millis(10)).await;
1969        }
1970        assert!(path.exists(), "daemon did not create {}", path.display());
1971    }
1972
1973    /// An in-process daemon with the default (disabled) cgroup placement must
1974    /// leave the host's live module cgroup tree exactly as it found it.
1975    /// Red-checking this test (making it fail) creates a
1976    /// `cgroup-isolation-probe-*` directory in the live cgroup tree, which must be removed by hand.
1977    #[cfg(target_os = "linux")]
1978    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1979    async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
1980        let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
1981        let module_id = format!("cgroup-isolation-probe-{}", process::id());
1982        let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
1983        assert!(
1984            !before.contains(&OsString::from(&module_id)),
1985            "isolation probe cgroup already exists before this daemon starts"
1986        );
1987        let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
1988        let module = ConfiguredModule {
1989            module_id,
1990            program: PathBuf::from("sh"),
1991            args: vec!["-c".to_string(), "sleep 30".to_string()],
1992            env: Vec::new(),
1993            log: None,
1994            enabled: true,
1995            reserved: false,
1996            reserved_prefixes: Vec::new(),
1997            protocol: ModuleProtocol::None,
1998            overlap: Default::default(),
1999            health: HealthConfig::default(),
2000            drain_timeout_ms: None,
2001            register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2002            route_bind_relay_timeout_ms: None,
2003            restart: RestartPolicy::default(),
2004        };
2005        let config = BootstrapConfig::new(temp.join("connection.json"), 0)
2006            .with_configured_modules([module])
2007            .with_capture_logs_dir(temp.join("logs"))
2008            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2009        let task = tokio::spawn(run_with_config(config));
2010
2011        wait_for_path(&capture, &task).await;
2012        let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2013
2014        task.abort();
2015        assert!(task
2016            .await
2017            .expect_err("aborted daemon task must cancel")
2018            .is_cancelled());
2019        assert_eq!(
2020            before, after,
2021            "an in-process daemon must not create or reconcile ambient module cgroups"
2022        );
2023    }
2024
2025    #[cfg(target_os = "linux")]
2026    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2027    async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2028        let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2029        let cgroup_root = temp.join("cgroup");
2030        fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2031        fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2032        let modules = cgroup_root.join("subc-modules");
2033        let config = BootstrapConfig::new(temp.join("connection.json"), 0)
2034            .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2035            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2036        let task = tokio::spawn(run_with_config(config));
2037
2038        wait_for_path(&modules, &task).await;
2039
2040        task.abort();
2041        assert!(task
2042            .await
2043            .expect_err("aborted daemon task must cancel")
2044            .is_cancelled());
2045        assert!(
2046            fs::read_dir(&modules)
2047                .expect("read prepared modules directory")
2048                .next()
2049                .is_none(),
2050            "the delegation probe must clean up after itself"
2051        );
2052    }
2053
2054    struct EnvGuard {
2055        key: &'static str,
2056        previous: Option<OsString>,
2057    }
2058
2059    impl EnvGuard {
2060        fn set(key: &'static str, value: &Path) -> Self {
2061            let previous = env::var_os(key);
2062            env::set_var(key, value);
2063            Self { key, previous }
2064        }
2065
2066        fn set_str(key: &'static str, value: &str) -> Self {
2067            let previous = env::var_os(key);
2068            env::set_var(key, value);
2069            Self { key, previous }
2070        }
2071
2072        fn unset(key: &'static str) -> Self {
2073            let previous = env::var_os(key);
2074            env::remove_var(key);
2075            Self { key, previous }
2076        }
2077    }
2078
2079    impl Drop for EnvGuard {
2080        fn drop(&mut self) {
2081            match &self.previous {
2082                Some(value) => env::set_var(self.key, value),
2083                None => env::remove_var(self.key),
2084            }
2085        }
2086    }
2087
2088    fn unique_temp_dir(name: &str) -> ScratchDir {
2089        ScratchDir::new(name)
2090    }
2091
2092    fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2093        let dir = unique_temp_dir(name);
2094        let path = dir.join("conn.json");
2095        (dir, path)
2096    }
2097
2098    fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2099        ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2100    }
2101
2102    fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2103        let auth = auth_for(&bound.connection_info);
2104        tokio::spawn(serve_listeners(
2105            bound.listeners,
2106            Arc::new(Router::with_default_self_handler()),
2107            auth,
2108        ))
2109    }
2110
2111    fn expect_bound(outcome: Outcome) -> BoundDaemon {
2112        match outcome {
2113            Outcome::Bound(bound) => bound,
2114            Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2115        }
2116    }
2117
2118    async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2119        let endpoint = conn
2120            .endpoints
2121            .first()
2122            .expect("test connection file should have an endpoint");
2123        let ip: IpAddr = endpoint.host.parse().unwrap();
2124        TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2125    }
2126
2127    fn make_connection_info(port: u16) -> ConnectionInfo {
2128        ConnectionInfo {
2129            schema: SCHEMA_VERSION,
2130            wire_version: Some(PROTOCOL_VERSION),
2131            endpoints: vec![Endpoint {
2132                host: "127.0.0.1".to_owned(),
2133                port,
2134            }],
2135            key: generate_key().unwrap(),
2136            daemon_id: generate_daemon_id().unwrap(),
2137            pid: process::id(),
2138            daemon_ver: "test-subc".to_owned(),
2139        }
2140    }
2141
2142    fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2143        fs::write(path, contents).unwrap();
2144        #[cfg(unix)]
2145        fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2146    }
2147
2148    fn assert_owner_only_connection_file(path: &Path) {
2149        // `path` is only inspected on Unix (mode bits); on Windows the owner-only
2150        // guarantee comes from the inherited %TEMP% ACL, nothing to assert here.
2151        #[cfg(unix)]
2152        {
2153            let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2154            assert_eq!(mode, 0o600);
2155        }
2156        #[cfg(not(unix))]
2157        let _ = path;
2158    }
2159
2160    #[test]
2161    fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2162        let _env_lock = ENV_LOCK.lock().unwrap();
2163        let runtime_dir = unique_temp_dir("xdg-runtime");
2164        let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2165
2166        assert_eq!(
2167            connection_file_path(),
2168            runtime_dir.join(CONNECTION_FILE_NAME)
2169        );
2170    }
2171
2172    #[test]
2173    fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2174        let runtime_dir = OsString::from("/run/user/1000");
2175
2176        let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2177
2178        assert_eq!(
2179            path,
2180            PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2181        );
2182        assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2183    }
2184
2185    #[test]
2186    fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2187        let _env_lock = ENV_LOCK.lock().unwrap();
2188        let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2189
2190        assert_eq!(
2191            connection_file_path(),
2192            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2193        );
2194    }
2195
2196    #[test]
2197    fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2198        let (path, source) = connection_file_path_with_source(None);
2199
2200        assert_eq!(
2201            path,
2202            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2203        );
2204        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2205    }
2206
2207    /// Concurrent callers must derive one token. The former temp-file uid probe
2208    /// could fail transiently (same-tick name collision, fd exhaustion) and send
2209    /// the loser down the env-derived fallback with a different identity for the
2210    /// same user; this fence keeps identity independent of filesystem luck.
2211    #[test]
2212    fn user_connection_token_is_stable_under_concurrent_callers() {
2213        let expected = user_connection_token();
2214        let workers: Vec<_> = (0..32)
2215            .map(|_| {
2216                std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2217            })
2218            .collect();
2219        for worker in workers {
2220            for token in worker.join().expect("probe thread") {
2221                assert_eq!(token, expected, "token diverged under concurrent probes");
2222            }
2223        }
2224    }
2225
2226    #[test]
2227    fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
2228        let (path, source) = connection_file_path_with_source(Some(OsString::new()));
2229
2230        assert_eq!(
2231            path,
2232            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2233        );
2234        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2235    }
2236
2237    /// The shipped binary is the only caller that journals terminal exits into
2238    /// the real run directory, so its constructor must supply that path itself;
2239    /// the in-process default leaves it unset.
2240    #[test]
2241    fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
2242        let _env_lock = ENV_LOCK.lock().unwrap();
2243        let root = unique_temp_dir("daemon-binary-config");
2244        let data_home = root.join("data");
2245        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
2246        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2247        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2248
2249        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
2250
2251        let run_dir = data_home.join("cortexkit").join("run");
2252        assert_eq!(
2253            config.terminal_journal_path,
2254            Some(run_dir.join("terminals.jsonl"))
2255        );
2256        assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
2257        assert_eq!(
2258            BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
2259            None,
2260            "an in-process config must not journal anywhere unless asked to"
2261        );
2262    }
2263
2264    #[test]
2265    fn daemon_binary_config_refuses_a_relative_data_home() {
2266        let _env_lock = ENV_LOCK.lock().unwrap();
2267        let root = unique_temp_dir("daemon-binary-relative-data");
2268        let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
2269        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2270        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2271
2272        let error = BootstrapConfig::from_env_for_daemon_binary()
2273            .expect_err("a relative data home must refuse the daemon binary's config");
2274        assert!(
2275            matches!(error, BootstrapError::RunDir(_)),
2276            "expected a run-directory refusal, got {error}"
2277        );
2278        assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
2279    }
2280
2281    #[test]
2282    fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
2283        let _env_lock = ENV_LOCK.lock().unwrap();
2284        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2285        let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
2286        let config_path = conn_path.with_file_name("subc.jsonc");
2287        for (doc, expected) in [
2288            (None, daemon_config::PrivilegedOperatorPrompt::Log),
2289            (
2290                Some(r#"{"version":1}"#),
2291                daemon_config::PrivilegedOperatorPrompt::Log,
2292            ),
2293            (
2294                Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
2295                daemon_config::PrivilegedOperatorPrompt::Log,
2296            ),
2297            (
2298                Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
2299                daemon_config::PrivilegedOperatorPrompt::Enforce,
2300            ),
2301        ] {
2302            if let Some(doc) = doc {
2303                fs::write(&config_path, doc).unwrap();
2304            }
2305            assert_eq!(
2306                BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2307                    .unwrap()
2308                    .privileged_operator_prompt,
2309                expected
2310            );
2311            assert_eq!(
2312                BootstrapConfig::new(&conn_path, 0)
2313                    .with_daemon_config_path(&config_path)
2314                    .unwrap()
2315                    .privileged_operator_prompt,
2316                expected
2317            );
2318        }
2319    }
2320
2321    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2322    async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
2323        let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
2324        let connection_path = dir.join("connection.json");
2325        let config_path = dir.join("subc.jsonc");
2326        fs::write(
2327            &config_path,
2328            r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
2329        )
2330        .unwrap();
2331        let config = BootstrapConfig::new(&connection_path, 0)
2332            .with_daemon_config_path(&config_path)
2333            .unwrap();
2334        let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
2335        tokio::time::timeout(Duration::from_secs(5), async {
2336            while !connection_path.exists() {
2337                assert!(
2338                    !daemon.handle.is_finished(),
2339                    "daemon exited before publishing"
2340                );
2341                tokio::task::yield_now().await;
2342            }
2343        })
2344        .await
2345        .unwrap();
2346        let info = connection_file::read(&connection_path).unwrap();
2347        let mut client = connect_from_info(&info).await.unwrap();
2348        authenticate_client(&mut client, &info, Duration::from_secs(5))
2349            .await
2350            .unwrap();
2351        for (correlation, mode, expected) in [
2352            (1, "enforce", Vec::new()),
2353            (2, "log", vec!["privileged_operator_prompt"]),
2354            (3, "log", vec!["privileged_operator_prompt"]),
2355        ] {
2356            fs::write(
2357                &config_path,
2358                serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
2359            )
2360            .unwrap();
2361            let frame = crate::Frame::build(
2362                subc_protocol::FrameType::Request,
2363                subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
2364                0,
2365                0,
2366                correlation,
2367                serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
2368                    preview: true,
2369                })
2370                .unwrap(),
2371            )
2372            .unwrap();
2373            crate::write_frame(&mut client, &frame).await.unwrap();
2374            let reply =
2375                tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
2376                    .await
2377                    .unwrap()
2378                    .unwrap()
2379                    .unwrap();
2380            assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
2381            assert_eq!(reply.header.corr, correlation);
2382            let subc_control::ClientControlResponse::SupervisorRescan { result } =
2383                serde_json::from_slice(&reply.body).unwrap()
2384            else {
2385                panic!("expected a rescan response");
2386            };
2387            assert_eq!(result.restart_required, expected);
2388        }
2389    }
2390
2391    #[test]
2392    fn configured_port_uses_default_config_and_env_override() {
2393        let _env_lock = ENV_LOCK.lock().unwrap();
2394        let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
2395        let config_path = conn_path.with_file_name("subc.jsonc");
2396
2397        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2398        assert_eq!(
2399            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2400                .unwrap()
2401                .port,
2402            DEFAULT_SUBC_PORT
2403        );
2404
2405        fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
2406        assert_eq!(
2407            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2408                .unwrap()
2409                .port,
2410            8123
2411        );
2412
2413        let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
2414        assert_eq!(
2415            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2416                .unwrap()
2417                .port,
2418            9012
2419        );
2420    }
2421
2422    #[tokio::test]
2423    async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
2424        let (_dir, path) = temp_connection_file_path("departing-peer");
2425        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2426        let info = make_connection_info(listener.local_addr().unwrap().port());
2427        write_connection_file(&path, &info).unwrap();
2428        let peer = tokio::spawn(async move {
2429            let (mut stream, _) = listener.accept().await.unwrap();
2430            subc_transport::authenticate_server(
2431                &mut stream,
2432                &info.key,
2433                &info.daemon_id,
2434                &info.daemon_ver,
2435                PROBE_AUTH_DEADLINE,
2436            )
2437            .await
2438            .unwrap();
2439        });
2440        let outcome = ensure_singleton(&path, 0).await.unwrap();
2441        peer.await.unwrap();
2442        assert!(
2443            matches!(outcome, Outcome::Bound(_)),
2444            "a departing peer is not AlreadyRunning"
2445        );
2446    }
2447
2448    #[tokio::test]
2449    async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
2450        let (_dir, path) = temp_connection_file_path("already-running");
2451
2452        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2453        let server = start_server(bound);
2454
2455        let second = ensure_singleton(&path, 0).await.unwrap();
2456        assert!(matches!(second, Outcome::AlreadyRunning));
2457
2458        server.abort();
2459        let _ = server.await;
2460    }
2461
2462    #[tokio::test]
2463    async fn daemon_connection_file_publishes_protocol_wire_version() {
2464        let (_dir, path) = temp_connection_file_path("wire-version");
2465        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2466        assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
2467        assert_eq!(
2468            connection_file::read(&path).unwrap().wire_version,
2469            Some(PROTOCOL_VERSION)
2470        );
2471
2472        drop(bound.listeners);
2473    }
2474
2475    #[tokio::test]
2476    async fn stale_unbound_connection_file_is_reclaimed() {
2477        let (_dir, path) = temp_connection_file_path("stale-reclaim");
2478        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2479        let stale_port = stale.local_addr().unwrap().port();
2480        drop(stale);
2481        let stale_info = make_connection_info(stale_port);
2482        write_atomic(&path, &stale_info).unwrap();
2483
2484        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2485        assert_ne!(bound.connection_info.key, stale_info.key);
2486        drop(bound.listeners);
2487    }
2488
2489    #[cfg(unix)]
2490    #[tokio::test]
2491    async fn ensure_singleton_reclaims_insecure_connection_file() {
2492        let (_dir, path) = temp_connection_file_path("insecure-reclaim");
2493        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2494        let stale_port = stale.local_addr().unwrap().port();
2495        drop(stale);
2496        let stale_info = make_connection_info(stale_port);
2497        write_atomic(&path, &stale_info).unwrap();
2498        fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
2499
2500        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2501        assert_ne!(bound.connection_info.key, stale_info.key);
2502        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2503        assert_owner_only_connection_file(&path);
2504
2505        drop(bound.listeners);
2506    }
2507
2508    #[tokio::test]
2509    async fn ensure_singleton_reclaims_non_loopback_connection_file() {
2510        let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
2511        let mut stale_info = make_connection_info(8757);
2512        stale_info.endpoints = vec![Endpoint {
2513            host: "192.0.2.10".to_owned(),
2514            port: 8757,
2515        }];
2516        write_atomic(&path, &stale_info).unwrap();
2517
2518        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2519        assert_ne!(bound.connection_info.key, stale_info.key);
2520        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2521        assert!(bound
2522            .connection_info
2523            .endpoints
2524            .iter()
2525            .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
2526        assert_owner_only_connection_file(&path);
2527
2528        drop(bound.listeners);
2529    }
2530
2531    #[tokio::test]
2532    async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
2533        let mut unsupported_schema = make_connection_info(8757);
2534        unsupported_schema.schema = SCHEMA_VERSION + 1;
2535
2536        let mut empty_endpoints = make_connection_info(8757);
2537        empty_endpoints.endpoints.clear();
2538
2539        let mut short_key = make_connection_info(8757);
2540        short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
2541
2542        let cases = vec![
2543            (
2544                "unsupported-schema",
2545                serde_json::to_vec(&unsupported_schema).unwrap(),
2546                Some(unsupported_schema),
2547            ),
2548            (
2549                "empty-endpoints",
2550                serde_json::to_vec(&empty_endpoints).unwrap(),
2551                Some(empty_endpoints),
2552            ),
2553            (
2554                "short-key",
2555                serde_json::to_vec(&short_key).unwrap(),
2556                Some(short_key),
2557            ),
2558            ("invalid-json", b"{not valid connection json".to_vec(), None),
2559        ];
2560
2561        for (label, contents, old_info) in cases {
2562            let (_dir, path) = temp_connection_file_path(label);
2563            write_raw_owner_only_connection_file(&path, &contents);
2564
2565            let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2566            if let Some(old_info) = old_info {
2567                assert_ne!(bound.connection_info.key, old_info.key, "{label}");
2568                assert_ne!(
2569                    bound.connection_info.daemon_id, old_info.daemon_id,
2570                    "{label}"
2571                );
2572            }
2573            assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
2574            assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
2575            assert_owner_only_connection_file(&path);
2576
2577            drop(bound.listeners);
2578        }
2579    }
2580
2581    #[tokio::test]
2582    async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
2583        let (_dir, path) = temp_connection_file_path("foreign-reclaim");
2584        let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2585        let foreign_port = foreign.local_addr().unwrap().port();
2586        write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
2587        let foreign_task = tokio::spawn(async move {
2588            if let Ok((mut stream, _)) = foreign.accept().await {
2589                let mut buf = [0u8; 64];
2590                let _ = stream.read(&mut buf).await;
2591            }
2592        });
2593
2594        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2595        assert!(bound
2596            .connection_info
2597            .endpoints
2598            .iter()
2599            .all(|endpoint| endpoint.port != foreign_port));
2600
2601        drop(bound.listeners);
2602        let _ = foreign_task.await;
2603    }
2604
2605    #[tokio::test]
2606    async fn stale_start_lock_file_is_reclaimable() {
2607        let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
2608        let lock_path = start_lock_path(&path);
2609        drop(open_owner_only_lock(&lock_path).unwrap());
2610        assert!(lock_path.is_file());
2611
2612        let lock = StartLock::acquire(&path).await.unwrap();
2613        assert!(lock_path.is_file());
2614
2615        drop(lock);
2616        assert!(lock_path.is_file());
2617    }
2618
2619    #[tokio::test]
2620    async fn held_start_lock_blocks_second_acquire_until_release() {
2621        let (_dir, path) = temp_connection_file_path("start-lock-held");
2622        let lock_path = start_lock_path(&path);
2623        let first = StartLock::acquire(&path).await.unwrap();
2624
2625        let err = match StartLock::acquire(&path).await {
2626            Ok(_) => panic!("second acquire while held must stay busy"),
2627            Err(err) => err,
2628        };
2629        assert!(matches!(
2630            err,
2631            BootstrapError::StartLockBusy {
2632                ref path,
2633                attempts: START_LOCK_RETRIES,
2634            } if path == &lock_path
2635        ));
2636
2637        drop(first);
2638
2639        let second = StartLock::acquire(&path)
2640            .await
2641            .expect("released advisory lock should be reclaimable");
2642        drop(second);
2643    }
2644
2645    #[tokio::test]
2646    async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
2647        let (_dir, path) = temp_connection_file_path("bind-conflict");
2648        let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2649        let occupied_port = occupied.local_addr().unwrap().port();
2650
2651        let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
2652        assert!(matches!(
2653            err,
2654            BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
2655        ));
2656        assert!(err.to_string().contains("set the port in config"));
2657
2658        drop(occupied);
2659    }
2660
2661    #[tokio::test]
2662    async fn ephemeral_ipv6_collision_still_serves_ipv4() {
2663        let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
2664            let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
2665            let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
2666            drop(occupied);
2667            result
2668        })
2669        .await
2670        .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
2671        assert_eq!(endpoints.len(), 1);
2672        assert_eq!(endpoints[0].host, "127.0.0.1");
2673        TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
2674            .await
2675            .unwrap();
2676        drop(listeners);
2677    }
2678
2679    #[tokio::test]
2680    async fn unpublished_boot_holds_singleton_lock_until_publication() {
2681        let dir = ScratchDir::new("unpublished-singleton-lock");
2682        let path = dir.join("connection.json");
2683        let bound = expect_bound(
2684            ensure_singleton_inner(BootstrapConfig::new(&path, 0), false)
2685                .await
2686                .unwrap(),
2687        );
2688        assert!(!path.exists());
2689        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2690        assert!(
2691            matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
2692            "an unpublished boot must keep its singleton claim"
2693        );
2694        drop(bound);
2695        FileExt::try_lock(&contender).unwrap();
2696    }
2697
2698    #[tokio::test]
2699    async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
2700        let dir = ScratchDir::new("shared-singleton-lock");
2701        let path = dir.join("connection.json");
2702        let bound = expect_bound(
2703            ensure_singleton_inner(BootstrapConfig::new(&path, 0), false)
2704                .await
2705                .unwrap(),
2706        );
2707        // try_clone shares the open file description just as fork does,
2708        // without relying on the timing of another test's child spawn.
2709        let inherited = bound
2710            .publication_lock
2711            .as_ref()
2712            .unwrap()
2713            ._file
2714            .try_clone()
2715            .unwrap();
2716        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2717        assert!(matches!(
2718            FileExt::try_lock(&contender),
2719            Err(TryLockError::WouldBlock)
2720        ));
2721        drop(bound);
2722        FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
2723        drop(inherited);
2724    }
2725
2726    #[tokio::test]
2727    async fn readiness_probe_requires_an_authenticating_server() {
2728        let dir = ScratchDir::new("unserved-readiness");
2729        let bound = expect_bound(
2730            ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
2731                .await
2732                .unwrap(),
2733        );
2734        assert!(matches!(
2735            verify_serving(&bound.connection_info).await,
2736            Err(BootstrapError::StartupNotServing)
2737        ));
2738    }
2739
2740    #[cfg(unix)]
2741    #[tokio::test]
2742    async fn orphan_sweep_finishes_before_connection_file_publication() {
2743        use crate::live_children::{ExecutableIdentity, LiveChild};
2744        let dir = ScratchDir::new("publish-after-sweep");
2745        let path = dir.join("connection.json");
2746        let record = dir.join("live-children.json");
2747        let ready = dir.join("ready");
2748        let mut child = std::process::Command::new("/bin/sh")
2749            .args([
2750                "-c",
2751                "trap '' TERM; : > \"$0\"; read -r _",
2752                ready.to_str().unwrap(),
2753            ])
2754            .stdin(std::process::Stdio::piped())
2755            .env("XDG_DATA_HOME", dir.path())
2756            .env("XDG_RUNTIME_DIR", dir.path())
2757            .env("XDG_CONFIG_HOME", dir.path())
2758            .spawn()
2759            .unwrap();
2760        timeout(Duration::from_secs(5), async {
2761            while !ready.exists() {
2762                sleep(Duration::from_millis(5)).await;
2763            }
2764        })
2765        .await
2766        .unwrap();
2767        let observed = subc_os::Process::open(child.id())
2768            .unwrap()
2769            .unwrap()
2770            .observe()
2771            .unwrap();
2772        crate::live_children::write_record(
2773            &record,
2774            &[LiveChild {
2775                module_id: "old-sleep".into(),
2776                pid: child.id(),
2777                protocol: subc_control::ModuleProtocol::None,
2778                start_time: Some(observed.start_time),
2779                executable: observed.executable.map(ExecutableIdentity::from),
2780                cgroup_name: None,
2781            }],
2782        )
2783        .unwrap();
2784        let config = BootstrapConfig::new(&path, 0).with_live_children_record(&record);
2785        let daemon = tokio::spawn(run_with_config(config));
2786        timeout(Duration::from_secs(5), async {
2787            while !path.exists() {
2788                sleep(Duration::from_millis(5)).await;
2789            }
2790        })
2791        .await
2792        .unwrap();
2793        assert!(path.exists());
2794        let ended = child.try_wait().unwrap().is_some();
2795        if !ended {
2796            child.kill().unwrap();
2797        }
2798        child.wait().unwrap();
2799        if ended {
2800            assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
2801        }
2802        daemon.abort();
2803        let _ = daemon.await;
2804        assert!(
2805            ended,
2806            "the published daemon must not still owe its orphan sweep"
2807        );
2808    }
2809
2810    /// Publishing the connection file fsyncs, which can take seconds on a busy
2811    /// disk, so the daemon must do it off the async worker thread. The hook
2812    /// blocks inside the write until another task on the same single-threaded
2813    /// runtime has run. Writing on the worker thread would starve that task: the
2814    /// hook then gives up after its deadline and the test fails, as it does if
2815    /// startup publishes without going through the hooked write at all.
2816    #[tokio::test(flavor = "current_thread")]
2817    async fn connection_file_publication_leaves_the_async_runtime_free() {
2818        use std::sync::atomic::{AtomicBool, Ordering};
2819
2820        let (_dir, path) = temp_connection_file_path("publish-off-runtime");
2821        let write_started = Arc::new(AtomicBool::new(false));
2822        let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
2823        let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
2824        let hook_started = Arc::clone(&write_started);
2825        connection_file_write_hook::install(&path, move || {
2826            hook_started.store(true, Ordering::SeqCst);
2827            let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
2828            let _ = verdict_tx.send(progressed);
2829        });
2830        // Signals only after the write has started, so a message can only come
2831        // from this task running while the write is still blocked.
2832        let ticker = tokio::spawn(async move {
2833            loop {
2834                if write_started.load(Ordering::SeqCst) {
2835                    let _ = progress_tx.send(());
2836                    return;
2837                }
2838                sleep(Duration::from_millis(5)).await;
2839            }
2840        });
2841
2842        let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
2843        let published = timeout(Duration::from_secs(15), async {
2844            while !path.exists() && !daemon.is_finished() {
2845                sleep(Duration::from_millis(5)).await;
2846            }
2847        })
2848        .await;
2849        let daemon_exited = daemon.is_finished();
2850        daemon.abort();
2851        let daemon_result = daemon.await;
2852        ticker.abort();
2853
2854        assert!(
2855            published.is_ok() && !daemon_exited,
2856            "the daemon must start and publish its connection file: {daemon_result:?}"
2857        );
2858        match verdict_rx.try_recv() {
2859            Ok(true) => {}
2860            Ok(false) => panic!(
2861                "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
2862            ),
2863            Err(_) => panic!("startup published the connection file without the hooked write"),
2864        }
2865    }
2866
2867    /// A write that never reports back must stop startup with an error naming
2868    /// the connection file, never be taken as published.
2869    #[tokio::test]
2870    async fn a_panicking_connection_file_write_fails_startup() {
2871        let (_dir, path) = temp_connection_file_path("publish-panics");
2872        connection_file_write_hook::install(&path, || {
2873            panic!("deliberate panic inside the connection-file write")
2874        });
2875
2876        let error = timeout(
2877            Duration::from_secs(15),
2878            run_with_config(BootstrapConfig::new(&path, 0)),
2879        )
2880        .await
2881        .expect("startup must end once the write task fails")
2882        .expect_err("startup must fail when the connection-file write panics");
2883
2884        assert!(
2885            matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
2886            "expected a connection-file write error, got {error:?}"
2887        );
2888        let message = error.to_string();
2889        assert!(message.contains(&path.display().to_string()), "{message}");
2890        assert!(message.contains("panicked"), "{message}");
2891        assert!(!path.exists(), "no connection file may be published");
2892    }
2893
2894    #[cfg(target_os = "macos")]
2895    #[test]
2896    fn nofile_raise_worker() {
2897        let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
2898            return;
2899        };
2900        let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
2901        rlimit::Resource::NOFILE.set(256, hard).unwrap();
2902        raise_nofile_limit_with_ceiling(Some(10240));
2903        std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
2904    }
2905
2906    #[cfg(target_os = "macos")]
2907    #[test]
2908    fn macos_nofile_raise_clamps_to_kernel_ceiling() {
2909        let dir = ScratchDir::new("nofile-ceiling");
2910        let result = dir.join("soft");
2911        let status = std::process::Command::new(std::env::current_exe().unwrap())
2912            .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
2913            .env("SUBC_NOFILE_TEST_FILE", &result)
2914            .env("XDG_DATA_HOME", dir.path())
2915            .env("XDG_RUNTIME_DIR", dir.path())
2916            .env("XDG_CONFIG_HOME", dir.path())
2917            .status()
2918            .unwrap();
2919        assert!(status.success());
2920        let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
2921        assert!(
2922            soft > 256 && soft <= 10240,
2923            "the inherited limit must be raised within the kernel ceiling; got {soft}"
2924        );
2925    }
2926
2927    #[tokio::test]
2928    async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
2929        let (_dir, path) = temp_connection_file_path("key-rotation");
2930        let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2931        let old_info = first.connection_info.clone();
2932        let fixed_port = old_info.endpoints[0].port;
2933        drop(first.listeners);
2934
2935        let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
2936        let new_info = second.connection_info.clone();
2937        assert_ne!(old_info.key, new_info.key);
2938        assert_ne!(old_info.daemon_id, new_info.daemon_id);
2939        let server = start_server(second);
2940
2941        let mut old_stream = connect_from_info(&old_info).await.unwrap();
2942        let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
2943        assert!(
2944            old_auth.is_err(),
2945            "old key must not authenticate after restart"
2946        );
2947
2948        let reread = connection_file::read(&path).unwrap();
2949        let mut new_stream = connect_from_info(&reread).await.unwrap();
2950        authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
2951            .await
2952            .unwrap();
2953
2954        server.abort();
2955        let _ = server.await;
2956    }
2957
2958    #[cfg(unix)]
2959    #[tokio::test]
2960    async fn published_connection_file_permissions_are_owner_only() {
2961        let (_dir, path) = temp_connection_file_path("permissions");
2962        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2963
2964        let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
2965        assert_eq!(mode, 0o600);
2966
2967        drop(bound.listeners);
2968    }
2969}