Skip to main content

subc_daemon/
bootstrap.rs

1use std::{
2    collections::BTreeMap,
3    env,
4    error::Error,
5    ffi::OsString,
6    fmt, fs, io,
7    net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8    path::{Path, PathBuf},
9    process,
10    time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17    authenticate_client, connection_file, generate_daemon_id, generate_key,
18    write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
19    LocalEndpoint, SCHEMA_VERSION,
20};
21use tokio::{
22    net::{TcpListener, TcpStream},
23    task::{JoinError, JoinHandle},
24    time::{sleep, timeout},
25};
26use tracing::{error, info, warn};
27
28use crate::{
29    daemon_config::{self, ConfiguredModule, DaemonConfigError},
30    local_listener::{self, BoundLocalListener, LocalListenerConfig},
31    server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
32    supervise::HealthConfig,
33    ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
34    ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
35    SupervisorProcessLiveness,
36};
37use std::sync::Arc;
38
39pub const DEFAULT_SUBC_PORT: u16 = 8757;
40pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
41use subc_transport::CONNECTION_FILE_NAME;
42const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
43const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
44const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
45const START_LOCK_RETRIES: usize = 40;
46const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
47
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
49pub enum ConnectionFileSource {
50    XdgRuntimeDir,
51    TempDirFallback,
52    Explicit,
53}
54
55impl ConnectionFileSource {
56    fn reason(self) -> &'static str {
57        match self {
58            Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
59            Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
60            Self::Explicit => "configured path",
61        }
62    }
63}
64
65impl fmt::Display for ConnectionFileSource {
66    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
67        formatter.write_str(match self {
68            Self::XdgRuntimeDir => "xdg_runtime_dir",
69            Self::TempDirFallback => "temp_dir_fallback",
70            Self::Explicit => "explicit",
71        })
72    }
73}
74
75/// Runtime bootstrap configuration. Production uses the default fixed port and
76/// optional daemon-config override; tests pass port 0 to let the OS assign a free
77/// loopback port and discover it from the connection file.
78#[derive(Debug, Clone, Default)]
79struct AdmissionFactsConfig {
80    carrier_module_id: Option<String>,
81    targets: Option<Vec<String>>,
82}
83
84/// Controls where module cgroups are prepared.
85///
86/// In-process daemons default to [`Self::Disabled`] so they never derive a
87/// production location from the host process. The shipped daemon explicitly uses
88/// [`Self::Current`]. The `ck-subc` binary accepts
89/// `SUBC_CGROUP_PLACEMENT=disabled` for isolated test processes; an unset
90/// variable retains `Current`, and every other value is rejected at startup.
91/// Tests that exercise placement can own a [`Self::Root`].
92#[derive(Debug, Clone, Default, PartialEq, Eq)]
93pub enum CgroupPlacementConfig {
94    #[default]
95    Disabled,
96    Current,
97    Root(PathBuf),
98}
99
100#[derive(Debug, Clone)]
101pub struct BootstrapConfig {
102    pub operator_queue_wait: Duration,
103    pub operator_timeout: Duration,
104    pub operator_stuck_grace: Duration,
105    #[cfg(feature = "test-support")]
106    pub operator_script: Option<PathBuf>,
107    #[cfg(feature = "test-support")]
108    pub operator_events: Option<PathBuf>,
109    pub connection_file_path: PathBuf,
110    pub port: u16,
111    pub daemon_ver: String,
112    configured_modules: Vec<ConfiguredModule>,
113    storage_config: Option<daemon_config::StorageConfig>,
114    admission_facts: AdmissionFactsConfig,
115    /// Module ids whose scopes may carry the attributes that grant authority.
116    scope_authority_owners: Vec<String>,
117    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
118    activity_config: daemon_config::ActivityConfig,
119    daemon_config_path: Option<PathBuf>,
120    configured_port: Option<u16>,
121    /// Daemon-wide route.bind relay budget in milliseconds (the fallback for
122    /// any module without a per-module override). `None` = built-in default
123    /// (12s — see `control::DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`).
124    route_bind_relay_default_ms: Option<u64>,
125    reserved_capabilities: BTreeMap<String, String>,
126    watchdog_config: DaemonSelfWatchdogConfig,
127    connection_file_source: ConnectionFileSource,
128    /// Where module cgroups are prepared. Disabled unless a caller explicitly
129    /// opts in, because `Current` derives a host location from `/proc/self/cgroup`.
130    cgroup_placement: CgroupPlacementConfig,
131    /// Directory the supervisor writes per-module stdout/stderr capture files
132    /// into. `None` disables capture; the shipped binary supplies its real run
133    /// directory explicitly.
134    capture_logs_dir: Option<PathBuf>,
135    /// File the supervisor appends every module's terminal exits to, so exit
136    /// history survives a daemon restart. `None` keeps terminal history in
137    /// memory only (each module's ring). Absent by default for the same reason
138    /// as `capture_logs_dir`: an in-process daemon booted by a test must not
139    /// append its exits to the operator's real `terminals.jsonl`, where they
140    /// would show up in `ck module terminals`. The shipped binary supplies
141    /// `<run dir>/terminals.jsonl` explicitly.
142    terminal_journal_path: Option<PathBuf>,
143    /// Where the machine id is read from, or minted into when absent. `None`
144    /// serves no machine id. Absent by default for the same reason as
145    /// `capture_logs_dir`: an in-process daemon booted by a test must never
146    /// derive the operator's real data home and mint into it. The shipped binary
147    /// supplies `<data home>/cortexkit/machine-id` explicitly.
148    machine_id_path: Option<PathBuf>,
149
150    /// Where this daemon's same-user local listener lives (a Unix-domain socket
151    /// on macOS and Linux; nothing on Windows, where the pipe name is minted per
152    /// start). The default serves no local endpoint, so an in-process daemon
153    /// never touches the operator's run directory. The shipped binary names its
154    /// run directory; [`BootstrapConfig::with_local_listener_socket_dir`] names
155    /// any other, and an in-process daemon that wants the endpoint names one.
156    local_listener: LocalListenerConfig,
157    /// The live-children record: every supervised process this daemon has
158    /// running, kept so the next daemon can end the ones a crash left behind.
159    /// At startup, before any module is spawned, the previous daemon's record
160    /// here is swept. `None` keeps no record and sweeps nothing, for the same
161    /// reason as `capture_logs_dir`: an in-process daemon booted by a test
162    /// must never signal processes listed in the operator's real run
163    /// directory. The shipped binary supplies `<run dir>/live-children.json`.
164    live_children_path: Option<PathBuf>,
165    privacy_trampoline: Option<PathBuf>,
166}
167
168impl BootstrapConfig {
169    pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
170        Self {
171            connection_file_path: connection_file_path.into(),
172            operator_queue_wait: Duration::from_secs(150),
173            operator_timeout: Duration::from_secs(120),
174            operator_stuck_grace: Duration::from_secs(10),
175            #[cfg(feature = "test-support")]
176            operator_script: None,
177            #[cfg(feature = "test-support")]
178            operator_events: None,
179            port,
180            daemon_ver: DAEMON_VERSION.to_owned(),
181            configured_modules: Vec::new(),
182            storage_config: None,
183            admission_facts: AdmissionFactsConfig::default(),
184            scope_authority_owners: daemon_config::default_scope_authority_owners(),
185            privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
186            activity_config: daemon_config::ActivityConfig::default(),
187            daemon_config_path: None,
188            configured_port: None,
189            route_bind_relay_default_ms: None,
190            reserved_capabilities: BTreeMap::new(),
191            watchdog_config: DaemonSelfWatchdogConfig::default(),
192            connection_file_source: ConnectionFileSource::Explicit,
193            cgroup_placement: CgroupPlacementConfig::default(),
194            capture_logs_dir: None,
195            terminal_journal_path: None,
196            machine_id_path: None,
197            local_listener: LocalListenerConfig { socket_dir: None },
198            live_children_path: None,
199            privacy_trampoline: None,
200        }
201    }
202
203    /// Keep the local listener's socket in `dir`. Embedding daemons and tests
204    /// pass a path inside their own fixture tree.
205    pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
206        self.local_listener = LocalListenerConfig::in_socket_dir(dir);
207        self
208    }
209
210    /// Set where the local listener lives, as [`LocalListenerConfig`] describes.
211    pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
212        self.local_listener = config;
213        self
214    }
215
216    /// Keep the live-children record at `path`, and at startup end the
217    /// processes a previous daemon recorded there that are still running.
218    /// Embedding daemons and tests pass a path inside their own fixture tree.
219    pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
220        self.live_children_path = Some(path.into());
221        self
222    }
223
224    /// macOS module launches require an explicit executable implementing the
225    /// subc-os trampoline entry point. Other systems ignore this setting.
226    pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
227        self.privacy_trampoline = Some(path.into());
228        self
229    }
230
231    /// Serve the machine id stored at `path`, minting it there at startup when
232    /// the file is absent. Embedding daemons and tests pass a path inside their
233    /// own fixture tree.
234    pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
235        self.machine_id_path = Some(path.into());
236        self
237    }
238
239    /// Selects module cgroup placement. The default is disabled.
240    pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
241        self.cgroup_placement = placement;
242        self
243    }
244
245    /// Redirects per-module stdout/stderr capture files out of the real run
246    /// directory. Tests that start an in-process daemon must call this with a
247    /// path inside their fixture tree.
248    pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
249        self.capture_logs_dir = Some(dir.into());
250        self
251    }
252
253    /// Redirects the daemon-private journal, allowing embedded daemons and tests
254    /// to keep their observations out of the operator's live run directory.
255    pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
256        self.terminal_journal_path = Some(path.into());
257        self
258    }
259
260    pub fn from_env() -> Result<Self, BootstrapError> {
261        Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
262    }
263
264    /// The SHIPPED BINARY's config, which is the only caller that should capture
265    /// child output into the operator's real run directory.
266    ///
267    /// Kept separate from `from_env` deliberately: see `capture_logs_dir` and
268    /// `terminal_journal_path` on this struct for why an absent value must mean
269    /// NO CAPTURE and NO JOURNAL rather than the operator's real run directory.
270    /// A run directory that cannot be resolved (a relative data home) refuses
271    /// startup instead of landing under the working directory.
272    pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
273        let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
274        let machine_id_path =
275            crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
276        let config = Self::from_env()?
277            .with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
278            .with_capture_logs_dir(run_dir.join("logs"))
279            .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
280            .with_live_children_record(crate::live_children::record_path(&run_dir))
281            .with_machine_id_path(machine_id_path);
282        #[cfg(feature = "test-support")]
283        let config = {
284            let mut config = config;
285            config.operator_script =
286                std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
287            config.operator_events =
288                std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
289            for (name, value) in [
290                (
291                    "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
292                    &mut config.operator_queue_wait,
293                ),
294                (
295                    "SUBC_TEST_OPERATOR_TIMEOUT_MS",
296                    &mut config.operator_timeout,
297                ),
298                (
299                    "SUBC_TEST_OPERATOR_STUCK_MS",
300                    &mut config.operator_stuck_grace,
301                ),
302            ] {
303                if let Ok(ms) = std::env::var(name) {
304                    if let Ok(ms) = ms.parse::<u64>() {
305                        *value = Duration::from_millis(ms);
306                    }
307                }
308            }
309            config
310        };
311        Ok(config)
312    }
313
314    pub fn from_env_with_daemon_config_path(
315        daemon_config_path: impl AsRef<Path>,
316    ) -> Result<Self, BootstrapError> {
317        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
318        let (daemon_config, activity_config) =
319            daemon_config::load_with_activity(&daemon_config_path)
320                .map_err(BootstrapError::DaemonConfig)?
321                .map(|(config, activity)| (Some(config), activity))
322                .unwrap_or_default();
323        let config_port = daemon_config.as_ref().and_then(|config| config.port);
324        let storage_config = daemon_config
325            .as_ref()
326            .and_then(|config| config.storage.clone());
327        let admission_facts_carrier_module_id = daemon_config
328            .as_ref()
329            .and_then(|config| config.admission_facts_carrier_module_id.clone());
330        let admission_facts_targets = daemon_config
331            .as_ref()
332            .and_then(|config| config.admission_facts_targets.clone());
333        let scope_authority_owners = daemon_config
334            .as_ref()
335            .map(|config| config.scope_authority_owners.clone())
336            .unwrap_or_else(daemon_config::default_scope_authority_owners);
337        let privileged_operator_prompt = daemon_config
338            .as_ref()
339            .map(|config| config.privileged_operator_prompt)
340            .unwrap_or_default();
341        let route_bind_relay_default_ms = daemon_config
342            .as_ref()
343            .and_then(|config| config.route_bind_relay_timeout_ms);
344        let reserved_capabilities = daemon_config
345            .as_ref()
346            .map(|config| config.reserved_capabilities.clone())
347            .unwrap_or_default();
348        let configured_modules = daemon_config
349            .map(|config| config.modules)
350            .unwrap_or_default();
351
352        let port = match env::var(SUBC_PORT_ENV) {
353            Ok(raw) if !raw.trim().is_empty() => {
354                let port = raw
355                    .parse::<u16>()
356                    .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
357                if let Some(config_port) = config_port {
358                    info!(
359                        env = SUBC_PORT_ENV,
360                        env_port = port,
361                        config_port,
362                        "SUBC_PORT overrides daemon config port"
363                    );
364                }
365                port
366            }
367            Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
368        };
369
370        let (connection_file_path, connection_file_source) =
371            connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
372        Ok(Self::new(connection_file_path, port)
373            .with_configured_modules(configured_modules)
374            .with_storage_config(storage_config)
375            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
376            .with_scope_authority_owners(scope_authority_owners)
377            .with_privileged_operator_prompt(privileged_operator_prompt)
378            .with_activity_config(activity_config)
379            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
380            .with_reserved_capabilities(reserved_capabilities)
381            .with_daemon_config_source(daemon_config_path, config_port)
382            .with_connection_file_source(connection_file_source))
383    }
384
385    pub fn with_daemon_config_path(
386        self,
387        daemon_config_path: impl AsRef<Path>,
388    ) -> Result<Self, BootstrapError> {
389        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
390        let (daemon_config, activity_config) =
391            daemon_config::load_with_activity(&daemon_config_path)
392                .map_err(BootstrapError::DaemonConfig)?
393                .map(|(config, activity)| (Some(config), activity))
394                .unwrap_or_default();
395        let configured_port = daemon_config.as_ref().and_then(|config| config.port);
396        let storage_config = daemon_config
397            .as_ref()
398            .and_then(|config| config.storage.clone());
399        let admission_facts_carrier_module_id = daemon_config
400            .as_ref()
401            .and_then(|config| config.admission_facts_carrier_module_id.clone());
402        let admission_facts_targets = daemon_config
403            .as_ref()
404            .and_then(|config| config.admission_facts_targets.clone());
405        let scope_authority_owners = daemon_config
406            .as_ref()
407            .map(|config| config.scope_authority_owners.clone())
408            .unwrap_or_else(daemon_config::default_scope_authority_owners);
409        let privileged_operator_prompt = daemon_config
410            .as_ref()
411            .map(|config| config.privileged_operator_prompt)
412            .unwrap_or_default();
413        let route_bind_relay_default_ms = daemon_config
414            .as_ref()
415            .and_then(|config| config.route_bind_relay_timeout_ms);
416        let reserved_capabilities = daemon_config
417            .as_ref()
418            .map(|config| config.reserved_capabilities.clone())
419            .unwrap_or_default();
420        let configured_modules = daemon_config
421            .map(|config| config.modules)
422            .unwrap_or_default();
423        Ok(self
424            .with_configured_modules(configured_modules)
425            .with_storage_config(storage_config)
426            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
427            .with_scope_authority_owners(scope_authority_owners)
428            .with_privileged_operator_prompt(privileged_operator_prompt)
429            .with_activity_config(activity_config)
430            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
431            .with_reserved_capabilities(reserved_capabilities)
432            .with_daemon_config_source(daemon_config_path, configured_port))
433    }
434
435    pub fn with_configured_modules(
436        mut self,
437        modules: impl IntoIterator<Item = ConfiguredModule>,
438    ) -> Self {
439        self.configured_modules = modules.into_iter().collect();
440        self.configured_modules
441            .sort_by(|left, right| left.module_id.cmp(&right.module_id));
442        self
443    }
444
445    pub fn with_storage_config(
446        mut self,
447        storage_config: Option<daemon_config::StorageConfig>,
448    ) -> Self {
449        self.storage_config = storage_config;
450        self
451    }
452
453    pub fn with_admission_facts_config(
454        mut self,
455        carrier_module_id: Option<String>,
456        targets: Option<Vec<String>>,
457    ) -> Self {
458        self.admission_facts = AdmissionFactsConfig {
459            carrier_module_id,
460            targets,
461        };
462        self
463    }
464
465    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
466    /// Absent from a config, this is `daemon_config::default_scope_authority_owners`.
467    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
468        self.scope_authority_owners = owners;
469        self
470    }
471
472    /// Choose the privileged-operation policy for the daemon's entire lifetime.
473    pub fn with_privileged_operator_prompt(
474        mut self,
475        mode: daemon_config::PrivilegedOperatorPrompt,
476    ) -> Self {
477        self.privileged_operator_prompt = mode;
478        self
479    }
480
481    pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
482        self.activity_config = config;
483        self
484    }
485
486    /// Set the daemon-wide route.bind relay default (the fallback for any
487    /// module without a per-module override). `None` preserves the built-in
488    /// default (12s). `serve_bound_daemon` reads this at startup and threads
489    /// it into the control handler's daemon-wide field.
490    pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
491        self.route_bind_relay_default_ms = ms;
492        self
493    }
494
495    pub fn with_reserved_capabilities(
496        mut self,
497        reserved_capabilities: BTreeMap<String, String>,
498    ) -> Self {
499        self.reserved_capabilities = reserved_capabilities;
500        self
501    }
502
503    fn with_daemon_config_source(
504        mut self,
505        daemon_config_path: PathBuf,
506        configured_port: Option<u16>,
507    ) -> Self {
508        self.daemon_config_path = Some(daemon_config_path);
509        self.configured_port = configured_port;
510        self
511    }
512
513    fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
514        self.connection_file_source = source;
515        self
516    }
517
518    pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
519        self.watchdog_config = watchdog_config;
520        self
521    }
522}
523
524/// Result of singleton discovery.
525// Built once per daemon start and immediately matched, so the size gap between
526// the variants costs nothing worth a box on the public shape.
527#[allow(clippy::large_enum_variant)]
528#[derive(Debug)]
529pub enum Outcome {
530    /// A live daemon authenticated from the connection file; this invocation should exit 0.
531    AlreadyRunning,
532    /// This process won the singleton race, owns bound loopback listener(s), and
533    /// has published a fresh connection file.
534    Bound(BoundDaemon),
535}
536
537#[derive(Debug)]
538pub struct BoundDaemon {
539    pub listeners: Vec<TcpListener>,
540    /// The same-user local listener, bound and not yet served. `None` when
541    /// binding it failed, in which case `local_endpoint` is `None` and
542    /// `local_endpoint_unavailable` says why.
543    pub local_listener: Option<BoundLocalListener>,
544    /// Why the daemon serves no local endpoint, when binding it failed. Reported
545    /// by `server.describe` so an operator can see it.
546    pub local_endpoint_unavailable: Option<String>,
547    /// What the connection file publishes under `local_endpoint`, so a client can
548    /// find the local listener without knowing how it was bound.
549    pub local_endpoint: Option<LocalEndpoint>,
550    pub connection_info: ConnectionInfo,
551    pub connection_file_path: PathBuf,
552    pub connection_file_source: ConnectionFileSource,
553    /// The machine id this daemon serves, established before any connection is
554    /// accepted. `None` when the config named no machine id path.
555    pub machine_id: Option<crate::machine_id::MachineId>,
556    /// Ownership of the run directory, held until the daemon exits so no other
557    /// daemon sweeps or writes this one's run state. `None` when the config
558    /// keeps no live-children record, and so has no run directory to own.
559    run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
560    /// The singleton start lock, kept when binding deferred publishing the
561    /// connection file. It stays held until the daemon has verified it is serving
562    /// and written the connection file, so a second daemon starting meanwhile
563    /// cannot claim the singleton too. `None` when the connection
564    /// file was already written at bind time.
565    publication_lock: Option<StartLock>,
566}
567
568/// Resolve subc's per-user TCP connection-file path.
569///
570/// `$XDG_RUNTIME_DIR/subc-connection.json` is preferred because the runtime
571/// directory is already per-user on Unix desktops. Without it, subc falls back
572/// to the system temp dir with a per-user token in the filename so different OS
573/// users do not collide on shared temp directories.
574pub fn connection_file_path() -> PathBuf {
575    connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
576}
577
578fn connection_file_path_with_source(
579    runtime_dir: Option<OsString>,
580) -> (PathBuf, ConnectionFileSource) {
581    if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
582        return (
583            PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
584            ConnectionFileSource::XdgRuntimeDir,
585        );
586    }
587
588    (
589        env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
590        ConnectionFileSource::TempDirFallback,
591    )
592}
593
594/// Resolve, claim, and serve the per-user daemon singleton.
595///
596/// A second invocation is successful: if a live daemon authenticates from the
597/// existing connection file, this returns `Ok(())` after logging and the caller
598/// exits with status 0.
599pub async fn run() -> Result<(), BootstrapError> {
600    // `run` is a binary entry point, so it opts into the production cgroup and
601    // child-log locations. In-process callers keep both features disabled by default.
602    run_with_config(
603        BootstrapConfig::from_env_for_daemon_binary()?
604            .with_cgroup_placement(CgroupPlacementConfig::Current),
605    )
606    .await
607}
608
609/// Serve a daemon from an explicit config. This is the entry point the twelve
610/// sibling repos use to boot an in-process daemon in their integration tests.
611///
612/// # THIS INSTALLS NO TRACING SUBSCRIBER, SO THE DAEMON IS SILENT BY DEFAULT
613///
614/// The daemon's own diagnostics go through `tracing`, and `tracing` DISCARDS
615/// every event when no subscriber is installed. The shipped binary installs one
616/// in `main` (`init_tracing`); this function deliberately does not, because a
617/// library that installs a global subscriber fights with whatever the host
618/// process already set up.
619///
620/// The consequence for a test harness is not "less verbose": it is that the
621/// daemon has NOTHING TO SAY about any failure, and a missing instrument reads
622/// exactly like a clean one. PLEX found this on 2026-09-18 while trying to
623/// capture daemon logs beside an intermittent bind failure, and discovered the
624/// daemon had been silent in every conformance run that repo had ever done --
625/// so the one client-side error string was all the evidence that could exist,
626/// and they had spent a real investigation on a failure whose second source was
627/// never being recorded.
628///
629/// Install one in the harness before calling this, and assert it did something
630/// (they measured 236 daemon lines with the subscriber installed, 0 with the
631/// call commented out) -- otherwise the fix is itself unverified.
632pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
633    let forwarding = Arc::new(ForwardingTable::default());
634    forwarding.operator_confirms().configure_bootstrap(&config);
635    let configured_modules = config.configured_modules.clone();
636    let storage_config = config.storage_config.clone();
637    let admission_facts = config.admission_facts.clone();
638    let scope_authority_owners = config.scope_authority_owners.clone();
639    let privileged_operator_prompt = config.privileged_operator_prompt;
640    let activity_config = config.activity_config;
641    let daemon_config_path = config.daemon_config_path.clone();
642    let configured_port = config.configured_port;
643    let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
644    let reserved_capabilities = config.reserved_capabilities.clone();
645    let watchdog_config = config.watchdog_config.clone();
646    let cgroup_placement_config = config.cgroup_placement.clone();
647    let capture_logs_dir = config.capture_logs_dir.clone();
648    let terminal_journal_path = config.terminal_journal_path.clone();
649    let live_children_path = config.live_children_path.clone();
650    let privacy_trampoline = config.privacy_trampoline.clone();
651    match ensure_singleton_inner(config, false).await? {
652        Outcome::AlreadyRunning => {
653            info!("subc daemon already running");
654            Ok(())
655        }
656        Outcome::Bound(bound) => {
657            #[cfg(target_os = "linux")]
658            let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
659            #[cfg(not(target_os = "linux"))]
660            let _ = cgroup_placement_config;
661            serve_bound_daemon(
662                bound,
663                forwarding,
664                configured_modules,
665                storage_config,
666                admission_facts,
667                scope_authority_owners,
668                privileged_operator_prompt,
669                activity_config,
670                daemon_config_path,
671                configured_port,
672                route_bind_relay_default_ms,
673                reserved_capabilities,
674                watchdog_config,
675                capture_logs_dir,
676                terminal_journal_path,
677                live_children_path,
678                privacy_trampoline,
679                #[cfg(target_os = "linux")]
680                cgroup_placement,
681            )
682            .await
683        }
684    }
685}
686
687#[cfg(target_os = "linux")]
688fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
689    let result = match config {
690        CgroupPlacementConfig::Disabled => return None,
691        CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
692        CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
693    };
694
695    match result {
696        Ok(Some(placement)) => Some(placement),
697        Ok(None) => {
698            warn!(
699                placement = ?config,
700                "module cgroup placement is disabled: configured cgroup root is not delegated"
701            );
702            None
703        }
704        Err(error) => {
705            warn!(
706                placement = ?config,
707                error = %error,
708                "module cgroup placement is disabled by an unexpected cgroup probe error"
709            );
710            None
711        }
712    }
713}
714
715/// Target soft limit for open file descriptors, applied to the daemon before any
716/// module is spawned so children inherit it. Multi-root modules (one process
717/// aggregating every project root's sqlite stores, index caches, watchers, and
718/// LSP pipes) trivially exceed the macOS default soft limit of 256; a launchd
719/// user agent does not pass login-shell ulimits through, so the raise must
720/// happen in-process.
721#[cfg(unix)]
722const NOFILE_TARGET: u64 = 65536;
723
724/// Raise RLIMIT_NOFILE to `NOFILE_TARGET` (clamped to the hard limit).
725/// Best-effort: failure is logged and never fatal, since the daemon can run
726/// under the inherited limit — modules with few roots just have less headroom.
727#[cfg(unix)]
728fn raise_nofile_limit() {
729    #[cfg(target_os = "macos")]
730    let ceiling = std::process::Command::new("/usr/sbin/sysctl")
731        .args(["-n", "kern.maxfilesperproc"])
732        .output()
733        .ok()
734        .filter(|output| output.status.success())
735        .and_then(|output| String::from_utf8(output.stdout).ok())
736        .and_then(|value| value.trim().parse::<u64>().ok())
737        .filter(|value| *value > 0);
738    #[cfg(not(target_os = "macos"))]
739    let ceiling = None;
740    raise_nofile_limit_with_ceiling(ceiling);
741}
742
743#[cfg(unix)]
744fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
745    match rlimit::Resource::NOFILE.get() {
746        Ok((soft, hard)) => {
747            // On XNU an infinite hard limit does not remove the separate
748            // per-process kernel ceiling. Exceeding it makes setrlimit fail
749            // without raising the inherited (often 256) soft limit at all.
750            let target = NOFILE_TARGET
751                .min(hard)
752                .min(kernel_ceiling.unwrap_or(u64::MAX));
753            if soft >= target {
754                return;
755            }
756            match rlimit::Resource::NOFILE.set(target, hard) {
757                Ok(()) => info!(
758                    previous_soft = soft,
759                    new_soft = target,
760                    hard,
761                    "raised open-file soft limit for daemon and module children"
762                ),
763                Err(err) => warn!(
764                    soft,
765                    hard,
766                    error = %err,
767                    "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
768                ),
769            }
770        }
771        Err(err) => warn!(error = %err, "could not read open-file limit"),
772    }
773}
774
775/// CRT stdio-stream target on Windows (the `_setmaxstdio` maximum). Win32
776/// HANDLEs — what Rust `File`, tokio sockets, and SQLite's Win32 VFS actually
777/// consume — have a per-process quota in the millions and need no raise; the
778/// C-runtime stream table (default 512) is the only low ceiling, and it is
779/// per-process rather than inherited, so supervised modules linking the CRT
780/// must raise their own. Raising it here covers the daemon itself.
781#[cfg(windows)]
782fn raise_nofile_limit() {
783    const MAXSTDIO_TARGET: u32 = 8192;
784    let current = rlimit::getmaxstdio();
785    if current >= MAXSTDIO_TARGET {
786        return;
787    }
788    match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
789        Ok(new_max) => info!(
790            previous = current,
791            new_max, "raised CRT stdio-stream limit for daemon"
792        ),
793        Err(err) => warn!(
794            current,
795            error = %err,
796            "could not raise CRT stdio-stream limit"
797        ),
798    }
799}
800
801#[cfg(not(any(unix, windows)))]
802fn raise_nofile_limit() {}
803
804pub async fn run_with_daemon_config_path(
805    config: BootstrapConfig,
806    daemon_config_path: impl AsRef<Path>,
807) -> Result<(), BootstrapError> {
808    run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
809}
810
811#[allow(clippy::too_many_arguments)]
812async fn serve_bound_daemon(
813    bound: BoundDaemon,
814    forwarding: Arc<ForwardingTable>,
815    configured_modules: Vec<ConfiguredModule>,
816    storage_config: Option<daemon_config::StorageConfig>,
817    admission_facts: AdmissionFactsConfig,
818    scope_authority_owners: Vec<String>,
819    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
820    activity_config: daemon_config::ActivityConfig,
821    daemon_config_path: Option<PathBuf>,
822    configured_port: Option<u16>,
823    route_bind_relay_default_ms: Option<u64>,
824    reserved_capabilities: BTreeMap<String, String>,
825    watchdog_config: DaemonSelfWatchdogConfig,
826    capture_logs_dir: Option<PathBuf>,
827    terminal_journal_path: Option<PathBuf>,
828    live_children_path: Option<PathBuf>,
829    privacy_trampoline: Option<PathBuf>,
830    #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
831) -> Result<(), BootstrapError> {
832    #[cfg(unix)]
833    let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
834        .map_err(BootstrapError::Signal)?;
835    // Windows has no SIGTERM. Ctrl-C is a different event, not an equivalent
836    // service-stop contract, so this shutdown handler is intentionally Unix-only.
837    raise_nofile_limit();
838    #[cfg(windows)]
839    if let Err(error) = subc_jobobject::ensure_root_job() {
840        // Keep the control plane available. The root-job API caches setup
841        // failures, so later module spawns refuse instead of running unprotected.
842        warn!(%error, "module launches disabled: Windows root job setup failed");
843    }
844
845    info!(
846        connection_file = %bound.connection_file_path.display(),
847        connection_file_source = %bound.connection_file_source,
848        connection_file_source_reason = bound.connection_file_source.reason(),
849        endpoints = ?bound.connection_info.endpoints,
850        configured_modules = configured_modules.len(),
851        machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
852        "subc daemon starting"
853    );
854
855    // Before anything can spawn a module (the configured modules below, or a
856    // client's start request once the listeners are served): a previous daemon
857    // that died without its shutdown stop may have left children running, and
858    // a fresh copy beside one would fight it for its port and stores. The
859    // record is not a running daemon's because this daemon holds the lock on
860    // the run directory the record lives in, and a running daemon holds that
861    // lock for its whole life. (Claiming the singleton is not enough: it is
862    // keyed on the connection file, which can live in a different runtime
863    // directory from the run directory.) The lock stays held until this
864    // function returns, which is when the daemon exits.
865    let run_dir_lock = bound.run_dir_lock;
866    let publication_lock = bound.publication_lock;
867    if let Some(owner) = &run_dir_lock {
868        crate::live_children::sweep_orphans(
869            owner,
870            &crate::live_children::AdoptedPids::none(),
871            crate::live_children::SweepBounds::default(),
872        )
873        .await;
874    }
875
876    let registry = Arc::new(Registry::default());
877    let process_liveness = Arc::new(SupervisorProcessLiveness::new());
878    let supervisor_handle = SupervisorHandle::new();
879    let connected_clients = ConnectedClients::new();
880    let daemon_incarnation = format!(
881        "{:032x}",
882        u128::from_be_bytes(bound.connection_info.daemon_id)
883    );
884    let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
885        .with_process_liveness(process_liveness.clone())
886        .with_forwarding(Arc::clone(&forwarding))
887        .with_handle(supervisor_handle.clone())
888        .with_connection_file_path(bound.connection_file_path.clone())
889        .with_daemon_incarnation(daemon_incarnation.clone());
890    let supervisor = match privacy_trampoline {
891        Some(path) => supervisor.with_privacy_trampoline(path),
892        None => supervisor,
893    };
894    // ABSENT MEANS NO CAPTURE AND NO JOURNAL, NOT "THE REAL RUN DIRECTORY", and
895    // the difference is a production-corruption hazard rather than a preference.
896    // Both fields below follow the same rule: `None` for `capture_logs_dir`
897    // means supervised output is not captured, and `None` for
898    // `terminal_journal_path` means terminal history lives only in each
899    // module's in-memory ring (`supervisor.terminals` still answers, with the
900    // journal counters at zero).
901    //
902    // The terminal journal used to fall back to
903    // `daemon_run_dir().join("terminals.jsonl")`, so an in-process test daemon
904    // appended its fixture exits to the operator's journal, where they then
905    // appeared in `ck module terminals`.
906    //
907    // The capture line used to be `unwrap_or_else(|| daemon_run_dir().join("logs"))`,
908    // so ANY caller that did not set the field captured supervised children into
909    // the operator's live `~/.local/share/cortexkit/run/logs/`. That is twelve
910    // sibling repos whose integration tests boot an in-process daemon through
911    // `run_with_config` -- none of which asked for it, and none of which can see
912    // it from their side.
913    //
914    // Harmless while fixture module ids are fixture-shaped: this host carries 20
915    // zero-byte files from subc's own tests (good-aft, missing-aft,
916    // preview-consumer...). THE HAZARD IS A COLLISION. A fixture named "broca"
917    // or "aft" appends to a PRODUCTION capture file that operators read
918    // forensically and that placement gates count lines in -- with no residue to
919    // notice, because the file legitimately exists and legitimately grows.
920    //
921    // Found by BROCA (2026-09-19) from the other side: their rigs spawn the
922    // SHIPPED ck-subc and set XDG_CONFIG_HOME + XDG_RUNTIME_DIR but not
923    // XDG_DATA_HOME, so every local rig run supervised a module named "broca"
924    // and captured it into production's broca.stderr.log -- the same file I
925    // count seal lines in before and after placing their binaries.
926    //
927    // The supervisor already treats `None` as no-capture and no-journal, so
928    // this only removes invented defaults. The binary keeps capturing and
929    // journaling via `BootstrapConfig::from_env_for_daemon_binary`.
930
931    // Keep both ledgers in the explicitly selected run directory. An embedded
932    // daemon without a journal path must not write to the operator's live ledger.
933    let activity_run_dir = terminal_journal_path
934        .as_ref()
935        .and_then(|path| path.parent())
936        .map(Path::to_path_buf);
937    let supervisor = match terminal_journal_path {
938        Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
939        None => supervisor,
940    };
941    let supervisor = match capture_logs_dir {
942        Some(dir) => supervisor.with_capture_logs_dir(dir),
943        None => supervisor,
944    };
945    let supervisor = match live_children_path {
946        Some(path) => supervisor.with_live_children_record(path),
947        None => supervisor,
948    };
949    #[cfg(target_os = "linux")]
950    let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
951    // Collect per-module route.bind relay overrides BEFORE handing the
952    // `configured_modules` vector to the supervisor (which only needs each
953    // module's `drain_timeout_ms`). Each entry was filled in by parse-time
954    // resolution (per-module > daemon-wide > absent), so modules with no
955    // override are absent from this map and the daemon-wide default applies.
956    let route_bind_relay_timeouts = configured_modules
957        .iter()
958        .filter_map(|module| {
959            module
960                .route_bind_relay_timeout_ms
961                .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
962        })
963        .collect::<std::collections::BTreeMap<_, _>>();
964    let control_start_clock = crate::clock::StartClock::capture();
965    let mut control = ControlHandler::with_forwarding(Arc::clone(&registry), forwarding)
966        .with_process_liveness(process_liveness)
967        .with_supervisor(supervisor_handle)
968        // The same flag `begin_daemon_shutdown` sets below, so requirement
969        // changes while modules exit during the shutdown log at INFO.
970        .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
971        .with_connected_clients(connected_clients.clone())
972        .with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
973        .with_storage_config(storage_config)
974        .with_machine_id(bound.machine_id.clone())
975        .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
976        .with_scope_authority_owners(scope_authority_owners)
977        .with_privileged_operator_prompt(privileged_operator_prompt)
978        .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
979        .with_daemon_provenance(
980            bound.connection_info.pid,
981            control_start_clock.started_at_ms(),
982            std::env::current_exe().ok(),
983            normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
984            normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
985        )
986        .with_daemon_start_clock(control_start_clock)
987        .with_capability_config(
988            configured_modules
989                .iter()
990                .map(|module| (module.module_id.clone(), module.enabled)),
991            reserved_capabilities,
992        );
993    if let Some(run_dir) = activity_run_dir {
994        control =
995            control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
996    }
997    if let Some(ms) = route_bind_relay_default_ms {
998        // A daemon-wide config value overrides the built-in default; a
999        // `None` here leaves the ControlHandler's 12s default in place.
1000        control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
1001    }
1002    if let Some(config_path) = daemon_config_path {
1003        control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
1004    }
1005    let control = Arc::new(control);
1006    let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
1007    let auth = ServerAuth::new(
1008        bound.connection_info.key.clone(),
1009        bound.connection_info.daemon_id,
1010        bound.connection_info.daemon_ver.clone(),
1011    )
1012    .with_connected_clients(connected_clients);
1013
1014    // Discovery must expose every accepted configured module, including disabled
1015    // ones. Registration creates status handles but neither starts monitors
1016    // nor launches processes: children need the published file for `--subc`.
1017    let mut registered_modules = Vec::with_capacity(configured_modules.len());
1018    for configured in configured_modules {
1019        let enabled = configured.enabled;
1020        let health = configured.health.clone();
1021        let module_id = configured.module_id.clone();
1022        let registered = match supervisor.register_configured_with_health(
1023            configured.module_spec(),
1024            enabled,
1025            health.clone(),
1026            configured.drain_timeout_ms,
1027            configured.restart,
1028            configured.register_timeout,
1029        ) {
1030            Ok(registered) => registered,
1031            Err(err) => {
1032                error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
1033                continue;
1034            }
1035        };
1036        registered_modules.push(registered);
1037        // A relaxed threshold can silently outlive the temporary allowance
1038        // that required it, so report the extra tolerance on every boot.
1039        let default_threshold = HealthConfig::default().failure_threshold;
1040        if enabled && health.failure_threshold > default_threshold {
1041            warn!(
1042                module_id = %module_id,
1043                failure_threshold = health.failure_threshold,
1044                default_threshold,
1045                tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1046                "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1047            );
1048        }
1049        info!(module_id = %module_id, enabled, "configured module registered");
1050    }
1051    control.refresh_capability_requirements();
1052
1053    // The local endpoint is served beside TCP and answers the same handshake.
1054    // It serves clients only, which the server enforces per connection.
1055    let local_serve_task = bound.local_listener.map(|local_listener| {
1056        AbortOnDrop::new(tokio::spawn(serve_local_listener(
1057            local_listener,
1058            Arc::clone(&router),
1059            auth.clone(),
1060        )))
1061    });
1062    // On Unix the task is dropped below to stop accepting at shutdown. Windows
1063    // leaves it to the process exit; naming it keeps the binding uniform.
1064    #[cfg(windows)]
1065    let _local_serve_task = local_serve_task;
1066    let mut serve_task =
1067        AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1068    verify_serving(&bound.connection_info).await?;
1069    // A published endpoint promises that authentication can complete now.
1070    // Orphan cleanup and control-plane construction can take longer than an
1071    // existing-daemon probe's deadline, so finish both before discovery.
1072    // The write fsyncs, so it runs on tokio's blocking pool (see
1073    // `publish_connection_file`). The singleton lock travels with it and comes
1074    // back only once the file is in place.
1075    let (published, publication_lock) = publish_connection_file(
1076        &bound.connection_file_path,
1077        &bound.connection_info,
1078        bound.local_endpoint.as_ref(),
1079        publication_lock,
1080    )
1081    .await;
1082    published?;
1083    #[cfg(test)]
1084    startup_publication_hook::run(&bound.connection_file_path).await;
1085    drop(publication_lock);
1086    let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1087    // Off the startup path: it runs `systemctl`, and only ever logs.
1088    #[cfg(target_os = "linux")]
1089    let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1090        crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1091    ));
1092    let _watchdog_task = AbortOnDrop::new(
1093        DaemonSelfWatchdog::new(
1094            bound.connection_info.clone(),
1095            bound.connection_file_path.clone(),
1096        )
1097        .with_config(watchdog_config)
1098        .spawn(),
1099    );
1100
1101    for registered in registered_modules {
1102        match supervisor.start_registered_configured(registered, None, true) {
1103            Ok(module) => {
1104                info!(module_id = %module.module_id(), "configured module supervised");
1105            }
1106            Err(err) => {
1107                error!(error = %err, "failed to start configured module; continuing daemon startup");
1108            }
1109        }
1110    }
1111
1112    control.refresh_capability_requirements();
1113    Arc::clone(&control).spawn_capability_deadline_loop();
1114    Arc::clone(&control).spawn_scope_expiry_loop();
1115
1116    #[cfg(unix)]
1117    {
1118        tokio::select! {
1119            result = serve_task.join() => {
1120                return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1121            }
1122            _ = terminate.recv() => {}
1123        }
1124        // First, before the notice, the drain, or any connection close: from
1125        // here on a module exit is recorded as `daemon_shutdown` and never
1126        // respawned. Also before allowing a second signal to cut the bounded
1127        // wait short, so the journal marker is always written.
1128        supervisor.begin_daemon_shutdown();
1129        // Stop the self-watchdog before closing the listener. Its next tick would
1130        // connect to that listener, fail, and log an ERROR indistinguishable from
1131        // a wedged daemon, once for every interval a planned stop lasts.
1132        drop(_watchdog_task);
1133        // Dropping the listener stops new accepts, not established connections:
1134        // their detached tasks must remain live throughout notice and drain. The
1135        // local endpoint stops accepting the same way.
1136        drop(serve_task);
1137        drop(local_serve_task);
1138        let second_sigterm = async {
1139            terminate.recv().await;
1140            std::time::Instant::now()
1141        };
1142        tokio::pin!(second_sigterm);
1143        let mut escalation_started = tokio::select! {
1144            biased;
1145            at = second_sigterm.as_mut() => {
1146                info!("second SIGTERM: abandoning daemon shutdown wait");
1147                Some(at)
1148            }
1149            result = supervisor.drain_for_daemon_shutdown() => {
1150                if let Err(error) = result {
1151                    warn!(%error, "daemon shutdown drain failed; exiting anyway");
1152                }
1153                None
1154            }
1155        };
1156        // Supervised modules lead their own process groups, so a service
1157        // manager's kill of this process's group does not reach them. (A
1158        // systemd unit with KillMode=control-group kills by cgroup instead and
1159        // does reach them; see `systemd_kill_mode`.) The daemon ends them
1160        // itself: EOF (or SIGTERM for a protocol none child) first, then
1161        // signals at each child's own drain deadline.
1162        let already_escalated = escalation_started.is_some();
1163        supervisor
1164            .end_children_for_daemon_shutdown(already_escalated, async {
1165                if already_escalated {
1166                    std::future::pending::<()>().await;
1167                } else {
1168                    escalation_started = Some(second_sigterm.await);
1169                }
1170            })
1171            .await;
1172        if let Some(at) = escalation_started {
1173            info!(
1174                second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1175                "daemon shutdown wait ended after second SIGTERM"
1176            );
1177        }
1178        info!("daemon exiting: SIGTERM shutdown complete");
1179        Ok(())
1180    }
1181    #[cfg(not(unix))]
1182    serve_task
1183        .join()
1184        .await
1185        .map_err(BootstrapError::ServeJoin)?
1186        .map_err(BootstrapError::Serve)
1187}
1188
1189fn normalized_build_provenance(value: &str) -> Option<String> {
1190    match value.trim() {
1191        "" | "unavailable" => None,
1192        value => Some(value.to_string()),
1193    }
1194}
1195
1196/// Find an existing daemon or atomically bind loopback TCP for this daemon.
1197///
1198/// The algorithm is intentionally connect-first: an endpoint from the connection
1199/// file is treated as live only after the TCP+key server-proof authenticates for
1200/// that file's key and daemon_id. Stale or foreign connection files are reclaimed
1201/// only while holding the per-user start lock; the TCP port is never the
1202/// singleton primitive.
1203pub async fn ensure_singleton(
1204    connection_file_path: impl AsRef<Path>,
1205    port: u16,
1206) -> Result<Outcome, BootstrapError> {
1207    ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1208}
1209
1210pub async fn ensure_singleton_with_config(
1211    config: BootstrapConfig,
1212) -> Result<Outcome, BootstrapError> {
1213    ensure_singleton_inner(config, true).await
1214}
1215
1216async fn ensure_singleton_inner(
1217    config: BootstrapConfig,
1218    publish: bool,
1219) -> Result<Outcome, BootstrapError> {
1220    let path = config.connection_file_path;
1221
1222    if stable_peer(&path).await? {
1223        return Ok(Outcome::AlreadyRunning);
1224    }
1225
1226    let lock = StartLock::acquire(&path).await?;
1227
1228    // Re-probe after acquiring the start lock so a peer that won the race between
1229    // our first failed probe and the lock acquisition is observed instead of
1230    // overwritten.
1231    if stable_peer(&path).await? {
1232        return Ok(Outcome::AlreadyRunning);
1233    }
1234
1235    // The start lock and the probe above only rule out a daemon using this
1236    // connection file. A daemon started with another runtime directory but
1237    // the same data home shares the run directory while passing both, so the
1238    // run directory needs its own owner: refuse at once if a live daemon holds
1239    // it, before anything below reads or writes run or data-home state.
1240    let run_dir_lock = config
1241        .live_children_path
1242        .as_deref()
1243        .map(crate::run_dir_lock::RunDirLock::acquire)
1244        .transpose()?;
1245
1246    remove_stale_connection_file_if_present(&path)?;
1247
1248    // Established under the start lock, and under the run-directory lock when
1249    // there is one, and before binding: a corrupt file stops boot before
1250    // anything is published, and two daemons racing to start cannot both mint.
1251    //
1252    // Minting a new machine id fsyncs its file, so loading or minting runs on
1253    // tokio's blocking pool. Both locks travel with that work, so neither is
1254    // released while a mint is in flight.
1255    let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1256        None => (None, lock, run_dir_lock),
1257        Some(machine_id_path) => {
1258            let task_path = machine_id_path.clone();
1259            match run_blocking_holding((lock, run_dir_lock), move || {
1260                crate::machine_id::load_or_mint(&task_path)
1261            })
1262            .await
1263            {
1264                Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1265                Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1266                Err(join_error) => {
1267                    return Err(BootstrapError::MachineId(
1268                        crate::machine_id::MachineIdFileError::Write {
1269                            path: machine_id_path,
1270                            source: io::Error::other(describe_blocking_task_failure(
1271                                "loading or minting the machine id",
1272                                &join_error,
1273                            )),
1274                        },
1275                    ))
1276                }
1277            }
1278        }
1279    };
1280
1281    let (listeners, endpoints) = bind_loopback(config.port).await?;
1282    // The local endpoint is an addition to the daemon, not a condition for it:
1283    // a bind failure leaves clients on TCP only, with the connection file naming
1284    // no local endpoint, and the reason reported by `server.describe`. A daemon
1285    // that refused to start would take every module down for a feature nothing
1286    // has to use yet.
1287    let (local_listener, local_endpoint, local_endpoint_unavailable) =
1288        if config.local_listener.configured_dir().is_some() {
1289            match local_listener::bind(&config.local_listener, &path) {
1290                Ok(local_listener) => {
1291                    let endpoint = local_listener.endpoint();
1292                    (Some(local_listener), Some(endpoint), None)
1293                }
1294                Err(error) => {
1295                    let reason = error.to_string();
1296                    warn!(
1297                        error = %reason,
1298                        "the local endpoint is unavailable; serving loopback TCP only"
1299                    );
1300                    (None, None, Some(reason))
1301                }
1302            }
1303        } else {
1304            (None, None, None)
1305        };
1306    let connection_info = ConnectionInfo {
1307        schema: SCHEMA_VERSION,
1308        wire_version: Some(PROTOCOL_VERSION),
1309        endpoints,
1310        key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1311        daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1312        pid: process::id(),
1313        daemon_ver: config.daemon_ver,
1314    };
1315
1316    // The low-level binding API historically publishes for callers that serve
1317    // these listeners themselves. The daemon entry point defers publication
1318    // until its own authentication server is running.
1319    let publication_lock = if publish {
1320        let (published, lock) =
1321            publish_connection_file(&path, &connection_info, local_endpoint.as_ref(), lock).await;
1322        if let Err(error) = published {
1323            if let Some(local_listener) = local_listener.as_ref() {
1324                local_listener::release(local_listener);
1325            }
1326            drop(listeners);
1327            return Err(error);
1328        }
1329        drop(lock);
1330        None
1331    } else {
1332        Some(lock)
1333    };
1334
1335    Ok(Outcome::Bound(BoundDaemon {
1336        listeners,
1337        local_listener,
1338        local_endpoint,
1339        local_endpoint_unavailable,
1340        connection_info,
1341        connection_file_path: path,
1342        connection_file_source: config.connection_file_source,
1343        machine_id,
1344        run_dir_lock,
1345        publication_lock,
1346    }))
1347}
1348
1349#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1350enum Probe {
1351    Live,
1352    StaleOrAbsent,
1353}
1354
1355async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1356    if !matches!(probe_existing(path).await?, Probe::Live) {
1357        return Ok(false);
1358    }
1359    // A terminating peer can still authenticate while its listeners drain.
1360    // Require another proof after a short settling interval before reporting
1361    // a successful duplicate start. If the peer left, proceed with startup;
1362    // any remaining bind/ownership conflict is an error, not exit 0 (which
1363    // would suppress launchd's restart-on-failure policy).
1364    tokio::time::sleep(Duration::from_millis(1500)).await;
1365    Ok(matches!(probe_existing(path).await?, Probe::Live))
1366}
1367
1368async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1369    let info = match connection_file::read(path) {
1370        Ok(info) => info,
1371        Err(source) if is_absent_or_stale_connection_file(&source) => {
1372            return Ok(Probe::StaleOrAbsent)
1373        }
1374        Err(source) => {
1375            return Err(BootstrapError::ConnectionFileRead {
1376                path: path.to_path_buf(),
1377                source,
1378            })
1379        }
1380    };
1381
1382    for endpoint in &info.endpoints {
1383        if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1384            return Ok(Probe::Live);
1385        }
1386    }
1387
1388    Ok(Probe::StaleOrAbsent)
1389}
1390
1391async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1392    let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1393        return Probe::StaleOrAbsent;
1394    };
1395    if !ip.is_loopback() {
1396        return Probe::StaleOrAbsent;
1397    }
1398    let addr = SocketAddr::new(ip, endpoint.port);
1399
1400    let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1401        Ok(Ok(stream)) => stream,
1402        Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1403    };
1404
1405    match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1406        Ok(()) => Probe::Live,
1407        Err(AuthError::DaemonIdMismatch)
1408        | Err(AuthError::InvalidServerProof)
1409        | Err(AuthError::UnexpectedEof { .. })
1410        | Err(AuthError::Timeout { .. })
1411        | Err(AuthError::JsonEncode { .. })
1412        | Err(AuthError::JsonDecode { .. })
1413        | Err(AuthError::Io { .. })
1414        | Err(AuthError::MessageTooLarge { .. })
1415        | Err(AuthError::KeyTooShort { .. })
1416        | Err(AuthError::Random(_))
1417        | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1418    }
1419}
1420
1421async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1422    // Scheduling a server task is not a readiness barrier. Authenticate once
1423    // against the mandatory IPv4 listener before exposing discovery to peers.
1424    // This uses the normal probe deadline, with no retries or widened budget.
1425    if let Some(endpoint) = info.endpoints.first() {
1426        if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1427            return Ok(());
1428        }
1429    }
1430    Err(BootstrapError::StartupNotServing)
1431}
1432
1433/// Run blocking filesystem work (an fsync, a rename) on tokio's blocking-thread
1434/// pool rather than on the async worker thread that awaits it. On a slow or
1435/// busy disk an fsync can take seconds, and a runtime has few worker threads
1436/// (an in-process test daemon may share two with everything else), so doing it
1437/// inline stalls unrelated tasks for that long.
1438///
1439/// `guards` (lock handles) move into the blocking task and are handed back
1440/// with the work's result. If the awaiting future is dropped first, the work
1441/// still runs to completion on its thread and the guards are released only
1442/// after it ends, so a lock is never given up while the work it protects is
1443/// still writing.
1444///
1445/// `Err` means the task panicked, or was cancelled before it started because
1446/// the runtime is shutting down; the guards have then been released already.
1447async fn run_blocking_holding<G, T>(
1448    guards: G,
1449    work: impl FnOnce() -> T + Send + 'static,
1450) -> Result<(T, G), JoinError>
1451where
1452    G: Send + 'static,
1453    T: Send + 'static,
1454{
1455    tokio::task::spawn_blocking(move || {
1456        let output = work();
1457        (output, guards)
1458    })
1459    .await
1460}
1461
1462/// Explain, for a startup error, why blocking work produced no result.
1463fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1464    if join_error.is_panic() {
1465        format!("the background thread {work} panicked before finishing: {join_error}")
1466    } else {
1467        format!(
1468            "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1469        )
1470    }
1471}
1472
1473/// Publish the connection file: owner-only permissions, written to a temporary
1474/// file, fsynced and renamed over `path` (all done by `write_atomic`). The
1475/// write runs on tokio's blocking pool through [`run_blocking_holding`], so the
1476/// fsync never holds an async worker thread.
1477///
1478/// `guard` is the singleton start lock (or whatever the caller must keep held
1479/// across publication); it is returned with the result. It comes back `None`
1480/// only when the write task panicked or was cancelled, in which case the
1481/// result is an error and the file must be treated as not published.
1482async fn publish_connection_file<G: Send + 'static>(
1483    path: &Path,
1484    info: &ConnectionInfo,
1485    local_endpoint: Option<&LocalEndpoint>,
1486    guard: G,
1487) -> (Result<(), BootstrapError>, Option<G>) {
1488    let task_path = path.to_path_buf();
1489    let task_info = info.clone();
1490    let task_endpoint = local_endpoint.cloned();
1491    let outcome = run_blocking_holding(guard, move || {
1492        write_connection_file(&task_path, &task_info, task_endpoint.as_ref())
1493    })
1494    .await;
1495    match outcome {
1496        Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1497        Ok((Err(source), guard)) => (
1498            Err(BootstrapError::ConnectionFileWrite {
1499                path: path.to_path_buf(),
1500                source,
1501            }),
1502            Some(guard),
1503        ),
1504        Err(join_error) => (
1505            Err(BootstrapError::ConnectionFileWrite {
1506                path: path.to_path_buf(),
1507                source: ConnectionFileError::Io {
1508                    op: "write",
1509                    path: path.to_path_buf(),
1510                    source: io::Error::other(describe_blocking_task_failure(
1511                        "writing the connection file",
1512                        &join_error,
1513                    )),
1514                },
1515            }),
1516            None,
1517        ),
1518    }
1519}
1520
1521/// The synchronous connection-file write, including its fsync. Call it only
1522/// through [`publish_connection_file`], which keeps it off the async workers.
1523///
1524/// `local_endpoint` is written through subc-transport's endpoint-aware writer,
1525/// so the file carries `local_endpoint` when there is a local listener and is
1526/// byte-identical to before when there is none.
1527fn write_connection_file(
1528    path: &Path,
1529    info: &ConnectionInfo,
1530    local_endpoint: Option<&LocalEndpoint>,
1531) -> Result<(), ConnectionFileError> {
1532    #[cfg(test)]
1533    connection_file_write_hook::run(path);
1534    write_atomic_with_local_endpoint(path, info, local_endpoint)
1535}
1536
1537/// Lets a test run code inside the connection-file write, on whichever thread
1538/// performs it, for example to block there and check what else can still run.
1539#[cfg(test)]
1540mod connection_file_write_hook {
1541    use std::{
1542        path::{Path, PathBuf},
1543        sync::{Mutex, PoisonError},
1544    };
1545
1546    type Hook = Box<dyn FnOnce() + Send>;
1547
1548    /// Keyed by connection-file path, so a hook fires only for the test that
1549    /// installed it while other tests publish their own files in parallel.
1550    static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
1551
1552    /// Run `hook` once, at the start of the next write of `path`.
1553    pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
1554        HOOKS
1555            .lock()
1556            .unwrap_or_else(PoisonError::into_inner)
1557            .push((path.to_path_buf(), Box::new(hook)));
1558    }
1559
1560    pub(super) fn run(path: &Path) {
1561        let hook = {
1562            let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
1563            hooks
1564                .iter()
1565                .position(|(hooked, _)| hooked == path)
1566                .map(|index| hooks.swap_remove(index).1)
1567        };
1568        if let Some(hook) = hook {
1569            hook();
1570        }
1571    }
1572}
1573
1574/// A path-scoped barrier after publication and before any configured activation.
1575/// The async wait leaves the listeners free to answer discovery-time requests.
1576#[cfg(test)]
1577mod startup_publication_hook {
1578    use std::{
1579        path::{Path, PathBuf},
1580        sync::{Mutex, PoisonError},
1581    };
1582    use tokio::sync::oneshot;
1583
1584    type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
1585    static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
1586
1587    pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
1588        let (entered_tx, entered_rx) = oneshot::channel();
1589        let (release_tx, release_rx) = oneshot::channel();
1590        BARRIERS
1591            .lock()
1592            .unwrap_or_else(PoisonError::into_inner)
1593            .push((path.to_path_buf(), (entered_tx, release_rx)));
1594        (entered_rx, release_tx)
1595    }
1596
1597    pub(super) async fn run(path: &Path) {
1598        let barrier = {
1599            let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
1600            barriers
1601                .iter()
1602                .position(|(hooked, _)| hooked == path)
1603                .map(|index| barriers.swap_remove(index).1)
1604        };
1605        if let Some((entered, release)) = barrier {
1606            let _ = entered.send(());
1607            let _ = release.await;
1608        }
1609    }
1610}
1611
1612fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
1613    match err {
1614        ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
1615        ConnectionFileError::JsonRead { .. }
1616        | ConnectionFileError::UnsupportedSchema { .. }
1617        | ConnectionFileError::Invalid { .. }
1618        | ConnectionFileError::KeyTooShort { .. }
1619        // A live daemon always publishes the file owner-only (0600), so a file
1620        // with insecure permissions is never a daemon we should defer to: treat it
1621        // as stale and take over (which republishes a correct 0600 file).
1622        | ConnectionFileError::InsecurePermissions { .. } => true,
1623        ConnectionFileError::MissingParent { .. }
1624        | ConnectionFileError::MissingFileName { .. }
1625        // A writable ancestor is an operator misconfiguration, never evidence
1626        // about whether a daemon is live. Reclaiming the file would republish key
1627        // material into the same directory the refusal is about.
1628        | ConnectionFileError::InsecureParentDirectory { .. }
1629        | ConnectionFileError::Io { .. }
1630        | ConnectionFileError::JsonWrite { .. }
1631        | ConnectionFileError::Random(_)
1632        // A wire mismatch may identify a newer live daemon, so never reclaim its
1633        // connection file merely because this binary cannot speak its envelope.
1634        | ConnectionFileError::WireVersionMismatch { .. } => false,
1635    }
1636}
1637
1638fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
1639    match fs::remove_file(path) {
1640        Ok(()) => Ok(()),
1641        Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
1642        Err(source) => Err(BootstrapError::RemoveStale {
1643            path: path.to_path_buf(),
1644            source,
1645        }),
1646    }
1647}
1648
1649async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
1650    bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
1651}
1652
1653async fn bind_loopback_with<F, Fut>(
1654    port: u16,
1655    mut bind_v6: F,
1656) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
1657where
1658    F: FnMut(u16) -> Fut,
1659    Fut: std::future::Future<Output = io::Result<TcpListener>>,
1660{
1661    let v4_host = Ipv4Addr::LOCALHOST;
1662    let v4 = TcpListener::bind((v4_host, port))
1663        .await
1664        .map_err(|source| BootstrapError::Bind {
1665            host: v4_host.to_string(),
1666            port,
1667            source,
1668        })?;
1669    let actual_port = v4
1670        .local_addr()
1671        .map_err(|source| BootstrapError::LocalAddr {
1672            host: v4_host.to_string(),
1673            source,
1674        })?
1675        .port();
1676
1677    let mut listeners = vec![v4];
1678    let mut endpoints = vec![Endpoint {
1679        host: v4_host.to_string(),
1680        port: actual_port,
1681    }];
1682
1683    let v6_host = Ipv6Addr::LOCALHOST;
1684    match bind_v6(actual_port).await {
1685        Ok(v6) => {
1686            listeners.push(v6);
1687            endpoints.push(Endpoint {
1688                host: v6_host.to_string(),
1689                port: actual_port,
1690            });
1691        }
1692        Err(err)
1693            if ipv6_loopback_unavailable(&err)
1694                || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
1695        {
1696            warn!(
1697                port = actual_port,
1698                error = %err,
1699                "IPv6 loopback unavailable; serving only IPv4 loopback"
1700            );
1701        }
1702        Err(source) => {
1703            drop(listeners);
1704            return Err(BootstrapError::Bind {
1705                host: v6_host.to_string(),
1706                port: actual_port,
1707                source,
1708            });
1709        }
1710    }
1711
1712    Ok((listeners, endpoints))
1713}
1714
1715fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
1716    matches!(
1717        err.kind(),
1718        io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
1719    ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
1720}
1721
1722struct AbortOnDrop<T> {
1723    handle: JoinHandle<T>,
1724}
1725
1726impl<T> AbortOnDrop<T> {
1727    fn new(handle: JoinHandle<T>) -> Self {
1728        Self { handle }
1729    }
1730
1731    async fn join(&mut self) -> Result<T, JoinError> {
1732        (&mut self.handle).await
1733    }
1734}
1735
1736impl<T> Drop for AbortOnDrop<T> {
1737    fn drop(&mut self) {
1738        if !self.handle.is_finished() {
1739            self.handle.abort();
1740        }
1741    }
1742}
1743
1744#[derive(Debug)]
1745struct StartLock {
1746    // Keep the locked file handle alive for the duration of bootstrap.
1747    _file: fs::File,
1748}
1749
1750impl Drop for StartLock {
1751    fn drop(&mut self) {
1752        // Closing alone leaves the lock held if a concurrent fork inherited
1753        // the open file description and has not exec'd yet. Unlock explicitly
1754        // so a restart can reclaim the singleton as soon as bootstrap ends.
1755        if let Err(error) = FileExt::unlock(&self._file) {
1756            tracing::warn!(%error, "failed to release the singleton start lock");
1757        }
1758    }
1759}
1760
1761impl StartLock {
1762    async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
1763        let path = start_lock_path(connection_file_path);
1764        for _ in 0..START_LOCK_RETRIES {
1765            let file = match open_owner_only_lock(&path) {
1766                Ok(file) => file,
1767                Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
1768            };
1769            match FileExt::try_lock(&file) {
1770                Ok(()) => return Ok(Self { _file: file }),
1771                Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
1772                Err(TryLockError::Error(source)) => {
1773                    return Err(BootstrapError::StartLockCreate { path, source });
1774                }
1775            }
1776        }
1777
1778        Err(BootstrapError::StartLockBusy {
1779            path,
1780            attempts: START_LOCK_RETRIES,
1781        })
1782    }
1783}
1784
1785pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
1786    let mut options = fs::OpenOptions::new();
1787    options.read(true).write(true).create(true);
1788    #[cfg(unix)]
1789    {
1790        use std::os::unix::fs::OpenOptionsExt;
1791        options.mode(0o600);
1792    }
1793    options.open(path)
1794}
1795
1796fn start_lock_path(connection_file_path: &Path) -> PathBuf {
1797    let file_name = connection_file_path
1798        .file_name()
1799        .map(|name| name.to_string_lossy())
1800        .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
1801    let lock_name = format!("{file_name}.start-lock");
1802    connection_file_path
1803        .parent()
1804        .filter(|parent| !parent.as_os_str().is_empty())
1805        .unwrap_or_else(|| Path::new("."))
1806        .join(lock_name)
1807}
1808
1809fn non_empty_os_var(key: &str) -> Option<OsString> {
1810    let value = env::var_os(key)?;
1811    if value.is_empty() {
1812        None
1813    } else {
1814        Some(value)
1815    }
1816}
1817
1818/// Bootstrap-layer errors are deliberately typed so startup never panics for
1819/// ordinary daemon-discovery races or stale filesystem state.
1820#[derive(Debug)]
1821pub enum BootstrapError {
1822    /// The bound server could not authenticate its own readiness probe.
1823    StartupNotServing,
1824    #[cfg(unix)]
1825    Signal(io::Error),
1826    InvalidPort {
1827        raw: String,
1828        source: std::num::ParseIntError,
1829    },
1830    ConnectionFileRead {
1831        path: PathBuf,
1832        source: ConnectionFileError,
1833    },
1834    ConnectionFileWrite {
1835        path: PathBuf,
1836        source: ConnectionFileError,
1837    },
1838    GenerateConnectionFile(ConnectionFileError),
1839    StartLockCreate {
1840        path: PathBuf,
1841        source: io::Error,
1842    },
1843    StartLockBusy {
1844        path: PathBuf,
1845        attempts: usize,
1846    },
1847    /// The run-directory lock file could not be created, opened or locked.
1848    RunDirLockCreate {
1849        path: PathBuf,
1850        source: io::Error,
1851    },
1852    /// Another live process, almost certainly a daemon started with a
1853    /// different runtime directory over the same data home, owns the run
1854    /// directory. The daemon does not start. `holder_pid` is read from the
1855    /// lock file when possible and is informational only.
1856    RunDirBusy {
1857        path: PathBuf,
1858        holder_pid: Option<u32>,
1859    },
1860    RemoveStale {
1861        path: PathBuf,
1862        source: io::Error,
1863    },
1864    Bind {
1865        host: String,
1866        port: u16,
1867        source: io::Error,
1868    },
1869    LocalAddr {
1870        host: String,
1871        source: io::Error,
1872    },
1873    DaemonConfig(DaemonConfigError),
1874    /// The machine id could not be established: its file is corrupt, unreadable
1875    /// or unwritable, or the data home is relative. The daemon does not start.
1876    MachineId(crate::machine_id::MachineIdFileError),
1877    /// The daemon run directory could not be resolved because the data home is
1878    /// relative. The daemon does not start.
1879    RunDir(daemon_config::DaemonRunDirError),
1880    Serve(ServerError),
1881    ServeJoin(tokio::task::JoinError),
1882}
1883
1884impl From<ServerError> for BootstrapError {
1885    fn from(error: ServerError) -> Self {
1886        Self::Serve(error)
1887    }
1888}
1889
1890impl fmt::Display for BootstrapError {
1891    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1892        match self {
1893            Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
1894            #[cfg(unix)]
1895            Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
1896            Self::InvalidPort { raw, source } => {
1897                write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
1898            }
1899            Self::ConnectionFileRead { path, source } => write!(
1900                f,
1901                "failed to read connection file {}: {source}",
1902                path.display()
1903            ),
1904            Self::ConnectionFileWrite { path, source } => write!(
1905                f,
1906                "failed to publish connection file {}: {source}",
1907                path.display()
1908            ),
1909            Self::GenerateConnectionFile(err) => {
1910                write!(f, "failed to generate connection-file auth material: {err}")
1911            }
1912            Self::StartLockCreate { path, source } => {
1913                write!(
1914                    f,
1915                    "failed to create start lock {}: {source}",
1916                    path.display()
1917                )
1918            }
1919            Self::StartLockBusy { path, attempts } => write!(
1920                f,
1921                "start lock {} remained busy after {attempts} attempts",
1922                path.display()
1923            ),
1924            Self::RunDirLockCreate { path, source } => write!(
1925                f,
1926                "refusing to start: failed to lock run directory via {}: {source}",
1927                path.display()
1928            ),
1929            Self::RunDirBusy { path, holder_pid } => {
1930                write!(
1931                    f,
1932                    "refusing to start: run directory lock {} is held by another daemon",
1933                    path.display()
1934                )?;
1935                if let Some(pid) = holder_pid {
1936                    write!(f, " (pid {pid})")?;
1937                }
1938                write!(
1939                    f,
1940                    "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
1941                )
1942            }
1943            Self::RemoveStale { path, source } => write!(
1944                f,
1945                "failed to remove stale connection file {}: {source}",
1946                path.display()
1947            ),
1948            Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
1949                write!(
1950                    f,
1951                    "port {port} in use on loopback {host}: {source}; set the port in config"
1952                )
1953            }
1954            Self::Bind { host, port, source } => {
1955                write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
1956            }
1957            Self::LocalAddr { host, source } => {
1958                write!(f, "failed to read local address for {host}: {source}")
1959            }
1960            Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
1961            Self::MachineId(err) => write!(f, "refusing to start: {err}"),
1962            Self::RunDir(err) => write!(f, "refusing to start: {err}"),
1963            Self::Serve(err) => write!(f, "daemon server failed: {err}"),
1964            Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
1965        }
1966    }
1967}
1968
1969impl Error for BootstrapError {
1970    fn source(&self) -> Option<&(dyn Error + 'static)> {
1971        match self {
1972            Self::StartupNotServing => None,
1973            #[cfg(unix)]
1974            Self::Signal(source) => Some(source),
1975            Self::InvalidPort { source, .. } => Some(source),
1976            Self::ConnectionFileRead { source, .. }
1977            | Self::ConnectionFileWrite { source, .. }
1978            | Self::GenerateConnectionFile(source) => Some(source),
1979            Self::StartLockCreate { source, .. }
1980            | Self::RunDirLockCreate { source, .. }
1981            | Self::RemoveStale { source, .. }
1982            | Self::Bind { source, .. }
1983            | Self::LocalAddr { source, .. } => Some(source),
1984            Self::DaemonConfig(err) => Some(err),
1985            Self::MachineId(err) => Some(err),
1986            Self::RunDir(err) => Some(err),
1987            Self::Serve(err) => Some(err),
1988            Self::ServeJoin(err) => Some(err),
1989            Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
1990        }
1991    }
1992}
1993
1994#[cfg(test)]
1995mod tests {
1996    mod activity {
1997        include!("activity_bootstrap_tests.rs");
1998    }
1999    mod startup_order {
2000        include!("startup_order_tests.rs");
2001    }
2002    use super::*;
2003    use crate::server::ServerAuth;
2004    use cortexkit_test_support::ScratchDir;
2005    #[cfg(target_os = "linux")]
2006    use std::collections::BTreeSet;
2007    use std::sync::Mutex;
2008    #[cfg(target_os = "linux")]
2009    use subc_control::ModuleProtocol;
2010    use subc_transport::{write_atomic, MIN_KEY_LEN};
2011    use tokio::io::AsyncReadExt;
2012    use tokio::task::JoinHandle;
2013
2014    #[cfg(unix)]
2015    use std::os::unix::fs::PermissionsExt;
2016
2017    static ENV_LOCK: Mutex<()> = Mutex::new(());
2018
2019    #[test]
2020    fn normalized_build_provenance_preserves_real_values() {
2021        assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
2022    }
2023
2024    #[test]
2025    fn normalized_build_provenance_omits_unavailable_and_empty_values() {
2026        assert_eq!(normalized_build_provenance("unavailable"), None);
2027        assert_eq!(normalized_build_provenance(""), None);
2028    }
2029
2030    #[cfg(target_os = "linux")]
2031    fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
2032        let cgroups = fs::read_to_string("/proc/self/cgroup")?;
2033        let relative = cgroups
2034            .lines()
2035            .find_map(|line| line.strip_prefix("0::"))
2036            .ok_or_else(|| {
2037                io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
2038            })?;
2039        Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
2040    }
2041
2042    #[cfg(target_os = "linux")]
2043    fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
2044        let modules = current_cgroup_path_for_test()?.join("subc-modules");
2045        let entries = match fs::read_dir(modules) {
2046            Ok(entries) => entries,
2047            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
2048            Err(error) => return Err(error),
2049        };
2050        let mut directories = BTreeSet::new();
2051        for entry in entries {
2052            let entry = entry?;
2053            if entry.file_type()?.is_dir() {
2054                directories.insert(entry.file_name());
2055            }
2056        }
2057        Ok(directories)
2058    }
2059
2060    #[cfg(target_os = "linux")]
2061    async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
2062        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2063        while !path.exists() && tokio::time::Instant::now() < deadline {
2064            assert!(
2065                !task.is_finished(),
2066                "daemon exited before creating {}",
2067                path.display()
2068            );
2069            sleep(Duration::from_millis(10)).await;
2070        }
2071        assert!(path.exists(), "daemon did not create {}", path.display());
2072    }
2073
2074    /// An in-process daemon with the default (disabled) cgroup placement must
2075    /// leave the host's live module cgroup tree exactly as it found it.
2076    /// Red-checking this test (making it fail) creates a
2077    /// `cgroup-isolation-probe-*` directory in the live cgroup tree, which must be removed by hand.
2078    #[cfg(target_os = "linux")]
2079    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2080    async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
2081        let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
2082        let module_id = format!("cgroup-isolation-probe-{}", process::id());
2083        let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
2084        assert!(
2085            !before.contains(&OsString::from(&module_id)),
2086            "isolation probe cgroup already exists before this daemon starts"
2087        );
2088        let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
2089        let module = ConfiguredModule {
2090            module_id,
2091            program: PathBuf::from("sh"),
2092            args: vec!["-c".to_string(), "sleep 30".to_string()],
2093            env: Vec::new(),
2094            log: None,
2095            enabled: true,
2096            reserved: false,
2097            reserved_prefixes: Vec::new(),
2098            protocol: ModuleProtocol::None,
2099            overlap: Default::default(),
2100            health: HealthConfig::default(),
2101            drain_timeout_ms: None,
2102            register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2103            route_bind_relay_timeout_ms: None,
2104            restart: RestartPolicy::default(),
2105        };
2106        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2107            .with_configured_modules([module])
2108            .with_capture_logs_dir(temp.join("logs"))
2109            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2110        let task = tokio::spawn(run_with_config(config));
2111
2112        wait_for_path(&capture, &task).await;
2113        let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2114
2115        task.abort();
2116        assert!(task
2117            .await
2118            .expect_err("aborted daemon task must cancel")
2119            .is_cancelled());
2120        assert_eq!(
2121            before, after,
2122            "an in-process daemon must not create or reconcile ambient module cgroups"
2123        );
2124    }
2125
2126    #[cfg(target_os = "linux")]
2127    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2128    async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2129        let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2130        let cgroup_root = temp.join("cgroup");
2131        fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2132        fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2133        let modules = cgroup_root.join("subc-modules");
2134        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2135            .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2136            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2137        let task = tokio::spawn(run_with_config(config));
2138
2139        wait_for_path(&modules, &task).await;
2140
2141        task.abort();
2142        assert!(task
2143            .await
2144            .expect_err("aborted daemon task must cancel")
2145            .is_cancelled());
2146        assert!(
2147            fs::read_dir(&modules)
2148                .expect("read prepared modules directory")
2149                .next()
2150                .is_none(),
2151            "the delegation probe must clean up after itself"
2152        );
2153    }
2154
2155    struct EnvGuard {
2156        key: &'static str,
2157        previous: Option<OsString>,
2158    }
2159
2160    impl EnvGuard {
2161        fn set(key: &'static str, value: &Path) -> Self {
2162            let previous = env::var_os(key);
2163            env::set_var(key, value);
2164            Self { key, previous }
2165        }
2166
2167        fn set_str(key: &'static str, value: &str) -> Self {
2168            let previous = env::var_os(key);
2169            env::set_var(key, value);
2170            Self { key, previous }
2171        }
2172
2173        fn unset(key: &'static str) -> Self {
2174            let previous = env::var_os(key);
2175            env::remove_var(key);
2176            Self { key, previous }
2177        }
2178    }
2179
2180    impl Drop for EnvGuard {
2181        fn drop(&mut self) {
2182            match &self.previous {
2183                Some(value) => env::set_var(self.key, value),
2184                None => env::remove_var(self.key),
2185            }
2186        }
2187    }
2188
2189    fn unique_temp_dir(name: &str) -> ScratchDir {
2190        ScratchDir::new(name)
2191    }
2192
2193    fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2194        let dir = unique_temp_dir(name);
2195        let path = dir.join("conn.json");
2196        (dir, path)
2197    }
2198
2199    fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2200        ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2201    }
2202
2203    fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2204        let auth = auth_for(&bound.connection_info);
2205        tokio::spawn(serve_listeners(
2206            bound.listeners,
2207            Arc::new(Router::with_default_self_handler()),
2208            auth,
2209        ))
2210    }
2211
2212    fn expect_bound(outcome: Outcome) -> BoundDaemon {
2213        match outcome {
2214            Outcome::Bound(bound) => bound,
2215            Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2216        }
2217    }
2218
2219    async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2220        let endpoint = conn
2221            .endpoints
2222            .first()
2223            .expect("test connection file should have an endpoint");
2224        let ip: IpAddr = endpoint.host.parse().unwrap();
2225        TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2226    }
2227
2228    fn make_connection_info(port: u16) -> ConnectionInfo {
2229        ConnectionInfo {
2230            schema: SCHEMA_VERSION,
2231            wire_version: Some(PROTOCOL_VERSION),
2232            endpoints: vec![Endpoint {
2233                host: "127.0.0.1".to_owned(),
2234                port,
2235            }],
2236            key: generate_key().unwrap(),
2237            daemon_id: generate_daemon_id().unwrap(),
2238            pid: process::id(),
2239            daemon_ver: "test-subc".to_owned(),
2240        }
2241    }
2242
2243    fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2244        fs::write(path, contents).unwrap();
2245        #[cfg(unix)]
2246        fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2247    }
2248
2249    fn assert_owner_only_connection_file(path: &Path) {
2250        // `path` is only inspected on Unix (mode bits); on Windows the owner-only
2251        // guarantee comes from the inherited %TEMP% ACL, nothing to assert here.
2252        #[cfg(unix)]
2253        {
2254            let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2255            assert_eq!(mode, 0o600);
2256        }
2257        #[cfg(not(unix))]
2258        let _ = path;
2259    }
2260
2261    #[test]
2262    fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2263        let _env_lock = ENV_LOCK.lock().unwrap();
2264        let runtime_dir = unique_temp_dir("xdg-runtime");
2265        let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2266
2267        assert_eq!(
2268            connection_file_path(),
2269            runtime_dir.join(CONNECTION_FILE_NAME)
2270        );
2271    }
2272
2273    #[test]
2274    fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2275        let runtime_dir = OsString::from("/run/user/1000");
2276
2277        let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2278
2279        assert_eq!(
2280            path,
2281            PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2282        );
2283        assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2284    }
2285
2286    #[test]
2287    fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2288        let _env_lock = ENV_LOCK.lock().unwrap();
2289        let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2290
2291        assert_eq!(
2292            connection_file_path(),
2293            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2294        );
2295    }
2296
2297    #[test]
2298    fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2299        let (path, source) = connection_file_path_with_source(None);
2300
2301        assert_eq!(
2302            path,
2303            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2304        );
2305        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2306    }
2307
2308    /// Concurrent callers must derive one token. The former temp-file uid probe
2309    /// could fail transiently (same-tick name collision, fd exhaustion) and send
2310    /// the loser down the env-derived fallback with a different identity for the
2311    /// same user; this fence keeps identity independent of filesystem luck.
2312    #[test]
2313    fn user_connection_token_is_stable_under_concurrent_callers() {
2314        let expected = user_connection_token();
2315        let workers: Vec<_> = (0..32)
2316            .map(|_| {
2317                std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2318            })
2319            .collect();
2320        for worker in workers {
2321            for token in worker.join().expect("probe thread") {
2322                assert_eq!(token, expected, "token diverged under concurrent probes");
2323            }
2324        }
2325    }
2326
2327    #[test]
2328    fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
2329        let (path, source) = connection_file_path_with_source(Some(OsString::new()));
2330
2331        assert_eq!(
2332            path,
2333            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2334        );
2335        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2336    }
2337
2338    /// The shipped binary is the only caller that journals terminal exits into
2339    /// the real run directory, so its constructor must supply that path itself;
2340    /// the in-process default leaves it unset.
2341    #[test]
2342    fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
2343        let _env_lock = ENV_LOCK.lock().unwrap();
2344        let root = unique_temp_dir("daemon-binary-config");
2345        let data_home = root.join("data");
2346        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
2347        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2348        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2349
2350        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
2351
2352        let run_dir = data_home.join("cortexkit").join("run");
2353        assert_eq!(
2354            config.terminal_journal_path,
2355            Some(run_dir.join("terminals.jsonl"))
2356        );
2357        assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
2358        assert_eq!(
2359            BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
2360            None,
2361            "an in-process config must not journal anywhere unless asked to"
2362        );
2363    }
2364
2365    #[test]
2366    fn daemon_binary_config_refuses_a_relative_data_home() {
2367        let _env_lock = ENV_LOCK.lock().unwrap();
2368        let root = unique_temp_dir("daemon-binary-relative-data");
2369        let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
2370        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2371        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2372
2373        let error = BootstrapConfig::from_env_for_daemon_binary()
2374            .expect_err("a relative data home must refuse the daemon binary's config");
2375        assert!(
2376            matches!(error, BootstrapError::RunDir(_)),
2377            "expected a run-directory refusal, got {error}"
2378        );
2379        assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
2380    }
2381
2382    #[test]
2383    fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
2384        let _env_lock = ENV_LOCK.lock().unwrap();
2385        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2386        let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
2387        let config_path = conn_path.with_file_name("subc.jsonc");
2388        for (doc, expected) in [
2389            (None, daemon_config::PrivilegedOperatorPrompt::Log),
2390            (
2391                Some(r#"{"version":1}"#),
2392                daemon_config::PrivilegedOperatorPrompt::Log,
2393            ),
2394            (
2395                Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
2396                daemon_config::PrivilegedOperatorPrompt::Log,
2397            ),
2398            (
2399                Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
2400                daemon_config::PrivilegedOperatorPrompt::Enforce,
2401            ),
2402        ] {
2403            if let Some(doc) = doc {
2404                fs::write(&config_path, doc).unwrap();
2405            }
2406            assert_eq!(
2407                BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2408                    .unwrap()
2409                    .privileged_operator_prompt,
2410                expected
2411            );
2412            assert_eq!(
2413                BootstrapConfig::new(&conn_path, 0)
2414                    .with_daemon_config_path(&config_path)
2415                    .unwrap()
2416                    .privileged_operator_prompt,
2417                expected
2418            );
2419        }
2420    }
2421
2422    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2423    async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
2424        let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
2425        let connection_path = dir.join("connection.json");
2426        let config_path = dir.join("subc.jsonc");
2427        fs::write(
2428            &config_path,
2429            r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
2430        )
2431        .unwrap();
2432        let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
2433            .with_daemon_config_path(&config_path)
2434            .unwrap();
2435        let connection_path = config.connection_file_path.clone();
2436        let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
2437        tokio::time::timeout(Duration::from_secs(5), async {
2438            while !connection_path.exists() {
2439                assert!(
2440                    !daemon.handle.is_finished(),
2441                    "daemon exited before publishing"
2442                );
2443                tokio::task::yield_now().await;
2444            }
2445        })
2446        .await
2447        .unwrap();
2448        let info = connection_file::read(&connection_path).unwrap();
2449        let mut client = connect_from_info(&info).await.unwrap();
2450        authenticate_client(&mut client, &info, Duration::from_secs(5))
2451            .await
2452            .unwrap();
2453        for (correlation, mode, expected) in [
2454            (1, "enforce", Vec::new()),
2455            (2, "log", vec!["privileged_operator_prompt"]),
2456            (3, "log", vec!["privileged_operator_prompt"]),
2457        ] {
2458            fs::write(
2459                &config_path,
2460                serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
2461            )
2462            .unwrap();
2463            let frame = crate::Frame::build(
2464                subc_protocol::FrameType::Request,
2465                subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
2466                0,
2467                0,
2468                correlation,
2469                serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
2470                    preview: true,
2471                })
2472                .unwrap(),
2473            )
2474            .unwrap();
2475            crate::write_frame(&mut client, &frame).await.unwrap();
2476            let reply =
2477                tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
2478                    .await
2479                    .unwrap()
2480                    .unwrap()
2481                    .unwrap();
2482            assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
2483            assert_eq!(reply.header.corr, correlation);
2484            let subc_control::ClientControlResponse::SupervisorRescan { result } =
2485                serde_json::from_slice(&reply.body).unwrap()
2486            else {
2487                panic!("expected a rescan response");
2488            };
2489            assert_eq!(result.restart_required, expected);
2490        }
2491    }
2492
2493    #[test]
2494    fn configured_port_uses_default_config_and_env_override() {
2495        let _env_lock = ENV_LOCK.lock().unwrap();
2496        let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
2497        let config_path = conn_path.with_file_name("subc.jsonc");
2498
2499        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2500        assert_eq!(
2501            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2502                .unwrap()
2503                .port,
2504            DEFAULT_SUBC_PORT
2505        );
2506
2507        fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
2508        assert_eq!(
2509            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2510                .unwrap()
2511                .port,
2512            8123
2513        );
2514
2515        let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
2516        assert_eq!(
2517            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2518                .unwrap()
2519                .port,
2520            9012
2521        );
2522    }
2523
2524    #[tokio::test]
2525    async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
2526        let (_dir, path) = temp_connection_file_path("departing-peer");
2527        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2528        let info = make_connection_info(listener.local_addr().unwrap().port());
2529        write_connection_file(&path, &info, None).unwrap();
2530        let peer = tokio::spawn(async move {
2531            let (mut stream, _) = listener.accept().await.unwrap();
2532            subc_transport::authenticate_server(
2533                &mut stream,
2534                &info.key,
2535                &info.daemon_id,
2536                &info.daemon_ver,
2537                PROBE_AUTH_DEADLINE,
2538            )
2539            .await
2540            .unwrap();
2541        });
2542        let outcome = ensure_singleton(&path, 0).await.unwrap();
2543        peer.await.unwrap();
2544        assert!(
2545            matches!(outcome, Outcome::Bound(_)),
2546            "a departing peer is not AlreadyRunning"
2547        );
2548    }
2549
2550    #[tokio::test]
2551    async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
2552        let (_dir, path) = temp_connection_file_path("already-running");
2553
2554        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2555        let server = start_server(bound);
2556
2557        let second = ensure_singleton(&path, 0).await.unwrap();
2558        assert!(matches!(second, Outcome::AlreadyRunning));
2559
2560        server.abort();
2561        let _ = server.await;
2562    }
2563
2564    #[tokio::test]
2565    async fn daemon_connection_file_publishes_protocol_wire_version() {
2566        let (_dir, path) = temp_connection_file_path("wire-version");
2567        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2568        assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
2569        assert_eq!(
2570            connection_file::read(&path).unwrap().wire_version,
2571            Some(PROTOCOL_VERSION)
2572        );
2573
2574        drop(bound.listeners);
2575    }
2576
2577    #[tokio::test]
2578    async fn stale_unbound_connection_file_is_reclaimed() {
2579        let (_dir, path) = temp_connection_file_path("stale-reclaim");
2580        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2581        let stale_port = stale.local_addr().unwrap().port();
2582        drop(stale);
2583        let stale_info = make_connection_info(stale_port);
2584        write_atomic(&path, &stale_info).unwrap();
2585
2586        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2587        assert_ne!(bound.connection_info.key, stale_info.key);
2588        drop(bound.listeners);
2589    }
2590
2591    #[cfg(unix)]
2592    #[tokio::test]
2593    async fn ensure_singleton_reclaims_insecure_connection_file() {
2594        let (_dir, path) = temp_connection_file_path("insecure-reclaim");
2595        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2596        let stale_port = stale.local_addr().unwrap().port();
2597        drop(stale);
2598        let stale_info = make_connection_info(stale_port);
2599        write_atomic(&path, &stale_info).unwrap();
2600        fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
2601
2602        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2603        assert_ne!(bound.connection_info.key, stale_info.key);
2604        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2605        assert_owner_only_connection_file(&path);
2606
2607        drop(bound.listeners);
2608    }
2609
2610    #[tokio::test]
2611    async fn ensure_singleton_reclaims_non_loopback_connection_file() {
2612        let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
2613        let mut stale_info = make_connection_info(8757);
2614        stale_info.endpoints = vec![Endpoint {
2615            host: "192.0.2.10".to_owned(),
2616            port: 8757,
2617        }];
2618        write_atomic(&path, &stale_info).unwrap();
2619
2620        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2621        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
2622        assert_ne!(bound.connection_info.key, stale_info.key);
2623        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2624        assert!(bound
2625            .connection_info
2626            .endpoints
2627            .iter()
2628            .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
2629        assert_owner_only_connection_file(&path);
2630
2631        drop(bound.listeners);
2632    }
2633
2634    #[tokio::test]
2635    async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
2636        let mut unsupported_schema = make_connection_info(8757);
2637        unsupported_schema.schema = SCHEMA_VERSION + 1;
2638
2639        let mut empty_endpoints = make_connection_info(8757);
2640        empty_endpoints.endpoints.clear();
2641
2642        let mut short_key = make_connection_info(8757);
2643        short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
2644
2645        let cases = vec![
2646            (
2647                "unsupported-schema",
2648                serde_json::to_vec(&unsupported_schema).unwrap(),
2649                Some(unsupported_schema),
2650            ),
2651            (
2652                "empty-endpoints",
2653                serde_json::to_vec(&empty_endpoints).unwrap(),
2654                Some(empty_endpoints),
2655            ),
2656            (
2657                "short-key",
2658                serde_json::to_vec(&short_key).unwrap(),
2659                Some(short_key),
2660            ),
2661            ("invalid-json", b"{not valid connection json".to_vec(), None),
2662        ];
2663
2664        for (label, contents, old_info) in cases {
2665            let (_dir, path) = temp_connection_file_path(label);
2666            write_raw_owner_only_connection_file(&path, &contents);
2667
2668            let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2669            if let Some(old_info) = old_info {
2670                assert_ne!(bound.connection_info.key, old_info.key, "{label}");
2671                assert_ne!(
2672                    bound.connection_info.daemon_id, old_info.daemon_id,
2673                    "{label}"
2674                );
2675            }
2676            assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
2677            assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
2678            assert_owner_only_connection_file(&path);
2679
2680            drop(bound.listeners);
2681        }
2682    }
2683
2684    #[tokio::test]
2685    async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
2686        let (_dir, path) = temp_connection_file_path("foreign-reclaim");
2687        let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2688        let foreign_port = foreign.local_addr().unwrap().port();
2689        write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
2690        let foreign_task = tokio::spawn(async move {
2691            if let Ok((mut stream, _)) = foreign.accept().await {
2692                let mut buf = [0u8; 64];
2693                let _ = stream.read(&mut buf).await;
2694            }
2695        });
2696
2697        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2698        assert!(bound
2699            .connection_info
2700            .endpoints
2701            .iter()
2702            .all(|endpoint| endpoint.port != foreign_port));
2703
2704        drop(bound.listeners);
2705        let _ = foreign_task.await;
2706    }
2707
2708    #[tokio::test]
2709    async fn stale_start_lock_file_is_reclaimable() {
2710        let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
2711        let lock_path = start_lock_path(&path);
2712        drop(open_owner_only_lock(&lock_path).unwrap());
2713        assert!(lock_path.is_file());
2714
2715        let lock = StartLock::acquire(&path).await.unwrap();
2716        assert!(lock_path.is_file());
2717
2718        drop(lock);
2719        assert!(lock_path.is_file());
2720    }
2721
2722    #[tokio::test]
2723    async fn held_start_lock_blocks_second_acquire_until_release() {
2724        let (_dir, path) = temp_connection_file_path("start-lock-held");
2725        let lock_path = start_lock_path(&path);
2726        let first = StartLock::acquire(&path).await.unwrap();
2727
2728        let err = match StartLock::acquire(&path).await {
2729            Ok(_) => panic!("second acquire while held must stay busy"),
2730            Err(err) => err,
2731        };
2732        assert!(matches!(
2733            err,
2734            BootstrapError::StartLockBusy {
2735                ref path,
2736                attempts: START_LOCK_RETRIES,
2737            } if path == &lock_path
2738        ));
2739
2740        drop(first);
2741
2742        let second = StartLock::acquire(&path)
2743            .await
2744            .expect("released advisory lock should be reclaimable");
2745        drop(second);
2746    }
2747
2748    #[tokio::test]
2749    async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
2750        let (_dir, path) = temp_connection_file_path("bind-conflict");
2751        let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2752        let occupied_port = occupied.local_addr().unwrap().port();
2753
2754        let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
2755        assert!(matches!(
2756            err,
2757            BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
2758        ));
2759        assert!(err.to_string().contains("set the port in config"));
2760
2761        drop(occupied);
2762    }
2763
2764    #[tokio::test]
2765    async fn ephemeral_ipv6_collision_still_serves_ipv4() {
2766        let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
2767            let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
2768            let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
2769            drop(occupied);
2770            result
2771        })
2772        .await
2773        .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
2774        assert_eq!(endpoints.len(), 1);
2775        assert_eq!(endpoints[0].host, "127.0.0.1");
2776        TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
2777            .await
2778            .unwrap();
2779        drop(listeners);
2780    }
2781
2782    #[tokio::test]
2783    async fn unpublished_boot_holds_singleton_lock_until_publication() {
2784        let dir = ScratchDir::new("ul");
2785        let path = dir.join("c");
2786        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2787        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2788        assert!(!path.exists());
2789        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2790        assert!(
2791            matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
2792            "an unpublished boot must keep its singleton claim"
2793        );
2794        drop(bound);
2795        FileExt::try_lock(&contender).unwrap();
2796    }
2797
2798    #[tokio::test]
2799    async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
2800        let dir = ScratchDir::new("sl");
2801        let path = dir.join("c");
2802        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2803        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2804        // try_clone shares the open file description just as fork does,
2805        // without relying on the timing of another test's child spawn.
2806        let inherited = bound
2807            .publication_lock
2808            .as_ref()
2809            .unwrap()
2810            ._file
2811            .try_clone()
2812            .unwrap();
2813        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2814        assert!(matches!(
2815            FileExt::try_lock(&contender),
2816            Err(TryLockError::WouldBlock)
2817        ));
2818        drop(bound);
2819        FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
2820        drop(inherited);
2821    }
2822
2823    #[tokio::test]
2824    async fn readiness_probe_requires_an_authenticating_server() {
2825        let dir = ScratchDir::new("unserved-readiness");
2826        let bound = expect_bound(
2827            ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
2828                .await
2829                .unwrap(),
2830        );
2831        assert!(matches!(
2832            verify_serving(&bound.connection_info).await,
2833            Err(BootstrapError::StartupNotServing)
2834        ));
2835    }
2836
2837    #[cfg(unix)]
2838    #[tokio::test]
2839    async fn orphan_sweep_finishes_before_connection_file_publication() {
2840        use crate::live_children::{ExecutableIdentity, LiveChild};
2841        let dir = ScratchDir::new("pas");
2842        // A one-letter connection file name, and no long socket path: the sweep
2843        // this test covers runs before the local endpoint is bound, so a longer
2844        // name would only add a way for the test to fail on something else.
2845        let path = dir.join("c");
2846        let record = dir.join("live-children.json");
2847        let ready = dir.join("ready");
2848        let mut child = std::process::Command::new("/bin/sh")
2849            .args([
2850                "-c",
2851                "trap '' TERM; : > \"$0\"; read -r _",
2852                ready.to_str().unwrap(),
2853            ])
2854            .stdin(std::process::Stdio::piped())
2855            .env("XDG_DATA_HOME", dir.path())
2856            .env("XDG_RUNTIME_DIR", dir.path())
2857            .env("XDG_CONFIG_HOME", dir.path())
2858            .spawn()
2859            .unwrap();
2860        timeout(Duration::from_secs(5), async {
2861            while !ready.exists() {
2862                sleep(Duration::from_millis(5)).await;
2863            }
2864        })
2865        .await
2866        .unwrap();
2867        let observed = subc_os::Process::open(child.id())
2868            .unwrap()
2869            .unwrap()
2870            .observe()
2871            .unwrap();
2872        crate::live_children::write_record(
2873            &record,
2874            &[LiveChild {
2875                module_id: "old-sleep".into(),
2876                pid: child.id(),
2877                protocol: subc_control::ModuleProtocol::None,
2878                start_time: Some(observed.start_time),
2879                executable: observed.executable.map(ExecutableIdentity::from),
2880                cgroup_name: None,
2881            }],
2882        )
2883        .unwrap();
2884        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
2885            .with_live_children_record(&record);
2886        let daemon = tokio::spawn(run_with_config(config));
2887        timeout(Duration::from_secs(5), async {
2888            while !path.exists() {
2889                sleep(Duration::from_millis(5)).await;
2890            }
2891        })
2892        .await
2893        .unwrap();
2894        assert!(path.exists());
2895        let ended = child.try_wait().unwrap().is_some();
2896        if !ended {
2897            child.kill().unwrap();
2898        }
2899        child.wait().unwrap();
2900        if ended {
2901            assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
2902        }
2903        daemon.abort();
2904        let _ = daemon.await;
2905        assert!(
2906            ended,
2907            "the published daemon must not still owe its orphan sweep"
2908        );
2909    }
2910
2911    /// Publishing the connection file fsyncs, which can take seconds on a busy
2912    /// disk, so the daemon must do it off the async worker thread. The hook
2913    /// blocks inside the write until another task on the same single-threaded
2914    /// runtime has run. Writing on the worker thread would starve that task: the
2915    /// hook then gives up after its deadline and the test fails, as it does if
2916    /// startup publishes without going through the hooked write at all.
2917    #[tokio::test(flavor = "current_thread")]
2918    async fn connection_file_publication_leaves_the_async_runtime_free() {
2919        use std::sync::atomic::{AtomicBool, Ordering};
2920
2921        let (_dir, path) = temp_connection_file_path("publish-off-runtime");
2922        let write_started = Arc::new(AtomicBool::new(false));
2923        let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
2924        let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
2925        let hook_started = Arc::clone(&write_started);
2926        connection_file_write_hook::install(&path, move || {
2927            hook_started.store(true, Ordering::SeqCst);
2928            let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
2929            let _ = verdict_tx.send(progressed);
2930        });
2931        // Signals only after the write has started, so a message can only come
2932        // from this task running while the write is still blocked.
2933        let ticker = tokio::spawn(async move {
2934            loop {
2935                if write_started.load(Ordering::SeqCst) {
2936                    let _ = progress_tx.send(());
2937                    return;
2938                }
2939                sleep(Duration::from_millis(5)).await;
2940            }
2941        });
2942
2943        let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
2944        let published = timeout(Duration::from_secs(15), async {
2945            while !path.exists() && !daemon.is_finished() {
2946                sleep(Duration::from_millis(5)).await;
2947            }
2948        })
2949        .await;
2950        let daemon_exited = daemon.is_finished();
2951        daemon.abort();
2952        let daemon_result = daemon.await;
2953        ticker.abort();
2954
2955        assert!(
2956            published.is_ok() && !daemon_exited,
2957            "the daemon must start and publish its connection file: {daemon_result:?}"
2958        );
2959        match verdict_rx.try_recv() {
2960            Ok(true) => {}
2961            Ok(false) => panic!(
2962                "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
2963            ),
2964            Err(_) => panic!("startup published the connection file without the hooked write"),
2965        }
2966    }
2967
2968    /// A write that never reports back must stop startup with an error naming
2969    /// the connection file, never be taken as published.
2970    #[tokio::test]
2971    async fn a_panicking_connection_file_write_fails_startup() {
2972        let (_dir, path) = temp_connection_file_path("publish-panics");
2973        connection_file_write_hook::install(&path, || {
2974            panic!("deliberate panic inside the connection-file write")
2975        });
2976
2977        let error = timeout(
2978            Duration::from_secs(15),
2979            run_with_config(BootstrapConfig::new(&path, 0)),
2980        )
2981        .await
2982        .expect("startup must end once the write task fails")
2983        .expect_err("startup must fail when the connection-file write panics");
2984
2985        assert!(
2986            matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
2987            "expected a connection-file write error, got {error:?}"
2988        );
2989        let message = error.to_string();
2990        assert!(message.contains(&path.display().to_string()), "{message}");
2991        assert!(message.contains("panicked"), "{message}");
2992        assert!(!path.exists(), "no connection file may be published");
2993    }
2994
2995    #[cfg(target_os = "macos")]
2996    #[test]
2997    fn nofile_raise_worker() {
2998        let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
2999            return;
3000        };
3001        let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
3002        rlimit::Resource::NOFILE.set(256, hard).unwrap();
3003        raise_nofile_limit_with_ceiling(Some(10240));
3004        std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
3005    }
3006
3007    #[cfg(target_os = "macos")]
3008    #[test]
3009    fn macos_nofile_raise_clamps_to_kernel_ceiling() {
3010        let dir = ScratchDir::new("nofile-ceiling");
3011        let result = dir.join("soft");
3012        let status = std::process::Command::new(std::env::current_exe().unwrap())
3013            .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
3014            .env("SUBC_NOFILE_TEST_FILE", &result)
3015            .env("XDG_DATA_HOME", dir.path())
3016            .env("XDG_RUNTIME_DIR", dir.path())
3017            .env("XDG_CONFIG_HOME", dir.path())
3018            .status()
3019            .unwrap();
3020        assert!(status.success());
3021        let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
3022        assert!(
3023            soft > 256 && soft <= 10240,
3024            "the inherited limit must be raised within the kernel ceiling; got {soft}"
3025        );
3026    }
3027
3028    #[tokio::test]
3029    async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
3030        let (_dir, path) = temp_connection_file_path("key-rotation");
3031        let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3032        let old_info = first.connection_info.clone();
3033        let fixed_port = old_info.endpoints[0].port;
3034        drop(first.listeners);
3035
3036        let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
3037        let new_info = second.connection_info.clone();
3038        assert_ne!(old_info.key, new_info.key);
3039        assert_ne!(old_info.daemon_id, new_info.daemon_id);
3040        let server = start_server(second);
3041
3042        let mut old_stream = connect_from_info(&old_info).await.unwrap();
3043        let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
3044        assert!(
3045            old_auth.is_err(),
3046            "old key must not authenticate after restart"
3047        );
3048
3049        let reread = connection_file::read(&path).unwrap();
3050        let mut new_stream = connect_from_info(&reread).await.unwrap();
3051        authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
3052            .await
3053            .unwrap();
3054
3055        server.abort();
3056        let _ = server.await;
3057    }
3058
3059    #[cfg(unix)]
3060    #[tokio::test]
3061    async fn published_connection_file_permissions_are_owner_only() {
3062        let (_dir, path) = temp_connection_file_path("permissions");
3063        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3064
3065        let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
3066        assert_eq!(mode, 0o600);
3067
3068        drop(bound.listeners);
3069    }
3070
3071    /// A config with the socket in `dir`'s own short subdirectory.
3072    ///
3073    /// A Unix socket path is limited to just over a hundred bytes, and this
3074    /// harness's temporary directories use nearly all of that, so a test that
3075    /// boots a real daemon passes a short directory and a one-letter connection
3076    /// file name; anything longer makes the bind refuse. On Windows there is no
3077    /// socket path to keep short, so the config is returned unchanged.
3078    fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
3079        let dir = config
3080            .connection_file_path
3081            .parent()
3082            .map(Path::to_path_buf)
3083            .unwrap_or_else(|| PathBuf::from("."));
3084        #[cfg(unix)]
3085        let config = BootstrapConfig {
3086            connection_file_path: dir.join("c"),
3087            local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
3088            ..config
3089        };
3090        #[cfg(not(unix))]
3091        let _ = dir;
3092        config
3093    }
3094
3095    /// The shipped binary puts the socket in its own run directory. The file is
3096    /// named from the connection file's path, so a restart finds it again.
3097    #[cfg(unix)]
3098    #[test]
3099    fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
3100        let _env_lock = ENV_LOCK.lock().unwrap();
3101        let root = ScratchDir::new("binary-local-socket");
3102        let data_home = root.join("data");
3103        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3104        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3105        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3106
3107        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3108
3109        let run_dir = data_home.join("cortexkit").join("run");
3110        assert_eq!(
3111            config.local_listener.configured_dir(),
3112            Some(run_dir.as_path()),
3113            "the binary serves from its run directory"
3114        );
3115        let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
3116            .expect("the binary config names a directory");
3117        assert_eq!(socket.parent().unwrap(), run_dir);
3118        assert_eq!(
3119            socket.file_name().unwrap().to_string_lossy(),
3120            "subc.sock",
3121            "a fixed name: the run directory belongs to one data home"
3122        );
3123    }
3124
3125    /// An in-process daemon that names no directory serves TCP only: it publishes
3126    /// no local endpoint, creates nothing under the data home, and says nothing is
3127    /// unavailable, because nothing was asked for.
3128    /// The environment lock covers only the setup: the daemon under test reads
3129    /// its data home from the process environment while it boots, and holding
3130    /// the lock across that await would block every other test that sets one.
3131    #[cfg(unix)]
3132    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3133    async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
3134        let root = ScratchDir::new("in-process-no-local-socket");
3135        let data_home = root.join("data");
3136        fs::create_dir_all(&data_home).unwrap();
3137        let connection_path = root.join("c");
3138        let _data;
3139        let _config;
3140        let _port;
3141        {
3142            let _env_lock = ENV_LOCK.lock().unwrap();
3143            _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3144            _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3145            _port = EnvGuard::unset(SUBC_PORT_ENV);
3146        }
3147
3148        let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
3149
3150        assert!(bound.local_endpoint.is_none());
3151        assert!(bound.local_endpoint_unavailable.is_none());
3152        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3153        assert!(published.local_endpoint.is_none());
3154        assert!(
3155            local_listener::socket_path_for(None, &connection_path).is_none(),
3156            "no configured directory, no socket"
3157        );
3158        let mut leftovers = Vec::new();
3159        let mut pending = vec![data_home.clone()];
3160        while let Some(dir) = pending.pop() {
3161            for entry in fs::read_dir(&dir).unwrap() {
3162                let entry = entry.unwrap();
3163                let path = entry.path();
3164                if entry.file_type().unwrap().is_dir() {
3165                    pending.push(path.clone());
3166                }
3167                if entry.file_name().to_string_lossy().ends_with(".sock") {
3168                    leftovers.push(path);
3169                }
3170            }
3171        }
3172        assert!(
3173            leftovers.is_empty(),
3174            "an in-process daemon must not create a socket in the data home: {leftovers:?}"
3175        );
3176    }
3177
3178    /// A local endpoint that cannot be bound does not stop the daemon: it keeps
3179    /// serving TCP, publishes no local endpoint, and reports the reason where an
3180    /// operator reads daemon facts. Every module stays up for a feature nothing
3181    /// has to use.
3182    #[cfg(unix)]
3183    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3184    async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
3185        let dir = ScratchDir::new("tcp-only");
3186        let canonical = std::fs::canonicalize(&dir).unwrap();
3187        let connection_path = canonical.join("c");
3188        // The socket directory is deep enough that the socket's path is over the
3189        // Unix socket limit, so binding it must fail.
3190        let deep = canonical.join("x".repeat(80));
3191        fs::create_dir(&deep).unwrap();
3192        fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
3193        let config =
3194            BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
3195        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3196        assert!(
3197            bound.local_endpoint.is_none(),
3198            "a socket that cannot be bound is not published"
3199        );
3200        assert!(
3201            bound
3202                .local_endpoint_unavailable
3203                .as_deref()
3204                .is_some_and(|reason| reason.contains("too long")),
3205            "{:?}",
3206            bound.local_endpoint_unavailable
3207        );
3208        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3209        assert!(published.local_endpoint.is_none());
3210        assert!(
3211            !published.info.endpoints.is_empty(),
3212            "TCP is still published"
3213        );
3214
3215        // The daemon serves TCP as usual.
3216        let server = start_server(bound);
3217        let info = connection_file::read(&connection_path).unwrap();
3218        let mut client = connect_from_info(&info).await.unwrap();
3219        authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
3220            .await
3221            .expect("a TCP client still completes the handshake");
3222        server.abort();
3223        let _ = server.await;
3224    }
3225
3226    /// The published file names the local endpoint the daemon actually serves,
3227    /// and a client that trusts the file completes the handshake over it.
3228    #[cfg(unix)]
3229    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3230    async fn published_connection_file_names_the_served_local_endpoint() {
3231        let (_dir, path) = temp_connection_file_path("local-endpoint");
3232        // The socket goes in the system temporary directory rather than inside
3233        // the harness's own directory, whose long lockdown-sandbox path leaves
3234        // no room inside the Unix socket path limit.
3235        let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
3236        let config =
3237            BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
3238        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3239        let endpoint = connection_file::read_with_local_endpoint(&path)
3240            .unwrap()
3241            .local_endpoint
3242            .expect("the daemon serves a local endpoint, so the file names it");
3243        assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
3244        let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
3245            panic!("a Unix daemon publishes a socket path");
3246        };
3247        assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
3248        assert_eq!(
3249            fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
3250            0o600
3251        );
3252
3253        let router = Arc::new(Router::with_default_self_handler());
3254        let auth = auth_for(&bound.connection_info);
3255        let server = tokio::spawn(serve_local_listener(
3256            bound.local_listener.expect("the local listener is bound"),
3257            router,
3258            auth,
3259        ));
3260        let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
3261        authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
3262            .await
3263            .expect("a socket client completes the same handshake as a TCP client");
3264
3265        server.abort();
3266        drop(bound.listeners);
3267    }
3268
3269    /// A configured directory that is not a directory refuses at bind rather
3270    /// than publishing an endpoint nothing serves.
3271    #[cfg(unix)]
3272    #[test]
3273    fn a_socket_directory_that_is_a_file_refuses() {
3274        let dir = ScratchDir::new("local-listener-file-as-dir");
3275        let file = dir.join("not-a-directory");
3276        fs::write(&file, b"x").unwrap();
3277        let error = local_listener::bind(
3278            &LocalListenerConfig::in_socket_dir(file),
3279            &dir.join("connection.json"),
3280        )
3281        .expect_err("a path that is not a directory cannot hold the socket");
3282        assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
3283    }
3284
3285    /// The socket sits directly in the configured directory under one fixed
3286    /// name, and a directory that would put the path over the platform limit is
3287    /// refused rather than shortened around.
3288    #[cfg(unix)]
3289    #[test]
3290    fn the_socket_path_stays_within_the_unix_limit() {
3291        let run_dir = PathBuf::from("/tmp/short-run");
3292        assert!(
3293            local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
3294            "no configured directory means no socket"
3295        );
3296        let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
3297            .expect("a configured directory gets a socket");
3298        assert_eq!(socket, run_dir.join("subc.sock"));
3299        assert_eq!(
3300            socket,
3301            local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
3302            "the name does not come from the connection file"
3303        );
3304
3305        let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
3306        let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
3307            .expect("a configured directory gets a socket");
3308        assert!(
3309            long.as_os_str().len() < 104,
3310            "{} is {} bytes",
3311            long.display(),
3312            long.as_os_str().len()
3313        );
3314    }
3315}