Skip to main content

subc_daemon/
bootstrap.rs

1use std::{
2    collections::BTreeMap,
3    env,
4    error::Error,
5    ffi::OsString,
6    fmt, fs, io,
7    net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8    path::{Path, PathBuf},
9    process,
10    time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17    authenticate_client, connection_file, generate_daemon_id, generate_key,
18    write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
19    LocalEndpoint, SCHEMA_VERSION,
20};
21use tokio::{
22    net::{TcpListener, TcpStream},
23    task::{JoinError, JoinHandle},
24    time::{sleep, timeout},
25};
26use tracing::{error, info, warn};
27
28use crate::{
29    daemon_config::{self, ConfiguredModule, DaemonConfigError},
30    local_listener::{self, BoundLocalListener, LocalListenerConfig},
31    server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
32    supervise::HealthConfig,
33    ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
34    ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
35    SupervisorProcessLiveness,
36};
37use std::sync::Arc;
38
39#[cfg(windows)]
40mod windows_connection_copy;
41
42pub const DEFAULT_SUBC_PORT: u16 = 8757;
43pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
44use subc_transport::CONNECTION_FILE_NAME;
45const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
46const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
47const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
48const START_LOCK_RETRIES: usize = 40;
49const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum ConnectionFileSource {
53    XdgRuntimeDir,
54    TempDirFallback,
55    Explicit,
56}
57
58impl ConnectionFileSource {
59    fn reason(self) -> &'static str {
60        match self {
61            Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
62            Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
63            Self::Explicit => "configured path",
64        }
65    }
66}
67
68impl fmt::Display for ConnectionFileSource {
69    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
70        formatter.write_str(match self {
71            Self::XdgRuntimeDir => "xdg_runtime_dir",
72            Self::TempDirFallback => "temp_dir_fallback",
73            Self::Explicit => "explicit",
74        })
75    }
76}
77
78/// Runtime bootstrap configuration. Production uses the default fixed port and
79/// optional daemon-config override; tests pass port 0 to let the OS assign a free
80/// loopback port and discover it from the connection file.
81#[derive(Debug, Clone, Default)]
82struct AdmissionFactsConfig {
83    carrier_module_id: Option<String>,
84    targets: Option<Vec<String>>,
85}
86
87/// Controls where module cgroups are prepared.
88///
89/// In-process daemons default to [`Self::Disabled`] so they never derive a
90/// production location from the host process. The shipped daemon explicitly uses
91/// [`Self::Current`]. The `ck-subc` binary accepts
92/// `SUBC_CGROUP_PLACEMENT=disabled` for isolated test processes; an unset
93/// variable retains `Current`, and every other value is rejected at startup.
94/// Tests that exercise placement can own a [`Self::Root`].
95#[derive(Debug, Clone, Default, PartialEq, Eq)]
96pub enum CgroupPlacementConfig {
97    #[default]
98    Disabled,
99    Current,
100    Root(PathBuf),
101}
102
103#[derive(Debug, Clone)]
104pub struct BootstrapConfig {
105    #[cfg(feature = "test-support")]
106    shutdown_intent_window: Duration,
107    pub operator_queue_wait: Duration,
108    pub operator_timeout: Duration,
109    pub operator_stuck_grace: Duration,
110    #[cfg(feature = "test-support")]
111    pub operator_script: Option<PathBuf>,
112    #[cfg(feature = "test-support")]
113    pub operator_events: Option<PathBuf>,
114    pub connection_file_path: PathBuf,
115    pub port: u16,
116    pub daemon_ver: String,
117    configured_modules: Vec<ConfiguredModule>,
118    storage_config: Option<daemon_config::StorageConfig>,
119    admission_facts: AdmissionFactsConfig,
120    /// Module ids whose scopes may carry the attributes that grant authority.
121    scope_authority_owners: Vec<String>,
122    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
123    activity_config: daemon_config::ActivityConfig,
124    daemon_config_path: Option<PathBuf>,
125    configured_port: Option<u16>,
126    /// Daemon-wide route.bind relay budget in milliseconds (the fallback for
127    /// any module without a per-module override). `None` = built-in default
128    /// (12s — see `control::DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`).
129    route_bind_relay_default_ms: Option<u64>,
130    reserved_capabilities: BTreeMap<String, String>,
131    watchdog_config: DaemonSelfWatchdogConfig,
132    connection_file_source: ConnectionFileSource,
133    #[cfg(all(test, windows))]
134    profile_override: Option<PathBuf>,
135    /// Where module cgroups are prepared. Disabled unless a caller explicitly
136    /// opts in, because `Current` derives a host location from `/proc/self/cgroup`.
137    cgroup_placement: CgroupPlacementConfig,
138    /// Directory the supervisor writes per-module stdout/stderr capture files
139    /// into. `None` disables capture; the shipped binary supplies its real run
140    /// directory explicitly.
141    capture_logs_dir: Option<PathBuf>,
142    /// File the supervisor appends every module's terminal exits to, so exit
143    /// history survives a daemon restart. `None` keeps terminal history in
144    /// memory only (each module's ring). Absent by default for the same reason
145    /// as `capture_logs_dir`: an in-process daemon booted by a test must not
146    /// append its exits to the operator's real `terminals.jsonl`, where they
147    /// would show up in `ck module terminals`. The shipped binary supplies
148    /// `<run dir>/terminals.jsonl` explicitly.
149    terminal_journal_path: Option<PathBuf>,
150    /// Where the machine id is read from, or minted into when absent. `None`
151    /// serves no machine id. Absent by default for the same reason as
152    /// `capture_logs_dir`: an in-process daemon booted by a test must never
153    /// derive the operator's real data home and mint into it. The shipped binary
154    /// supplies `<data home>/cortexkit/machine-id` explicitly.
155    machine_id_path: Option<PathBuf>,
156
157    /// Where this daemon's same-user local listener lives (a Unix-domain socket
158    /// on macOS and Linux; nothing on Windows, where the pipe name is minted per
159    /// start). The default serves no local endpoint, so an in-process daemon
160    /// never touches the operator's run directory. The shipped binary names its
161    /// run directory; [`BootstrapConfig::with_local_listener_socket_dir`] names
162    /// any other, and an in-process daemon that wants the endpoint names one.
163    local_listener: LocalListenerConfig,
164    /// The live-children record: every supervised process this daemon has
165    /// running, kept so the next daemon can end the ones a crash left behind.
166    /// At startup, before any module is spawned, the previous daemon's record
167    /// here is swept. `None` keeps no record and sweeps nothing, for the same
168    /// reason as `capture_logs_dir`: an in-process daemon booted by a test
169    /// must never signal processes listed in the operator's real run
170    /// directory. The shipped binary supplies `<run dir>/live-children.json`.
171    live_children_path: Option<PathBuf>,
172    privacy_trampoline: Option<PathBuf>,
173}
174
175impl BootstrapConfig {
176    /// Shorten restart intent validity in daemon integration fixtures.
177    #[cfg(feature = "test-support")]
178    pub fn with_shutdown_intent_window(mut self, window: Duration) -> Self {
179        self.shutdown_intent_window = window;
180        self
181    }
182
183    pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
184        Self {
185            #[cfg(feature = "test-support")]
186            shutdown_intent_window: Duration::from_secs(120),
187            connection_file_path: connection_file_path.into(),
188            operator_queue_wait: Duration::from_secs(150),
189            operator_timeout: Duration::from_secs(120),
190            operator_stuck_grace: Duration::from_secs(10),
191            #[cfg(feature = "test-support")]
192            operator_script: None,
193            #[cfg(feature = "test-support")]
194            operator_events: None,
195            port,
196            daemon_ver: DAEMON_VERSION.to_owned(),
197            configured_modules: Vec::new(),
198            storage_config: None,
199            admission_facts: AdmissionFactsConfig::default(),
200            scope_authority_owners: daemon_config::default_scope_authority_owners(),
201            privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
202            activity_config: daemon_config::ActivityConfig::default(),
203            daemon_config_path: None,
204            configured_port: None,
205            route_bind_relay_default_ms: None,
206            reserved_capabilities: BTreeMap::new(),
207            watchdog_config: DaemonSelfWatchdogConfig::default(),
208            connection_file_source: ConnectionFileSource::Explicit,
209            #[cfg(all(test, windows))]
210            profile_override: None,
211            cgroup_placement: CgroupPlacementConfig::default(),
212            capture_logs_dir: None,
213            terminal_journal_path: None,
214            machine_id_path: None,
215            local_listener: LocalListenerConfig { socket_dir: None },
216            live_children_path: None,
217            privacy_trampoline: None,
218        }
219    }
220
221    /// Keep the local listener's socket in `dir`. Embedding daemons and tests
222    /// pass a path inside their own fixture tree.
223    pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
224        self.local_listener = LocalListenerConfig::in_socket_dir(dir);
225        self
226    }
227
228    /// Set where the local listener lives, as [`LocalListenerConfig`] describes.
229    pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
230        self.local_listener = config;
231        self
232    }
233
234    /// Keep the live-children record at `path`, and at startup end the
235    /// processes a previous daemon recorded there that are still running.
236    /// Embedding daemons and tests pass a path inside their own fixture tree.
237    pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
238        self.live_children_path = Some(path.into());
239        self
240    }
241
242    /// macOS module launches require an explicit executable implementing the
243    /// subc-os trampoline entry point. Other systems ignore this setting.
244    pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
245        self.privacy_trampoline = Some(path.into());
246        self
247    }
248
249    /// Serve the machine id stored at `path`, minting it there at startup when
250    /// the file is absent. Embedding daemons and tests pass a path inside their
251    /// own fixture tree.
252    pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
253        self.machine_id_path = Some(path.into());
254        self
255    }
256
257    /// Selects module cgroup placement. The default is disabled.
258    pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
259        self.cgroup_placement = placement;
260        self
261    }
262
263    /// Redirects per-module stdout/stderr capture files out of the real run
264    /// directory. Tests that start an in-process daemon must call this with a
265    /// path inside their fixture tree.
266    pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
267        self.capture_logs_dir = Some(dir.into());
268        self
269    }
270
271    /// Redirects the daemon-private journal, allowing embedded daemons and tests
272    /// to keep their observations out of the operator's live run directory.
273    pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
274        self.terminal_journal_path = Some(path.into());
275        self
276    }
277
278    pub fn from_env() -> Result<Self, BootstrapError> {
279        Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
280    }
281
282    /// The SHIPPED BINARY's config, which is the only caller that should capture
283    /// child output into the operator's real run directory.
284    ///
285    /// Kept separate from `from_env` deliberately: see `capture_logs_dir` and
286    /// `terminal_journal_path` on this struct for why an absent value must mean
287    /// NO CAPTURE and NO JOURNAL rather than the operator's real run directory.
288    /// A run directory that cannot be resolved (a relative data home) refuses
289    /// startup instead of landing under the working directory.
290    pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
291        let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
292        let machine_id_path =
293            crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
294        let config = Self::from_env()?
295            .with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
296            .with_capture_logs_dir(run_dir.join("logs"))
297            .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
298            .with_live_children_record(crate::live_children::record_path(&run_dir))
299            .with_machine_id_path(machine_id_path);
300        #[cfg(feature = "test-support")]
301        let config = {
302            let mut config = config;
303            config.operator_script =
304                std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
305            config.operator_events =
306                std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
307            for (name, value) in [
308                (
309                    "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
310                    &mut config.operator_queue_wait,
311                ),
312                (
313                    "SUBC_TEST_OPERATOR_TIMEOUT_MS",
314                    &mut config.operator_timeout,
315                ),
316                (
317                    "SUBC_TEST_OPERATOR_STUCK_MS",
318                    &mut config.operator_stuck_grace,
319                ),
320            ] {
321                if let Ok(ms) = std::env::var(name) {
322                    if let Ok(ms) = ms.parse::<u64>() {
323                        *value = Duration::from_millis(ms);
324                    }
325                }
326            }
327            config
328        };
329        Ok(config)
330    }
331
332    pub fn from_env_with_daemon_config_path(
333        daemon_config_path: impl AsRef<Path>,
334    ) -> Result<Self, BootstrapError> {
335        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
336        let (daemon_config, activity_config) =
337            daemon_config::load_with_activity(&daemon_config_path)
338                .map_err(BootstrapError::DaemonConfig)?
339                .map(|(config, activity)| (Some(config), activity))
340                .unwrap_or_default();
341        let config_port = daemon_config.as_ref().and_then(|config| config.port);
342        let storage_config = daemon_config
343            .as_ref()
344            .and_then(|config| config.storage.clone());
345        let admission_facts_carrier_module_id = daemon_config
346            .as_ref()
347            .and_then(|config| config.admission_facts_carrier_module_id.clone());
348        let admission_facts_targets = daemon_config
349            .as_ref()
350            .and_then(|config| config.admission_facts_targets.clone());
351        let scope_authority_owners = daemon_config
352            .as_ref()
353            .map(|config| config.scope_authority_owners.clone())
354            .unwrap_or_else(daemon_config::default_scope_authority_owners);
355        let privileged_operator_prompt = daemon_config
356            .as_ref()
357            .map(|config| config.privileged_operator_prompt)
358            .unwrap_or_default();
359        let route_bind_relay_default_ms = daemon_config
360            .as_ref()
361            .and_then(|config| config.route_bind_relay_timeout_ms);
362        let reserved_capabilities = daemon_config
363            .as_ref()
364            .map(|config| config.reserved_capabilities.clone())
365            .unwrap_or_default();
366        let configured_modules = daemon_config
367            .map(|config| config.modules)
368            .unwrap_or_default();
369
370        let port = match env::var(SUBC_PORT_ENV) {
371            Ok(raw) if !raw.trim().is_empty() => {
372                let port = raw
373                    .parse::<u16>()
374                    .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
375                if let Some(config_port) = config_port {
376                    info!(
377                        env = SUBC_PORT_ENV,
378                        env_port = port,
379                        config_port,
380                        "SUBC_PORT overrides daemon config port"
381                    );
382                }
383                port
384            }
385            Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
386        };
387
388        let (connection_file_path, connection_file_source) =
389            connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
390        Ok(Self::new(connection_file_path, port)
391            .with_configured_modules(configured_modules)
392            .with_storage_config(storage_config)
393            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
394            .with_scope_authority_owners(scope_authority_owners)
395            .with_privileged_operator_prompt(privileged_operator_prompt)
396            .with_activity_config(activity_config)
397            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
398            .with_reserved_capabilities(reserved_capabilities)
399            .with_daemon_config_source(daemon_config_path, config_port)
400            .with_connection_file_source(connection_file_source))
401    }
402
403    pub fn with_daemon_config_path(
404        self,
405        daemon_config_path: impl AsRef<Path>,
406    ) -> Result<Self, BootstrapError> {
407        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
408        let (daemon_config, activity_config) =
409            daemon_config::load_with_activity(&daemon_config_path)
410                .map_err(BootstrapError::DaemonConfig)?
411                .map(|(config, activity)| (Some(config), activity))
412                .unwrap_or_default();
413        let configured_port = daemon_config.as_ref().and_then(|config| config.port);
414        let storage_config = daemon_config
415            .as_ref()
416            .and_then(|config| config.storage.clone());
417        let admission_facts_carrier_module_id = daemon_config
418            .as_ref()
419            .and_then(|config| config.admission_facts_carrier_module_id.clone());
420        let admission_facts_targets = daemon_config
421            .as_ref()
422            .and_then(|config| config.admission_facts_targets.clone());
423        let scope_authority_owners = daemon_config
424            .as_ref()
425            .map(|config| config.scope_authority_owners.clone())
426            .unwrap_or_else(daemon_config::default_scope_authority_owners);
427        let privileged_operator_prompt = daemon_config
428            .as_ref()
429            .map(|config| config.privileged_operator_prompt)
430            .unwrap_or_default();
431        let route_bind_relay_default_ms = daemon_config
432            .as_ref()
433            .and_then(|config| config.route_bind_relay_timeout_ms);
434        let reserved_capabilities = daemon_config
435            .as_ref()
436            .map(|config| config.reserved_capabilities.clone())
437            .unwrap_or_default();
438        let configured_modules = daemon_config
439            .map(|config| config.modules)
440            .unwrap_or_default();
441        Ok(self
442            .with_configured_modules(configured_modules)
443            .with_storage_config(storage_config)
444            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
445            .with_scope_authority_owners(scope_authority_owners)
446            .with_privileged_operator_prompt(privileged_operator_prompt)
447            .with_activity_config(activity_config)
448            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
449            .with_reserved_capabilities(reserved_capabilities)
450            .with_daemon_config_source(daemon_config_path, configured_port))
451    }
452
453    pub fn with_configured_modules(
454        mut self,
455        modules: impl IntoIterator<Item = ConfiguredModule>,
456    ) -> Self {
457        self.configured_modules = modules.into_iter().collect();
458        self.configured_modules
459            .sort_by(|left, right| left.module_id.cmp(&right.module_id));
460        self
461    }
462
463    pub fn with_storage_config(
464        mut self,
465        storage_config: Option<daemon_config::StorageConfig>,
466    ) -> Self {
467        self.storage_config = storage_config;
468        self
469    }
470
471    pub fn with_admission_facts_config(
472        mut self,
473        carrier_module_id: Option<String>,
474        targets: Option<Vec<String>>,
475    ) -> Self {
476        self.admission_facts = AdmissionFactsConfig {
477            carrier_module_id,
478            targets,
479        };
480        self
481    }
482
483    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
484    /// Absent from a config, this is `daemon_config::default_scope_authority_owners`.
485    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
486        self.scope_authority_owners = owners;
487        self
488    }
489
490    /// Choose the privileged-operation policy for the daemon's entire lifetime.
491    pub fn with_privileged_operator_prompt(
492        mut self,
493        mode: daemon_config::PrivilegedOperatorPrompt,
494    ) -> Self {
495        self.privileged_operator_prompt = mode;
496        self
497    }
498
499    pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
500        self.activity_config = config;
501        self
502    }
503
504    /// Set the daemon-wide route.bind relay default (the fallback for any
505    /// module without a per-module override). `None` preserves the built-in
506    /// default (12s). `serve_bound_daemon` reads this at startup and threads
507    /// it into the control handler's daemon-wide field.
508    pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
509        self.route_bind_relay_default_ms = ms;
510        self
511    }
512
513    pub fn with_reserved_capabilities(
514        mut self,
515        reserved_capabilities: BTreeMap<String, String>,
516    ) -> Self {
517        self.reserved_capabilities = reserved_capabilities;
518        self
519    }
520
521    fn with_daemon_config_source(
522        mut self,
523        daemon_config_path: PathBuf,
524        configured_port: Option<u16>,
525    ) -> Self {
526        self.daemon_config_path = Some(daemon_config_path);
527        self.configured_port = configured_port;
528        self
529    }
530
531    fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
532        self.connection_file_source = source;
533        self
534    }
535
536    pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
537        self.watchdog_config = watchdog_config;
538        self
539    }
540}
541
542/// Result of singleton discovery.
543// Built once per daemon start and immediately matched, so the size gap between
544// the variants costs nothing worth a box on the public shape.
545#[allow(clippy::large_enum_variant)]
546#[derive(Debug)]
547pub enum Outcome {
548    /// A live daemon authenticated from the connection file; this invocation should exit 0.
549    AlreadyRunning,
550    /// This process won the singleton race, owns bound loopback listener(s), and
551    /// has published a fresh connection file.
552    Bound(BoundDaemon),
553}
554
555#[derive(Debug)]
556pub struct BoundDaemon {
557    pub listeners: Vec<TcpListener>,
558    /// The same-user local listener, bound and not yet served. `None` when
559    /// binding it failed, in which case `local_endpoint` is `None` and
560    /// `local_endpoint_unavailable` says why.
561    pub local_listener: Option<BoundLocalListener>,
562    /// Why the daemon serves no local endpoint, when binding it failed. Reported
563    /// by `server.describe` so an operator can see it.
564    pub local_endpoint_unavailable: Option<String>,
565    /// What the connection file publishes under `local_endpoint`, so a client can
566    /// find the local listener without knowing how it was bound.
567    pub local_endpoint: Option<LocalEndpoint>,
568    pub connection_info: ConnectionInfo,
569    pub connection_file_path: PathBuf,
570    pub connection_file_source: ConnectionFileSource,
571    #[cfg(windows)]
572    connection_file_copy: Option<PathBuf>,
573    /// The machine id this daemon serves, established before any connection is
574    /// accepted. `None` when the config named no machine id path.
575    pub machine_id: Option<crate::machine_id::MachineId>,
576    /// Ownership of the run directory, held until the daemon exits so no other
577    /// daemon sweeps or writes this one's run state. `None` when the config
578    /// keeps no live-children record, and so has no run directory to own.
579    run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
580    /// The singleton start lock, kept when binding deferred publishing the
581    /// connection file. It stays held until the daemon has verified it is serving
582    /// and written the connection file, so a second daemon starting meanwhile
583    /// cannot claim the singleton too. `None` when the connection
584    /// file was already written at bind time.
585    publication_lock: Option<StartLock>,
586}
587
588/// Resolve subc's per-user TCP connection-file path.
589///
590/// `$XDG_RUNTIME_DIR/subc-connection.json` is preferred because the runtime
591/// directory is already per-user on Unix desktops. Without it, subc falls back
592/// to the system temp dir with a per-user token in the filename so different OS
593/// users do not collide on shared temp directories.
594pub fn connection_file_path() -> PathBuf {
595    connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
596}
597
598fn connection_file_path_with_source(
599    runtime_dir: Option<OsString>,
600) -> (PathBuf, ConnectionFileSource) {
601    if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
602        return (
603            PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
604            ConnectionFileSource::XdgRuntimeDir,
605        );
606    }
607
608    (
609        env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
610        ConnectionFileSource::TempDirFallback,
611    )
612}
613
614/// Resolve, claim, and serve the per-user daemon singleton.
615///
616/// A second invocation is successful: if a live daemon authenticates from the
617/// existing connection file, this returns `Ok(())` after logging and the caller
618/// exits with status 0.
619pub async fn run() -> Result<(), BootstrapError> {
620    // `run` is a binary entry point, so it opts into the production cgroup and
621    // child-log locations. In-process callers keep both features disabled by default.
622    run_with_config(
623        BootstrapConfig::from_env_for_daemon_binary()?
624            .with_cgroup_placement(CgroupPlacementConfig::Current),
625    )
626    .await
627}
628
629/// Serve a daemon from an explicit config. This is the entry point the twelve
630/// sibling repos use to boot an in-process daemon in their integration tests.
631///
632/// # THIS INSTALLS NO TRACING SUBSCRIBER, SO THE DAEMON IS SILENT BY DEFAULT
633///
634/// The daemon's own diagnostics go through `tracing`, and `tracing` DISCARDS
635/// every event when no subscriber is installed. The shipped binary installs one
636/// in `main` (`init_tracing`); this function deliberately does not, because a
637/// library that installs a global subscriber fights with whatever the host
638/// process already set up.
639///
640/// The consequence for a test harness is not "less verbose": it is that the
641/// daemon has NOTHING TO SAY about any failure, and a missing instrument reads
642/// exactly like a clean one. PLEX found this on 2026-09-18 while trying to
643/// capture daemon logs beside an intermittent bind failure, and discovered the
644/// daemon had been silent in every conformance run that repo had ever done --
645/// so the one client-side error string was all the evidence that could exist,
646/// and they had spent a real investigation on a failure whose second source was
647/// never being recorded.
648///
649/// Install one in the harness before calling this, and assert it did something
650/// (they measured 236 daemon lines with the subscriber installed, 0 with the
651/// call commented out) -- otherwise the fix is itself unverified.
652pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
653    #[cfg(feature = "test-support")]
654    let shutdown_intent_window = config.shutdown_intent_window;
655    let forwarding = Arc::new(ForwardingTable::default());
656    forwarding.operator_confirms().configure_bootstrap(&config);
657    let configured_modules = config.configured_modules.clone();
658    let storage_config = config.storage_config.clone();
659    let admission_facts = config.admission_facts.clone();
660    let scope_authority_owners = config.scope_authority_owners.clone();
661    let privileged_operator_prompt = config.privileged_operator_prompt;
662    let activity_config = config.activity_config;
663    let daemon_config_path = config.daemon_config_path.clone();
664    let configured_port = config.configured_port;
665    let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
666    let reserved_capabilities = config.reserved_capabilities.clone();
667    let watchdog_config = config.watchdog_config.clone();
668    let cgroup_placement_config = config.cgroup_placement.clone();
669    let capture_logs_dir = config.capture_logs_dir.clone();
670    let terminal_journal_path = config.terminal_journal_path.clone();
671    let live_children_path = config.live_children_path.clone();
672    let privacy_trampoline = config.privacy_trampoline.clone();
673    match ensure_singleton_inner(config, false).await? {
674        Outcome::AlreadyRunning => {
675            info!("subc daemon already running");
676            Ok(())
677        }
678        Outcome::Bound(bound) => {
679            #[cfg(target_os = "linux")]
680            let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
681            #[cfg(not(target_os = "linux"))]
682            let _ = cgroup_placement_config;
683            serve_bound_daemon(
684                bound,
685                forwarding,
686                configured_modules,
687                storage_config,
688                admission_facts,
689                scope_authority_owners,
690                privileged_operator_prompt,
691                activity_config,
692                daemon_config_path,
693                configured_port,
694                route_bind_relay_default_ms,
695                reserved_capabilities,
696                watchdog_config,
697                capture_logs_dir,
698                terminal_journal_path,
699                live_children_path,
700                privacy_trampoline,
701                #[cfg(feature = "test-support")]
702                shutdown_intent_window,
703                #[cfg(target_os = "linux")]
704                cgroup_placement,
705            )
706            .await
707        }
708    }
709}
710
711#[cfg(target_os = "linux")]
712fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
713    let result = match config {
714        CgroupPlacementConfig::Disabled => return None,
715        CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
716        CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
717    };
718
719    match result {
720        Ok(Some(placement)) => Some(placement),
721        Ok(None) => {
722            warn!(
723                placement = ?config,
724                "module cgroup placement is disabled: configured cgroup root is not delegated"
725            );
726            None
727        }
728        Err(error) => {
729            warn!(
730                placement = ?config,
731                error = %error,
732                "module cgroup placement is disabled by an unexpected cgroup probe error"
733            );
734            None
735        }
736    }
737}
738
739/// Target soft limit for open file descriptors, applied to the daemon before any
740/// module is spawned so children inherit it. Multi-root modules (one process
741/// aggregating every project root's sqlite stores, index caches, watchers, and
742/// LSP pipes) trivially exceed the macOS default soft limit of 256; a launchd
743/// user agent does not pass login-shell ulimits through, so the raise must
744/// happen in-process.
745#[cfg(unix)]
746const NOFILE_TARGET: u64 = 65536;
747
748/// Raise RLIMIT_NOFILE to `NOFILE_TARGET` (clamped to the hard limit).
749/// Best-effort: failure is logged and never fatal, since the daemon can run
750/// under the inherited limit — modules with few roots just have less headroom.
751#[cfg(unix)]
752fn raise_nofile_limit() {
753    #[cfg(target_os = "macos")]
754    let ceiling = std::process::Command::new("/usr/sbin/sysctl")
755        .args(["-n", "kern.maxfilesperproc"])
756        .output()
757        .ok()
758        .filter(|output| output.status.success())
759        .and_then(|output| String::from_utf8(output.stdout).ok())
760        .and_then(|value| value.trim().parse::<u64>().ok())
761        .filter(|value| *value > 0);
762    #[cfg(not(target_os = "macos"))]
763    let ceiling = None;
764    raise_nofile_limit_with_ceiling(ceiling);
765}
766
767#[cfg(unix)]
768fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
769    match rlimit::Resource::NOFILE.get() {
770        Ok((soft, hard)) => {
771            // On XNU an infinite hard limit does not remove the separate
772            // per-process kernel ceiling. Exceeding it makes setrlimit fail
773            // without raising the inherited (often 256) soft limit at all.
774            let target = NOFILE_TARGET
775                .min(hard)
776                .min(kernel_ceiling.unwrap_or(u64::MAX));
777            if soft >= target {
778                return;
779            }
780            match rlimit::Resource::NOFILE.set(target, hard) {
781                Ok(()) => info!(
782                    previous_soft = soft,
783                    new_soft = target,
784                    hard,
785                    "raised open-file soft limit for daemon and module children"
786                ),
787                Err(err) => warn!(
788                    soft,
789                    hard,
790                    error = %err,
791                    "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
792                ),
793            }
794        }
795        Err(err) => warn!(error = %err, "could not read open-file limit"),
796    }
797}
798
799/// CRT stdio-stream target on Windows (the `_setmaxstdio` maximum). Win32
800/// HANDLEs — what Rust `File`, tokio sockets, and SQLite's Win32 VFS actually
801/// consume — have a per-process quota in the millions and need no raise; the
802/// C-runtime stream table (default 512) is the only low ceiling, and it is
803/// per-process rather than inherited, so supervised modules linking the CRT
804/// must raise their own. Raising it here covers the daemon itself.
805#[cfg(windows)]
806fn raise_nofile_limit() {
807    const MAXSTDIO_TARGET: u32 = 8192;
808    let current = rlimit::getmaxstdio();
809    if current >= MAXSTDIO_TARGET {
810        return;
811    }
812    match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
813        Ok(new_max) => info!(
814            previous = current,
815            new_max, "raised CRT stdio-stream limit for daemon"
816        ),
817        Err(err) => warn!(
818            current,
819            error = %err,
820            "could not raise CRT stdio-stream limit"
821        ),
822    }
823}
824
825#[cfg(not(any(unix, windows)))]
826fn raise_nofile_limit() {}
827
828pub async fn run_with_daemon_config_path(
829    config: BootstrapConfig,
830    daemon_config_path: impl AsRef<Path>,
831) -> Result<(), BootstrapError> {
832    run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
833}
834
835#[allow(clippy::too_many_arguments)]
836async fn serve_bound_daemon(
837    bound: BoundDaemon,
838    forwarding: Arc<ForwardingTable>,
839    configured_modules: Vec<ConfiguredModule>,
840    storage_config: Option<daemon_config::StorageConfig>,
841    admission_facts: AdmissionFactsConfig,
842    scope_authority_owners: Vec<String>,
843    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
844    activity_config: daemon_config::ActivityConfig,
845    daemon_config_path: Option<PathBuf>,
846    configured_port: Option<u16>,
847    route_bind_relay_default_ms: Option<u64>,
848    reserved_capabilities: BTreeMap<String, String>,
849    watchdog_config: DaemonSelfWatchdogConfig,
850    capture_logs_dir: Option<PathBuf>,
851    terminal_journal_path: Option<PathBuf>,
852    live_children_path: Option<PathBuf>,
853    privacy_trampoline: Option<PathBuf>,
854    #[cfg(feature = "test-support")] shutdown_intent_window: Duration,
855    #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
856) -> Result<(), BootstrapError> {
857    #[cfg(unix)]
858    let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
859        .map_err(BootstrapError::Signal)?;
860    // Windows has no SIGTERM. Ctrl-C is a different event, not an equivalent
861    // service-stop contract, so this shutdown handler is intentionally Unix-only.
862    raise_nofile_limit();
863    #[cfg(windows)]
864    if let Err(error) = subc_jobobject::ensure_root_job() {
865        // Keep the control plane available. The root-job API caches setup
866        // failures, so later module spawns refuse instead of running unprotected.
867        warn!(%error, "module launches disabled: Windows root job setup failed");
868    }
869
870    info!(
871        connection_file = %bound.connection_file_path.display(),
872        connection_file_source = %bound.connection_file_source,
873        connection_file_source_reason = bound.connection_file_source.reason(),
874        endpoints = ?bound.connection_info.endpoints,
875        configured_modules = configured_modules.len(),
876        machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
877        "subc daemon starting"
878    );
879
880    // Before anything can spawn a module (the configured modules below, or a
881    // client's start request once the listeners are served): a previous daemon
882    // that died without its shutdown stop may have left children running, and
883    // a fresh copy beside one would fight it for its port and stores. The
884    // record is not a running daemon's because this daemon holds the lock on
885    // the run directory the record lives in, and a running daemon holds that
886    // lock for its whole life. (Claiming the singleton is not enough: it is
887    // keyed on the connection file, which can live in a different runtime
888    // directory from the run directory.) The lock stays held until this
889    // function returns, which is when the daemon exits.
890    let run_dir_lock = bound.run_dir_lock;
891    let publication_lock = bound.publication_lock;
892    if let Some(owner) = &run_dir_lock {
893        #[cfg(target_os = "linux")]
894        crate::live_children::sweep_orphans_with_placement(
895            owner,
896            &crate::live_children::AdoptedPids::none(),
897            crate::live_children::SweepBounds::default(),
898            cgroup_placement.as_ref(),
899        )
900        .await;
901        #[cfg(not(target_os = "linux"))]
902        crate::live_children::sweep_orphans(
903            owner,
904            &crate::live_children::AdoptedPids::none(),
905            crate::live_children::SweepBounds::default(),
906        )
907        .await;
908    }
909
910    let registry = Arc::new(Registry::default());
911    let process_liveness = Arc::new(SupervisorProcessLiveness::new());
912    let supervisor_handle = SupervisorHandle::new();
913    #[cfg(feature = "test-support")]
914    supervisor_handle.set_shutdown_intent_window(shutdown_intent_window);
915    let connected_clients = ConnectedClients::new();
916    let daemon_incarnation = format!(
917        "{:032x}",
918        u128::from_be_bytes(bound.connection_info.daemon_id)
919    );
920    let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
921        .with_process_liveness(process_liveness.clone())
922        .with_forwarding(Arc::clone(&forwarding))
923        .with_handle(supervisor_handle.clone())
924        .with_connection_file_path(bound.connection_file_path.clone())
925        .with_daemon_incarnation(daemon_incarnation.clone());
926    let supervisor = match privacy_trampoline {
927        Some(path) => supervisor.with_privacy_trampoline(path),
928        None => supervisor,
929    };
930    // ABSENT MEANS NO CAPTURE AND NO JOURNAL, NOT "THE REAL RUN DIRECTORY", and
931    // the difference is a production-corruption hazard rather than a preference.
932    // Both fields below follow the same rule: `None` for `capture_logs_dir`
933    // means supervised output is not captured, and `None` for
934    // `terminal_journal_path` means terminal history lives only in each
935    // module's in-memory ring (`supervisor.terminals` still answers, with the
936    // journal counters at zero).
937    //
938    // The terminal journal used to fall back to
939    // `daemon_run_dir().join("terminals.jsonl")`, so an in-process test daemon
940    // appended its fixture exits to the operator's journal, where they then
941    // appeared in `ck module terminals`.
942    //
943    // The capture line used to be `unwrap_or_else(|| daemon_run_dir().join("logs"))`,
944    // so ANY caller that did not set the field captured supervised children into
945    // the operator's live `~/.local/share/cortexkit/run/logs/`. That is twelve
946    // sibling repos whose integration tests boot an in-process daemon through
947    // `run_with_config` -- none of which asked for it, and none of which can see
948    // it from their side.
949    //
950    // Harmless while fixture module ids are fixture-shaped: this host carries 20
951    // zero-byte files from subc's own tests (good-aft, missing-aft,
952    // preview-consumer...). THE HAZARD IS A COLLISION. A fixture named "broca"
953    // or "aft" appends to a PRODUCTION capture file that operators read
954    // forensically and that placement gates count lines in -- with no residue to
955    // notice, because the file legitimately exists and legitimately grows.
956    //
957    // Found by BROCA (2026-09-19) from the other side: their rigs spawn the
958    // SHIPPED ck-subc and set XDG_CONFIG_HOME + XDG_RUNTIME_DIR but not
959    // XDG_DATA_HOME, so every local rig run supervised a module named "broca"
960    // and captured it into production's broca.stderr.log -- the same file I
961    // count seal lines in before and after placing their binaries.
962    //
963    // The supervisor already treats `None` as no-capture and no-journal, so
964    // this only removes invented defaults. The binary keeps capturing and
965    // journaling via `BootstrapConfig::from_env_for_daemon_binary`.
966
967    // Keep both ledgers in the explicitly selected run directory. An embedded
968    // daemon without a journal path must not write to the operator's live ledger.
969    let activity_run_dir = terminal_journal_path
970        .as_ref()
971        .and_then(|path| path.parent())
972        .map(Path::to_path_buf);
973    let supervisor = match terminal_journal_path {
974        Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
975        None => supervisor,
976    };
977    let supervisor = match capture_logs_dir {
978        Some(dir) => supervisor.with_capture_logs_dir(dir),
979        None => supervisor,
980    };
981    let supervisor = match live_children_path {
982        Some(path) => supervisor.with_live_children_record(path),
983        None => supervisor,
984    };
985    #[cfg(target_os = "linux")]
986    let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
987    // Collect per-module route.bind relay overrides BEFORE handing the
988    // `configured_modules` vector to the supervisor (which only needs each
989    // module's `drain_timeout_ms`). Each entry was filled in by parse-time
990    // resolution (per-module > daemon-wide > absent), so modules with no
991    // override are absent from this map and the daemon-wide default applies.
992    let route_bind_relay_timeouts = configured_modules
993        .iter()
994        .filter_map(|module| {
995            module
996                .route_bind_relay_timeout_ms
997                .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
998        })
999        .collect::<std::collections::BTreeMap<_, _>>();
1000    let control_start_clock = crate::clock::StartClock::capture();
1001    let mut control = ControlHandler::with_forwarding(Arc::clone(&registry), forwarding)
1002        .with_daemon_version(bound.connection_info.daemon_ver.clone())
1003        .with_process_liveness(process_liveness)
1004        .with_supervisor(supervisor_handle)
1005        // The same flag `begin_daemon_shutdown` sets below, so requirement
1006        // changes while modules exit during the shutdown log at INFO.
1007        .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
1008        .with_connected_clients(connected_clients.clone())
1009        .with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
1010        .with_storage_config(storage_config)
1011        .with_machine_id(bound.machine_id.clone())
1012        .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
1013        .with_scope_authority_owners(scope_authority_owners)
1014        .with_privileged_operator_prompt(privileged_operator_prompt)
1015        .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
1016        .with_daemon_provenance(
1017            bound.connection_info.pid,
1018            control_start_clock.started_at_ms(),
1019            std::env::current_exe().ok(),
1020            normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
1021            normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
1022        )
1023        .with_daemon_start_clock(control_start_clock)
1024        .with_capability_config(
1025            configured_modules
1026                .iter()
1027                .map(|module| (module.module_id.clone(), module.enabled)),
1028            reserved_capabilities,
1029        );
1030    if let Some(run_dir) = activity_run_dir {
1031        control =
1032            control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
1033    }
1034    if let Some(ms) = route_bind_relay_default_ms {
1035        // A daemon-wide config value overrides the built-in default; a
1036        // `None` here leaves the ControlHandler's 12s default in place.
1037        control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
1038    }
1039    if let Some(config_path) = daemon_config_path {
1040        control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
1041    }
1042    let control = Arc::new(control);
1043    let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
1044    let auth = ServerAuth::new(
1045        bound.connection_info.key.clone(),
1046        bound.connection_info.daemon_id,
1047        bound.connection_info.daemon_ver.clone(),
1048    )
1049    .with_connected_clients(connected_clients);
1050
1051    // Discovery must expose every accepted configured module, including disabled
1052    // ones. Registration creates status handles but neither starts monitors
1053    // nor launches processes: children need the published file for `--subc`.
1054    let mut registered_modules = Vec::with_capacity(configured_modules.len());
1055    for configured in configured_modules {
1056        let enabled = configured.enabled;
1057        let health = configured.health.clone();
1058        let module_id = configured.module_id.clone();
1059        let registered = match supervisor.register_configured_with_health(
1060            configured.module_spec(),
1061            enabled,
1062            health.clone(),
1063            configured.drain_timeout_ms,
1064            configured.restart,
1065            configured.register_timeout,
1066        ) {
1067            Ok(registered) => registered,
1068            Err(err) => {
1069                error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
1070                continue;
1071            }
1072        };
1073        registered_modules.push(registered);
1074        // A relaxed threshold can silently outlive the temporary allowance
1075        // that required it, so report the extra tolerance on every boot.
1076        let default_threshold = HealthConfig::default().failure_threshold;
1077        if enabled && health.failure_threshold > default_threshold {
1078            warn!(
1079                module_id = %module_id,
1080                failure_threshold = health.failure_threshold,
1081                default_threshold,
1082                tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1083                "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1084            );
1085        }
1086        info!(module_id = %module_id, enabled, "configured module registered");
1087    }
1088    control.refresh_capability_requirements();
1089
1090    // The local endpoint is served beside TCP and answers the same handshake.
1091    // It serves clients only, which the server enforces per connection.
1092    let local_serve_task = bound.local_listener.map(|local_listener| {
1093        AbortOnDrop::new(tokio::spawn(serve_local_listener(
1094            local_listener,
1095            Arc::clone(&router),
1096            auth.clone(),
1097        )))
1098    });
1099    // On Unix the task is dropped below to stop accepting at shutdown. Windows
1100    // leaves it to the process exit; naming it keeps the binding uniform.
1101    #[cfg(windows)]
1102    let _local_serve_task = local_serve_task;
1103    let mut serve_task =
1104        AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1105    verify_serving(&bound.connection_info).await?;
1106    // A published endpoint promises that authentication can complete now.
1107    // Orphan cleanup and control-plane construction can take longer than an
1108    // existing-daemon probe's deadline, so finish both before discovery.
1109    // The write fsyncs, so it runs on tokio's blocking pool (see
1110    // `publish_connection_file`). The singleton lock travels with it and comes
1111    // back only once the file is in place.
1112    let (published, publication_lock) = publish_connection_file(
1113        &bound.connection_file_path,
1114        #[cfg(windows)]
1115        bound.connection_file_copy.as_deref(),
1116        &bound.connection_info,
1117        bound.local_endpoint.as_ref(),
1118        publication_lock,
1119    )
1120    .await;
1121    published?;
1122    #[cfg(test)]
1123    startup_publication_hook::run(&bound.connection_file_path).await;
1124    drop(publication_lock);
1125    let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1126    // Off the startup path: it runs `systemctl`, and only ever logs.
1127    #[cfg(target_os = "linux")]
1128    let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1129        crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1130    ));
1131    let _watchdog_task = AbortOnDrop::new(
1132        DaemonSelfWatchdog::new(
1133            bound.connection_info.clone(),
1134            bound.connection_file_path.clone(),
1135        )
1136        .with_config(watchdog_config)
1137        .spawn(),
1138    );
1139
1140    for registered in registered_modules {
1141        match supervisor.start_registered_configured(registered, None, true) {
1142            Ok(module) => {
1143                info!(module_id = %module.module_id(), "configured module supervised");
1144            }
1145            Err(err) => {
1146                error!(error = %err, "failed to start configured module; continuing daemon startup");
1147            }
1148        }
1149    }
1150
1151    control.refresh_capability_requirements();
1152    Arc::clone(&control).spawn_capability_deadline_loop();
1153    Arc::clone(&control).spawn_scope_expiry_loop();
1154    control.spawn_resource_sampler();
1155
1156    #[cfg(unix)]
1157    {
1158        tokio::select! {
1159            result = serve_task.join() => {
1160                return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1161            }
1162            _ = terminate.recv() => {}
1163        }
1164        // First, before the notice, the drain, or any connection close: from
1165        // here on a module exit is recorded as `daemon_shutdown` and never
1166        // respawned. Also before allowing a second signal to cut the bounded
1167        // wait short, so the journal marker is always written.
1168        supervisor.begin_daemon_shutdown();
1169        // Stop the self-watchdog before closing the listener. Its next tick would
1170        // connect to that listener, fail, and log an ERROR indistinguishable from
1171        // a wedged daemon, once for every interval a planned stop lasts.
1172        drop(_watchdog_task);
1173        // Dropping the listener stops new accepts, not established connections:
1174        // their detached tasks must remain live throughout notice and drain. The
1175        // local endpoint stops accepting the same way.
1176        drop(serve_task);
1177        drop(local_serve_task);
1178        let second_sigterm = async {
1179            terminate.recv().await;
1180            std::time::Instant::now()
1181        };
1182        tokio::pin!(second_sigterm);
1183        let mut escalation_started = tokio::select! {
1184            biased;
1185            at = second_sigterm.as_mut() => {
1186                info!("second SIGTERM: abandoning daemon shutdown wait");
1187                Some(at)
1188            }
1189            result = supervisor.drain_for_daemon_shutdown() => {
1190                if let Err(error) = result {
1191                    warn!(%error, "daemon shutdown drain failed; exiting anyway");
1192                }
1193                None
1194            }
1195        };
1196        // Supervised modules lead their own process groups, so a service
1197        // manager's kill of this process's group does not reach them. (A
1198        // systemd unit with KillMode=control-group kills by cgroup instead and
1199        // does reach them; see `systemd_kill_mode`.) The daemon ends them
1200        // itself: EOF (or SIGTERM for a protocol none child) first, then
1201        // signals at each child's own drain deadline.
1202        let already_escalated = escalation_started.is_some();
1203        supervisor
1204            .end_children_for_daemon_shutdown(already_escalated, async {
1205                if already_escalated {
1206                    std::future::pending::<()>().await;
1207                } else {
1208                    escalation_started = Some(second_sigterm.await);
1209                }
1210            })
1211            .await;
1212        if let Some(at) = escalation_started {
1213            info!(
1214                second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1215                "daemon shutdown wait ended after second SIGTERM"
1216            );
1217        }
1218        info!("daemon exiting: SIGTERM shutdown complete");
1219        Ok(())
1220    }
1221    #[cfg(not(unix))]
1222    serve_task
1223        .join()
1224        .await
1225        .map_err(BootstrapError::ServeJoin)?
1226        .map_err(BootstrapError::Serve)
1227}
1228
1229fn normalized_build_provenance(value: &str) -> Option<String> {
1230    match value.trim() {
1231        "" | "unavailable" => None,
1232        value => Some(value.to_string()),
1233    }
1234}
1235
1236/// Find an existing daemon or atomically bind loopback TCP for this daemon.
1237///
1238/// The algorithm is intentionally connect-first: an endpoint from the connection
1239/// file is treated as live only after the TCP+key server-proof authenticates for
1240/// that file's key and daemon_id. Stale or foreign connection files are reclaimed
1241/// only while holding the per-user start lock; the TCP port is never the
1242/// singleton primitive.
1243pub async fn ensure_singleton(
1244    connection_file_path: impl AsRef<Path>,
1245    port: u16,
1246) -> Result<Outcome, BootstrapError> {
1247    ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1248}
1249
1250pub async fn ensure_singleton_with_config(
1251    config: BootstrapConfig,
1252) -> Result<Outcome, BootstrapError> {
1253    ensure_singleton_inner(config, true).await
1254}
1255
1256async fn ensure_singleton_inner(
1257    config: BootstrapConfig,
1258    publish: bool,
1259) -> Result<Outcome, BootstrapError> {
1260    #[cfg(windows)]
1261    let copy_path = windows_connection_copy::resolve(&config);
1262    let path = config.connection_file_path;
1263
1264    if stable_peer(&path).await? {
1265        return Ok(Outcome::AlreadyRunning);
1266    }
1267
1268    let lock = StartLock::acquire(&path).await?;
1269
1270    // Re-probe after acquiring the start lock so a peer that won the race between
1271    // our first failed probe and the lock acquisition is observed instead of
1272    // overwritten.
1273    if stable_peer(&path).await? {
1274        return Ok(Outcome::AlreadyRunning);
1275    }
1276
1277    // The start lock and the probe above only rule out a daemon using this
1278    // connection file. A daemon started with another runtime directory but
1279    // the same data home shares the run directory while passing both, so the
1280    // run directory needs its own owner: refuse at once if a live daemon holds
1281    // it, before anything below reads or writes run or data-home state.
1282    let run_dir_lock = config
1283        .live_children_path
1284        .as_deref()
1285        .map(crate::run_dir_lock::RunDirLock::acquire)
1286        .transpose()?;
1287
1288    remove_stale_connection_file_if_present(&path)?;
1289    #[cfg(windows)]
1290    let copy_path = windows_connection_copy::reclaim(&path, copy_path).await;
1291
1292    // Established under the start lock, and under the run-directory lock when
1293    // there is one, and before binding: a corrupt file stops boot before
1294    // anything is published, and two daemons racing to start cannot both mint.
1295    //
1296    // Minting a new machine id fsyncs its file, so loading or minting runs on
1297    // tokio's blocking pool. Both locks travel with that work, so neither is
1298    // released while a mint is in flight.
1299    let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1300        None => (None, lock, run_dir_lock),
1301        Some(machine_id_path) => {
1302            let task_path = machine_id_path.clone();
1303            match run_blocking_holding((lock, run_dir_lock), move || {
1304                crate::machine_id::load_or_mint(&task_path)
1305            })
1306            .await
1307            {
1308                Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1309                Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1310                Err(join_error) => {
1311                    return Err(BootstrapError::MachineId(
1312                        crate::machine_id::MachineIdFileError::Write {
1313                            path: machine_id_path,
1314                            source: io::Error::other(describe_blocking_task_failure(
1315                                "loading or minting the machine id",
1316                                &join_error,
1317                            )),
1318                        },
1319                    ))
1320                }
1321            }
1322        }
1323    };
1324
1325    let (listeners, endpoints) = bind_loopback(config.port).await?;
1326    // The local endpoint is an addition to the daemon, not a condition for it:
1327    // a bind failure leaves clients on TCP only, with the connection file naming
1328    // no local endpoint, and the reason reported by `server.describe`. A daemon
1329    // that refused to start would take every module down for a feature nothing
1330    // has to use yet.
1331    let (local_listener, local_endpoint, local_endpoint_unavailable) =
1332        if config.local_listener.configured_dir().is_some() {
1333            match local_listener::bind(&config.local_listener, &path) {
1334                Ok(local_listener) => {
1335                    let endpoint = local_listener.endpoint();
1336                    (Some(local_listener), Some(endpoint), None)
1337                }
1338                Err(error) => {
1339                    let reason = error.to_string();
1340                    warn!(
1341                        error = %reason,
1342                        "the local endpoint is unavailable; serving loopback TCP only"
1343                    );
1344                    (None, None, Some(reason))
1345                }
1346            }
1347        } else {
1348            (None, None, None)
1349        };
1350    let connection_info = ConnectionInfo {
1351        schema: SCHEMA_VERSION,
1352        wire_version: Some(PROTOCOL_VERSION),
1353        endpoints,
1354        key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1355        daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1356        pid: process::id(),
1357        daemon_ver: config.daemon_ver,
1358    };
1359
1360    // The low-level binding API historically publishes for callers that serve
1361    // these listeners themselves. The daemon entry point defers publication
1362    // until its own authentication server is running.
1363    let publication_lock = if publish {
1364        let (published, lock) = publish_connection_file(
1365            &path,
1366            #[cfg(windows)]
1367            copy_path.as_deref(),
1368            &connection_info,
1369            local_endpoint.as_ref(),
1370            lock,
1371        )
1372        .await;
1373        if let Err(error) = published {
1374            if let Some(local_listener) = local_listener.as_ref() {
1375                local_listener::release(local_listener);
1376            }
1377            drop(listeners);
1378            return Err(error);
1379        }
1380        drop(lock);
1381        None
1382    } else {
1383        Some(lock)
1384    };
1385
1386    Ok(Outcome::Bound(BoundDaemon {
1387        listeners,
1388        local_listener,
1389        local_endpoint,
1390        local_endpoint_unavailable,
1391        connection_info,
1392        connection_file_path: path,
1393        connection_file_source: config.connection_file_source,
1394        #[cfg(windows)]
1395        connection_file_copy: copy_path,
1396        machine_id,
1397        run_dir_lock,
1398        publication_lock,
1399    }))
1400}
1401
1402#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1403enum Probe {
1404    Live,
1405    StaleOrAbsent,
1406}
1407
1408async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1409    if !matches!(probe_existing(path).await?, Probe::Live) {
1410        return Ok(false);
1411    }
1412    // A terminating peer can still authenticate while its listeners drain.
1413    // Require another proof after a short settling interval before reporting
1414    // a successful duplicate start. If the peer left, proceed with startup;
1415    // any remaining bind/ownership conflict is an error, not exit 0 (which
1416    // would suppress launchd's restart-on-failure policy).
1417    tokio::time::sleep(Duration::from_millis(1500)).await;
1418    Ok(matches!(probe_existing(path).await?, Probe::Live))
1419}
1420
1421async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1422    let info = match connection_file::read(path) {
1423        Ok(info) => info,
1424        Err(source) if is_absent_or_stale_connection_file(&source) => {
1425            return Ok(Probe::StaleOrAbsent)
1426        }
1427        Err(source) => {
1428            return Err(BootstrapError::ConnectionFileRead {
1429                path: path.to_path_buf(),
1430                source,
1431            })
1432        }
1433    };
1434
1435    for endpoint in &info.endpoints {
1436        if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1437            return Ok(Probe::Live);
1438        }
1439    }
1440
1441    Ok(Probe::StaleOrAbsent)
1442}
1443
1444async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1445    let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1446        return Probe::StaleOrAbsent;
1447    };
1448    if !ip.is_loopback() {
1449        return Probe::StaleOrAbsent;
1450    }
1451    let addr = SocketAddr::new(ip, endpoint.port);
1452
1453    let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1454        Ok(Ok(stream)) => stream,
1455        Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1456    };
1457
1458    match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1459        Ok(()) => Probe::Live,
1460        Err(AuthError::DaemonIdMismatch)
1461        | Err(AuthError::InvalidServerProof)
1462        | Err(AuthError::UnexpectedEof { .. })
1463        | Err(AuthError::Timeout { .. })
1464        | Err(AuthError::JsonEncode { .. })
1465        | Err(AuthError::JsonDecode { .. })
1466        | Err(AuthError::Io { .. })
1467        | Err(AuthError::MessageTooLarge { .. })
1468        | Err(AuthError::KeyTooShort { .. })
1469        | Err(AuthError::Random(_))
1470        | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1471    }
1472}
1473
1474async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1475    // Scheduling a server task is not a readiness barrier. Authenticate once
1476    // against the mandatory IPv4 listener before exposing discovery to peers.
1477    // This uses the normal probe deadline, with no retries or widened budget.
1478    if let Some(endpoint) = info.endpoints.first() {
1479        if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1480            return Ok(());
1481        }
1482    }
1483    Err(BootstrapError::StartupNotServing)
1484}
1485
1486/// Run blocking filesystem work (an fsync, a rename) on tokio's blocking-thread
1487/// pool rather than on the async worker thread that awaits it. On a slow or
1488/// busy disk an fsync can take seconds, and a runtime has few worker threads
1489/// (an in-process test daemon may share two with everything else), so doing it
1490/// inline stalls unrelated tasks for that long.
1491///
1492/// `guards` (lock handles) move into the blocking task and are handed back
1493/// with the work's result. If the awaiting future is dropped first, the work
1494/// still runs to completion on its thread and the guards are released only
1495/// after it ends, so a lock is never given up while the work it protects is
1496/// still writing.
1497///
1498/// `Err` means the task panicked, or was cancelled before it started because
1499/// the runtime is shutting down; the guards have then been released already.
1500async fn run_blocking_holding<G, T>(
1501    guards: G,
1502    work: impl FnOnce() -> T + Send + 'static,
1503) -> Result<(T, G), JoinError>
1504where
1505    G: Send + 'static,
1506    T: Send + 'static,
1507{
1508    tokio::task::spawn_blocking(move || {
1509        let output = work();
1510        (output, guards)
1511    })
1512    .await
1513}
1514
1515/// Explain, for a startup error, why blocking work produced no result.
1516fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1517    if join_error.is_panic() {
1518        format!("the background thread {work} panicked before finishing: {join_error}")
1519    } else {
1520        format!(
1521            "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1522        )
1523    }
1524}
1525
1526/// Publish the connection file: owner-only permissions, written to a temporary
1527/// file, fsynced and renamed over `path` (all done by `write_atomic`). The
1528/// write runs on tokio's blocking pool through [`run_blocking_holding`], so the
1529/// fsync never holds an async worker thread.
1530///
1531/// `guard` is the singleton start lock (or whatever the caller must keep held
1532/// across publication); it is returned with the result. It comes back `None`
1533/// only when the write task panicked or was cancelled, in which case the
1534/// result is an error and the file must be treated as not published.
1535///
1536/// On Windows, `copy_path` asks for the profile copy as well. Its start lock is
1537/// taken before the primary write. Once `path` is written, an error returned
1538/// while preparing or writing the copy only logs a warning, and the result is
1539/// still success. The two writes share one blocking task but aren't one atomic
1540/// step, and a panic or cancellation of that task is still a publication
1541/// error.
1542async fn publish_connection_file<G: Send + 'static>(
1543    path: &Path,
1544    #[cfg(windows)] copy_path: Option<&Path>,
1545    info: &ConnectionInfo,
1546    local_endpoint: Option<&LocalEndpoint>,
1547    guard: G,
1548) -> (Result<(), BootstrapError>, Option<G>) {
1549    let task_path = path.to_path_buf();
1550    let task_info = info.clone();
1551    let task_endpoint = local_endpoint.cloned();
1552    #[cfg(windows)]
1553    let copy = windows_connection_copy::claim_for_publication(path, copy_path, info).await;
1554    let outcome = run_blocking_holding(guard, move || {
1555        write_connection_file(&task_path, &task_info, task_endpoint.as_ref())?;
1556        #[cfg(windows)]
1557        if let Some((copy_path, _lock)) = copy {
1558            if let Err(error) =
1559                write_connection_file(&copy_path, &task_info, task_endpoint.as_ref())
1560            {
1561                windows_connection_copy::warn_unavailable(&task_path, Some(&copy_path), &error);
1562            }
1563        }
1564        Ok(())
1565    })
1566    .await;
1567    match outcome {
1568        Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1569        Ok((Err(source), guard)) => (
1570            Err(BootstrapError::ConnectionFileWrite {
1571                path: path.to_path_buf(),
1572                source,
1573            }),
1574            Some(guard),
1575        ),
1576        Err(join_error) => (
1577            Err(BootstrapError::ConnectionFileWrite {
1578                path: path.to_path_buf(),
1579                source: ConnectionFileError::Io {
1580                    op: "write",
1581                    path: path.to_path_buf(),
1582                    source: io::Error::other(describe_blocking_task_failure(
1583                        "writing the connection file",
1584                        &join_error,
1585                    )),
1586                },
1587            }),
1588            None,
1589        ),
1590    }
1591}
1592
1593/// The synchronous connection-file write, including its fsync. Call it only
1594/// through [`publish_connection_file`], which keeps it off the async workers.
1595///
1596/// `local_endpoint` is written through subc-transport's endpoint-aware writer,
1597/// so the file carries `local_endpoint` when there is a local listener and is
1598/// byte-identical to before when there is none.
1599fn write_connection_file(
1600    path: &Path,
1601    info: &ConnectionInfo,
1602    local_endpoint: Option<&LocalEndpoint>,
1603) -> Result<(), ConnectionFileError> {
1604    #[cfg(test)]
1605    connection_file_write_hook::run(path);
1606    write_atomic_with_local_endpoint(path, info, local_endpoint)
1607}
1608
1609/// Lets a test run code inside the connection-file write, on whichever thread
1610/// performs it, for example to block there and check what else can still run.
1611#[cfg(test)]
1612mod connection_file_write_hook {
1613    use std::{
1614        path::{Path, PathBuf},
1615        sync::{Mutex, PoisonError},
1616    };
1617
1618    type Hook = Box<dyn FnOnce() + Send>;
1619
1620    /// Keyed by connection-file path, so a hook fires only for the test that
1621    /// installed it while other tests publish their own files in parallel.
1622    static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
1623
1624    /// Run `hook` once, at the start of the next write of `path`.
1625    pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
1626        HOOKS
1627            .lock()
1628            .unwrap_or_else(PoisonError::into_inner)
1629            .push((path.to_path_buf(), Box::new(hook)));
1630    }
1631
1632    pub(super) fn run(path: &Path) {
1633        let hook = {
1634            let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
1635            hooks
1636                .iter()
1637                .position(|(hooked, _)| hooked == path)
1638                .map(|index| hooks.swap_remove(index).1)
1639        };
1640        if let Some(hook) = hook {
1641            hook();
1642        }
1643    }
1644}
1645
1646/// A path-scoped barrier after publication and before any configured activation.
1647/// The async wait leaves the listeners free to answer discovery-time requests.
1648#[cfg(test)]
1649mod startup_publication_hook {
1650    use std::{
1651        path::{Path, PathBuf},
1652        sync::{Mutex, PoisonError},
1653    };
1654    use tokio::sync::oneshot;
1655
1656    type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
1657    static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
1658
1659    pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
1660        let (entered_tx, entered_rx) = oneshot::channel();
1661        let (release_tx, release_rx) = oneshot::channel();
1662        BARRIERS
1663            .lock()
1664            .unwrap_or_else(PoisonError::into_inner)
1665            .push((path.to_path_buf(), (entered_tx, release_rx)));
1666        (entered_rx, release_tx)
1667    }
1668
1669    pub(super) async fn run(path: &Path) {
1670        let barrier = {
1671            let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
1672            barriers
1673                .iter()
1674                .position(|(hooked, _)| hooked == path)
1675                .map(|index| barriers.swap_remove(index).1)
1676        };
1677        if let Some((entered, release)) = barrier {
1678            let _ = entered.send(());
1679            let _ = release.await;
1680        }
1681    }
1682}
1683
1684fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
1685    match err {
1686        ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
1687        ConnectionFileError::JsonRead { .. }
1688        | ConnectionFileError::UnsupportedSchema { .. }
1689        | ConnectionFileError::Invalid { .. }
1690        | ConnectionFileError::KeyTooShort { .. }
1691        // A live daemon always publishes the file owner-only (0600), so a file
1692        // with insecure permissions is never a daemon we should defer to: treat it
1693        // as stale and take over (which republishes a correct 0600 file).
1694        | ConnectionFileError::InsecurePermissions { .. } => true,
1695        ConnectionFileError::MissingParent { .. }
1696        | ConnectionFileError::MissingFileName { .. }
1697        // A writable ancestor is an operator misconfiguration, never evidence
1698        // about whether a daemon is live. Reclaiming the file would republish key
1699        // material into the same directory the refusal is about.
1700        | ConnectionFileError::InsecureParentDirectory { .. }
1701        | ConnectionFileError::Io { .. }
1702        | ConnectionFileError::JsonWrite { .. }
1703        | ConnectionFileError::Random(_)
1704        // A wire mismatch may identify a newer live daemon, so never reclaim its
1705        // connection file merely because this binary cannot speak its envelope.
1706        | ConnectionFileError::WireVersionMismatch { .. } => false,
1707    }
1708}
1709
1710fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
1711    match fs::remove_file(path) {
1712        Ok(()) => Ok(()),
1713        Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
1714        Err(source) => Err(BootstrapError::RemoveStale {
1715            path: path.to_path_buf(),
1716            source,
1717        }),
1718    }
1719}
1720
1721async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
1722    bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
1723}
1724
1725async fn bind_loopback_with<F, Fut>(
1726    port: u16,
1727    mut bind_v6: F,
1728) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
1729where
1730    F: FnMut(u16) -> Fut,
1731    Fut: std::future::Future<Output = io::Result<TcpListener>>,
1732{
1733    let v4_host = Ipv4Addr::LOCALHOST;
1734    let v4 = TcpListener::bind((v4_host, port))
1735        .await
1736        .map_err(|source| BootstrapError::Bind {
1737            host: v4_host.to_string(),
1738            port,
1739            source,
1740        })?;
1741    let actual_port = v4
1742        .local_addr()
1743        .map_err(|source| BootstrapError::LocalAddr {
1744            host: v4_host.to_string(),
1745            source,
1746        })?
1747        .port();
1748
1749    let mut listeners = vec![v4];
1750    let mut endpoints = vec![Endpoint {
1751        host: v4_host.to_string(),
1752        port: actual_port,
1753    }];
1754
1755    let v6_host = Ipv6Addr::LOCALHOST;
1756    match bind_v6(actual_port).await {
1757        Ok(v6) => {
1758            listeners.push(v6);
1759            endpoints.push(Endpoint {
1760                host: v6_host.to_string(),
1761                port: actual_port,
1762            });
1763        }
1764        Err(err)
1765            if ipv6_loopback_unavailable(&err)
1766                || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
1767        {
1768            warn!(
1769                port = actual_port,
1770                error = %err,
1771                "IPv6 loopback unavailable; serving only IPv4 loopback"
1772            );
1773        }
1774        Err(source) => {
1775            drop(listeners);
1776            return Err(BootstrapError::Bind {
1777                host: v6_host.to_string(),
1778                port: actual_port,
1779                source,
1780            });
1781        }
1782    }
1783
1784    Ok((listeners, endpoints))
1785}
1786
1787fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
1788    matches!(
1789        err.kind(),
1790        io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
1791    ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
1792}
1793
1794struct AbortOnDrop<T> {
1795    handle: JoinHandle<T>,
1796}
1797
1798impl<T> AbortOnDrop<T> {
1799    fn new(handle: JoinHandle<T>) -> Self {
1800        Self { handle }
1801    }
1802
1803    async fn join(&mut self) -> Result<T, JoinError> {
1804        (&mut self.handle).await
1805    }
1806}
1807
1808impl<T> Drop for AbortOnDrop<T> {
1809    fn drop(&mut self) {
1810        if !self.handle.is_finished() {
1811            self.handle.abort();
1812        }
1813    }
1814}
1815
1816#[derive(Debug)]
1817struct StartLock {
1818    // Keep the locked file handle alive for the duration of bootstrap.
1819    _file: fs::File,
1820}
1821
1822impl Drop for StartLock {
1823    fn drop(&mut self) {
1824        // Closing alone leaves the lock held if a concurrent fork inherited
1825        // the open file description and has not exec'd yet. Unlock explicitly
1826        // so a restart can reclaim the singleton as soon as bootstrap ends.
1827        if let Err(error) = FileExt::unlock(&self._file) {
1828            tracing::warn!(%error, "failed to release the singleton start lock");
1829        }
1830    }
1831}
1832
1833impl StartLock {
1834    async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
1835        let path = start_lock_path(connection_file_path);
1836        for _ in 0..START_LOCK_RETRIES {
1837            let file = match open_owner_only_lock(&path) {
1838                Ok(file) => file,
1839                Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
1840            };
1841            match FileExt::try_lock(&file) {
1842                Ok(()) => return Ok(Self { _file: file }),
1843                Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
1844                Err(TryLockError::Error(source)) => {
1845                    return Err(BootstrapError::StartLockCreate { path, source });
1846                }
1847            }
1848        }
1849
1850        Err(BootstrapError::StartLockBusy {
1851            path,
1852            attempts: START_LOCK_RETRIES,
1853        })
1854    }
1855}
1856
1857pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
1858    let mut options = fs::OpenOptions::new();
1859    options.read(true).write(true).create(true);
1860    #[cfg(unix)]
1861    {
1862        use std::os::unix::fs::OpenOptionsExt;
1863        options.mode(0o600);
1864    }
1865    options.open(path)
1866}
1867
1868fn start_lock_path(connection_file_path: &Path) -> PathBuf {
1869    let file_name = connection_file_path
1870        .file_name()
1871        .map(|name| name.to_string_lossy())
1872        .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
1873    let lock_name = format!("{file_name}.start-lock");
1874    connection_file_path
1875        .parent()
1876        .filter(|parent| !parent.as_os_str().is_empty())
1877        .unwrap_or_else(|| Path::new("."))
1878        .join(lock_name)
1879}
1880
1881fn non_empty_os_var(key: &str) -> Option<OsString> {
1882    let value = env::var_os(key)?;
1883    if value.is_empty() {
1884        None
1885    } else {
1886        Some(value)
1887    }
1888}
1889
1890/// Bootstrap-layer errors are deliberately typed so startup never panics for
1891/// ordinary daemon-discovery races or stale filesystem state.
1892#[derive(Debug)]
1893pub enum BootstrapError {
1894    /// The bound server could not authenticate its own readiness probe.
1895    StartupNotServing,
1896    #[cfg(unix)]
1897    Signal(io::Error),
1898    InvalidPort {
1899        raw: String,
1900        source: std::num::ParseIntError,
1901    },
1902    ConnectionFileRead {
1903        path: PathBuf,
1904        source: ConnectionFileError,
1905    },
1906    ConnectionFileWrite {
1907        path: PathBuf,
1908        source: ConnectionFileError,
1909    },
1910    GenerateConnectionFile(ConnectionFileError),
1911    StartLockCreate {
1912        path: PathBuf,
1913        source: io::Error,
1914    },
1915    StartLockBusy {
1916        path: PathBuf,
1917        attempts: usize,
1918    },
1919    /// The run-directory lock file could not be created, opened or locked.
1920    RunDirLockCreate {
1921        path: PathBuf,
1922        source: io::Error,
1923    },
1924    /// Another live process, almost certainly a daemon started with a
1925    /// different runtime directory over the same data home, owns the run
1926    /// directory. The daemon does not start. `holder_pid` is read from the
1927    /// lock file when possible and is informational only.
1928    RunDirBusy {
1929        path: PathBuf,
1930        holder_pid: Option<u32>,
1931    },
1932    RemoveStale {
1933        path: PathBuf,
1934        source: io::Error,
1935    },
1936    Bind {
1937        host: String,
1938        port: u16,
1939        source: io::Error,
1940    },
1941    LocalAddr {
1942        host: String,
1943        source: io::Error,
1944    },
1945    DaemonConfig(DaemonConfigError),
1946    /// The machine id could not be established: its file is corrupt, unreadable
1947    /// or unwritable, or the data home is relative. The daemon does not start.
1948    MachineId(crate::machine_id::MachineIdFileError),
1949    /// The daemon run directory could not be resolved because the data home is
1950    /// relative. The daemon does not start.
1951    RunDir(daemon_config::DaemonRunDirError),
1952    Serve(ServerError),
1953    ServeJoin(tokio::task::JoinError),
1954}
1955
1956impl From<ServerError> for BootstrapError {
1957    fn from(error: ServerError) -> Self {
1958        Self::Serve(error)
1959    }
1960}
1961
1962impl fmt::Display for BootstrapError {
1963    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1964        match self {
1965            Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
1966            #[cfg(unix)]
1967            Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
1968            Self::InvalidPort { raw, source } => {
1969                write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
1970            }
1971            Self::ConnectionFileRead { path, source } => write!(
1972                f,
1973                "failed to read connection file {}: {source}",
1974                path.display()
1975            ),
1976            Self::ConnectionFileWrite { path, source } => write!(
1977                f,
1978                "failed to publish connection file {}: {source}",
1979                path.display()
1980            ),
1981            Self::GenerateConnectionFile(err) => {
1982                write!(f, "failed to generate connection-file auth material: {err}")
1983            }
1984            Self::StartLockCreate { path, source } => {
1985                write!(
1986                    f,
1987                    "failed to create start lock {}: {source}",
1988                    path.display()
1989                )
1990            }
1991            Self::StartLockBusy { path, attempts } => write!(
1992                f,
1993                "start lock {} remained busy after {attempts} attempts",
1994                path.display()
1995            ),
1996            Self::RunDirLockCreate { path, source } => write!(
1997                f,
1998                "refusing to start: failed to lock run directory via {}: {source}",
1999                path.display()
2000            ),
2001            Self::RunDirBusy { path, holder_pid } => {
2002                write!(
2003                    f,
2004                    "refusing to start: run directory lock {} is held by another daemon",
2005                    path.display()
2006                )?;
2007                if let Some(pid) = holder_pid {
2008                    write!(f, " (pid {pid})")?;
2009                }
2010                write!(
2011                    f,
2012                    "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
2013                )
2014            }
2015            Self::RemoveStale { path, source } => write!(
2016                f,
2017                "failed to remove stale connection file {}: {source}",
2018                path.display()
2019            ),
2020            Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
2021                write!(
2022                    f,
2023                    "port {port} in use on loopback {host}: {source}; set the port in config"
2024                )
2025            }
2026            Self::Bind { host, port, source } => {
2027                write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
2028            }
2029            Self::LocalAddr { host, source } => {
2030                write!(f, "failed to read local address for {host}: {source}")
2031            }
2032            Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
2033            Self::MachineId(err) => write!(f, "refusing to start: {err}"),
2034            Self::RunDir(err) => write!(f, "refusing to start: {err}"),
2035            Self::Serve(err) => write!(f, "daemon server failed: {err}"),
2036            Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
2037        }
2038    }
2039}
2040
2041impl Error for BootstrapError {
2042    fn source(&self) -> Option<&(dyn Error + 'static)> {
2043        match self {
2044            Self::StartupNotServing => None,
2045            #[cfg(unix)]
2046            Self::Signal(source) => Some(source),
2047            Self::InvalidPort { source, .. } => Some(source),
2048            Self::ConnectionFileRead { source, .. }
2049            | Self::ConnectionFileWrite { source, .. }
2050            | Self::GenerateConnectionFile(source) => Some(source),
2051            Self::StartLockCreate { source, .. }
2052            | Self::RunDirLockCreate { source, .. }
2053            | Self::RemoveStale { source, .. }
2054            | Self::Bind { source, .. }
2055            | Self::LocalAddr { source, .. } => Some(source),
2056            Self::DaemonConfig(err) => Some(err),
2057            Self::MachineId(err) => Some(err),
2058            Self::RunDir(err) => Some(err),
2059            Self::Serve(err) => Some(err),
2060            Self::ServeJoin(err) => Some(err),
2061            Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
2062        }
2063    }
2064}
2065
2066#[cfg(test)]
2067mod tests {
2068    mod activity {
2069        include!("activity_bootstrap_tests.rs");
2070    }
2071    mod startup_order {
2072        include!("startup_order_tests.rs");
2073    }
2074    use super::*;
2075    use crate::server::ServerAuth;
2076    use cortexkit_test_support::ScratchDir;
2077    #[cfg(target_os = "linux")]
2078    use std::collections::BTreeSet;
2079    use std::sync::Mutex;
2080    #[cfg(target_os = "linux")]
2081    use subc_control::ModuleProtocol;
2082    use subc_transport::{write_atomic, MIN_KEY_LEN};
2083    use tokio::io::AsyncReadExt;
2084    use tokio::task::JoinHandle;
2085
2086    #[cfg(unix)]
2087    use std::os::unix::fs::PermissionsExt;
2088
2089    static ENV_LOCK: Mutex<()> = Mutex::new(());
2090
2091    #[test]
2092    fn normalized_build_provenance_preserves_real_values() {
2093        assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
2094    }
2095
2096    #[test]
2097    fn normalized_build_provenance_omits_unavailable_and_empty_values() {
2098        assert_eq!(normalized_build_provenance("unavailable"), None);
2099        assert_eq!(normalized_build_provenance(""), None);
2100    }
2101
2102    #[cfg(target_os = "linux")]
2103    fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
2104        let cgroups = fs::read_to_string("/proc/self/cgroup")?;
2105        let relative = cgroups
2106            .lines()
2107            .find_map(|line| line.strip_prefix("0::"))
2108            .ok_or_else(|| {
2109                io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
2110            })?;
2111        Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
2112    }
2113
2114    #[cfg(target_os = "linux")]
2115    fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
2116        let modules = current_cgroup_path_for_test()?.join("subc-modules");
2117        let entries = match fs::read_dir(modules) {
2118            Ok(entries) => entries,
2119            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
2120            Err(error) => return Err(error),
2121        };
2122        let mut directories = BTreeSet::new();
2123        for entry in entries {
2124            let entry = entry?;
2125            if entry.file_type()?.is_dir() {
2126                directories.insert(entry.file_name());
2127            }
2128        }
2129        Ok(directories)
2130    }
2131
2132    #[cfg(target_os = "linux")]
2133    async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
2134        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2135        while !path.exists() && tokio::time::Instant::now() < deadline {
2136            assert!(
2137                !task.is_finished(),
2138                "daemon exited before creating {}",
2139                path.display()
2140            );
2141            sleep(Duration::from_millis(10)).await;
2142        }
2143        assert!(path.exists(), "daemon did not create {}", path.display());
2144    }
2145
2146    /// An in-process daemon with the default (disabled) cgroup placement must
2147    /// leave the host's live module cgroup tree exactly as it found it.
2148    /// Red-checking this test (making it fail) creates a
2149    /// `cgroup-isolation-probe-*` directory in the live cgroup tree, which must be removed by hand.
2150    #[cfg(target_os = "linux")]
2151    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2152    async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
2153        let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
2154        let module_id = format!("cgroup-isolation-probe-{}", process::id());
2155        let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
2156        assert!(
2157            !before.contains(&OsString::from(&module_id)),
2158            "isolation probe cgroup already exists before this daemon starts"
2159        );
2160        let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
2161        let module = ConfiguredModule {
2162            carry_containment: false,
2163            module_id,
2164            program: PathBuf::from("sh"),
2165            args: vec!["-c".to_string(), "sleep 30".to_string()],
2166            env: Vec::new(),
2167            log: None,
2168            enabled: true,
2169            reserved: false,
2170            reserved_prefixes: Vec::new(),
2171            protocol: ModuleProtocol::None,
2172            overlap: Default::default(),
2173            health: HealthConfig::default(),
2174            drain_timeout_ms: None,
2175            register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2176            route_bind_relay_timeout_ms: None,
2177            restart: RestartPolicy::default(),
2178        };
2179        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2180            .with_configured_modules([module])
2181            .with_capture_logs_dir(temp.join("logs"))
2182            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2183        let task = tokio::spawn(run_with_config(config));
2184
2185        wait_for_path(&capture, &task).await;
2186        let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2187
2188        task.abort();
2189        assert!(task
2190            .await
2191            .expect_err("aborted daemon task must cancel")
2192            .is_cancelled());
2193        assert_eq!(
2194            before, after,
2195            "an in-process daemon must not create or reconcile ambient module cgroups"
2196        );
2197    }
2198
2199    #[cfg(target_os = "linux")]
2200    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2201    async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2202        let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2203        let cgroup_root = temp.join("cgroup");
2204        fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2205        fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2206        let modules = cgroup_root.join("subc-modules");
2207        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2208            .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2209            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2210        let task = tokio::spawn(run_with_config(config));
2211
2212        wait_for_path(&modules, &task).await;
2213
2214        task.abort();
2215        assert!(task
2216            .await
2217            .expect_err("aborted daemon task must cancel")
2218            .is_cancelled());
2219        assert!(
2220            fs::read_dir(&modules)
2221                .expect("read prepared modules directory")
2222                .next()
2223                .is_none(),
2224            "the delegation probe must clean up after itself"
2225        );
2226    }
2227
2228    struct EnvGuard {
2229        key: &'static str,
2230        previous: Option<OsString>,
2231    }
2232
2233    impl EnvGuard {
2234        fn set(key: &'static str, value: &Path) -> Self {
2235            let previous = env::var_os(key);
2236            env::set_var(key, value);
2237            Self { key, previous }
2238        }
2239
2240        fn set_str(key: &'static str, value: &str) -> Self {
2241            let previous = env::var_os(key);
2242            env::set_var(key, value);
2243            Self { key, previous }
2244        }
2245
2246        fn unset(key: &'static str) -> Self {
2247            let previous = env::var_os(key);
2248            env::remove_var(key);
2249            Self { key, previous }
2250        }
2251    }
2252
2253    impl Drop for EnvGuard {
2254        fn drop(&mut self) {
2255            match &self.previous {
2256                Some(value) => env::set_var(self.key, value),
2257                None => env::remove_var(self.key),
2258            }
2259        }
2260    }
2261
2262    fn unique_temp_dir(name: &str) -> ScratchDir {
2263        ScratchDir::new(name)
2264    }
2265
2266    fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2267        let dir = unique_temp_dir(name);
2268        let path = dir.join("conn.json");
2269        (dir, path)
2270    }
2271
2272    fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2273        ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2274    }
2275
2276    fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2277        let auth = auth_for(&bound.connection_info);
2278        tokio::spawn(serve_listeners(
2279            bound.listeners,
2280            Arc::new(Router::with_default_self_handler()),
2281            auth,
2282        ))
2283    }
2284
2285    fn expect_bound(outcome: Outcome) -> BoundDaemon {
2286        match outcome {
2287            Outcome::Bound(bound) => bound,
2288            Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2289        }
2290    }
2291
2292    async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2293        let endpoint = conn
2294            .endpoints
2295            .first()
2296            .expect("test connection file should have an endpoint");
2297        let ip: IpAddr = endpoint.host.parse().unwrap();
2298        TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2299    }
2300
2301    fn make_connection_info(port: u16) -> ConnectionInfo {
2302        ConnectionInfo {
2303            schema: SCHEMA_VERSION,
2304            wire_version: Some(PROTOCOL_VERSION),
2305            endpoints: vec![Endpoint {
2306                host: "127.0.0.1".to_owned(),
2307                port,
2308            }],
2309            key: generate_key().unwrap(),
2310            daemon_id: generate_daemon_id().unwrap(),
2311            pid: process::id(),
2312            daemon_ver: "test-subc".to_owned(),
2313        }
2314    }
2315
2316    fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2317        fs::write(path, contents).unwrap();
2318        #[cfg(unix)]
2319        fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2320    }
2321
2322    fn assert_owner_only_connection_file(path: &Path) {
2323        // `path` is only inspected on Unix (mode bits); on Windows the owner-only
2324        // guarantee comes from the inherited %TEMP% ACL, nothing to assert here.
2325        #[cfg(unix)]
2326        {
2327            let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2328            assert_eq!(mode, 0o600);
2329        }
2330        #[cfg(not(unix))]
2331        let _ = path;
2332    }
2333
2334    #[test]
2335    fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2336        let _env_lock = ENV_LOCK.lock().unwrap();
2337        let runtime_dir = unique_temp_dir("xdg-runtime");
2338        let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2339
2340        assert_eq!(
2341            connection_file_path(),
2342            runtime_dir.join(CONNECTION_FILE_NAME)
2343        );
2344    }
2345
2346    #[test]
2347    fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2348        let runtime_dir = OsString::from("/run/user/1000");
2349
2350        let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2351
2352        assert_eq!(
2353            path,
2354            PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2355        );
2356        assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2357    }
2358
2359    #[test]
2360    fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2361        let _env_lock = ENV_LOCK.lock().unwrap();
2362        let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2363
2364        assert_eq!(
2365            connection_file_path(),
2366            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2367        );
2368    }
2369
2370    #[test]
2371    fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2372        let (path, source) = connection_file_path_with_source(None);
2373
2374        assert_eq!(
2375            path,
2376            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2377        );
2378        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2379    }
2380
2381    /// Concurrent callers must derive one token. The former temp-file uid probe
2382    /// could fail transiently (same-tick name collision, fd exhaustion) and send
2383    /// the loser down the env-derived fallback with a different identity for the
2384    /// same user; this fence keeps identity independent of filesystem luck.
2385    #[test]
2386    fn user_connection_token_is_stable_under_concurrent_callers() {
2387        let expected = user_connection_token();
2388        let workers: Vec<_> = (0..32)
2389            .map(|_| {
2390                std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2391            })
2392            .collect();
2393        for worker in workers {
2394            for token in worker.join().expect("probe thread") {
2395                assert_eq!(token, expected, "token diverged under concurrent probes");
2396            }
2397        }
2398    }
2399
2400    #[test]
2401    fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
2402        let (path, source) = connection_file_path_with_source(Some(OsString::new()));
2403
2404        assert_eq!(
2405            path,
2406            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2407        );
2408        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2409    }
2410
2411    /// The shipped binary is the only caller that journals terminal exits into
2412    /// the real run directory, so its constructor must supply that path itself;
2413    /// the in-process default leaves it unset.
2414    #[test]
2415    fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
2416        let _env_lock = ENV_LOCK.lock().unwrap();
2417        let root = unique_temp_dir("daemon-binary-config");
2418        let data_home = root.join("data");
2419        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
2420        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2421        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2422
2423        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
2424
2425        let run_dir = data_home.join("cortexkit").join("run");
2426        assert_eq!(
2427            config.terminal_journal_path,
2428            Some(run_dir.join("terminals.jsonl"))
2429        );
2430        assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
2431        assert_eq!(
2432            BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
2433            None,
2434            "an in-process config must not journal anywhere unless asked to"
2435        );
2436    }
2437
2438    #[test]
2439    fn daemon_binary_config_refuses_a_relative_data_home() {
2440        let _env_lock = ENV_LOCK.lock().unwrap();
2441        let root = unique_temp_dir("daemon-binary-relative-data");
2442        let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
2443        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2444        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2445
2446        let error = BootstrapConfig::from_env_for_daemon_binary()
2447            .expect_err("a relative data home must refuse the daemon binary's config");
2448        assert!(
2449            matches!(error, BootstrapError::RunDir(_)),
2450            "expected a run-directory refusal, got {error}"
2451        );
2452        assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
2453    }
2454
2455    #[test]
2456    fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
2457        let _env_lock = ENV_LOCK.lock().unwrap();
2458        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2459        let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
2460        let config_path = conn_path.with_file_name("subc.jsonc");
2461        for (doc, expected) in [
2462            (None, daemon_config::PrivilegedOperatorPrompt::Log),
2463            (
2464                Some(r#"{"version":1}"#),
2465                daemon_config::PrivilegedOperatorPrompt::Log,
2466            ),
2467            (
2468                Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
2469                daemon_config::PrivilegedOperatorPrompt::Log,
2470            ),
2471            (
2472                Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
2473                daemon_config::PrivilegedOperatorPrompt::Enforce,
2474            ),
2475        ] {
2476            if let Some(doc) = doc {
2477                fs::write(&config_path, doc).unwrap();
2478            }
2479            assert_eq!(
2480                BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2481                    .unwrap()
2482                    .privileged_operator_prompt,
2483                expected
2484            );
2485            assert_eq!(
2486                BootstrapConfig::new(&conn_path, 0)
2487                    .with_daemon_config_path(&config_path)
2488                    .unwrap()
2489                    .privileged_operator_prompt,
2490                expected
2491            );
2492        }
2493    }
2494
2495    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2496    async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
2497        let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
2498        let connection_path = dir.join("connection.json");
2499        let config_path = dir.join("subc.jsonc");
2500        fs::write(
2501            &config_path,
2502            r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
2503        )
2504        .unwrap();
2505        let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
2506            .with_daemon_config_path(&config_path)
2507            .unwrap();
2508        let connection_path = config.connection_file_path.clone();
2509        let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
2510        tokio::time::timeout(Duration::from_secs(5), async {
2511            while !connection_path.exists() {
2512                assert!(
2513                    !daemon.handle.is_finished(),
2514                    "daemon exited before publishing"
2515                );
2516                tokio::task::yield_now().await;
2517            }
2518        })
2519        .await
2520        .unwrap();
2521        let info = connection_file::read(&connection_path).unwrap();
2522        let mut client = connect_from_info(&info).await.unwrap();
2523        authenticate_client(&mut client, &info, Duration::from_secs(5))
2524            .await
2525            .unwrap();
2526        for (correlation, mode, expected) in [
2527            (1, "enforce", Vec::new()),
2528            (2, "log", vec!["privileged_operator_prompt"]),
2529            (3, "log", vec!["privileged_operator_prompt"]),
2530        ] {
2531            fs::write(
2532                &config_path,
2533                serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
2534            )
2535            .unwrap();
2536            let frame = crate::Frame::build(
2537                subc_protocol::FrameType::Request,
2538                subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
2539                0,
2540                0,
2541                correlation,
2542                serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
2543                    preview: true,
2544                })
2545                .unwrap(),
2546            )
2547            .unwrap();
2548            crate::write_frame(&mut client, &frame).await.unwrap();
2549            let reply =
2550                tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
2551                    .await
2552                    .unwrap()
2553                    .unwrap()
2554                    .unwrap();
2555            assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
2556            assert_eq!(reply.header.corr, correlation);
2557            let subc_control::ClientControlResponse::SupervisorRescan { result } =
2558                serde_json::from_slice(&reply.body).unwrap()
2559            else {
2560                panic!("expected a rescan response");
2561            };
2562            assert_eq!(result.restart_required, expected);
2563        }
2564    }
2565
2566    #[test]
2567    fn configured_port_uses_default_config_and_env_override() {
2568        let _env_lock = ENV_LOCK.lock().unwrap();
2569        let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
2570        let config_path = conn_path.with_file_name("subc.jsonc");
2571
2572        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2573        assert_eq!(
2574            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2575                .unwrap()
2576                .port,
2577            DEFAULT_SUBC_PORT
2578        );
2579
2580        fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
2581        assert_eq!(
2582            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2583                .unwrap()
2584                .port,
2585            8123
2586        );
2587
2588        let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
2589        assert_eq!(
2590            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2591                .unwrap()
2592                .port,
2593            9012
2594        );
2595    }
2596
2597    #[tokio::test]
2598    async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
2599        let (_dir, path) = temp_connection_file_path("departing-peer");
2600        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2601        let info = make_connection_info(listener.local_addr().unwrap().port());
2602        write_connection_file(&path, &info, None).unwrap();
2603        let peer = tokio::spawn(async move {
2604            let (mut stream, _) = listener.accept().await.unwrap();
2605            subc_transport::authenticate_server(
2606                &mut stream,
2607                &info.key,
2608                &info.daemon_id,
2609                &info.daemon_ver,
2610                PROBE_AUTH_DEADLINE,
2611            )
2612            .await
2613            .unwrap();
2614        });
2615        let outcome = ensure_singleton(&path, 0).await.unwrap();
2616        peer.await.unwrap();
2617        assert!(
2618            matches!(outcome, Outcome::Bound(_)),
2619            "a departing peer is not AlreadyRunning"
2620        );
2621    }
2622
2623    #[tokio::test]
2624    async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
2625        let (_dir, path) = temp_connection_file_path("already-running");
2626
2627        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2628        let server = start_server(bound);
2629
2630        let second = ensure_singleton(&path, 0).await.unwrap();
2631        assert!(matches!(second, Outcome::AlreadyRunning));
2632
2633        server.abort();
2634        let _ = server.await;
2635    }
2636
2637    #[tokio::test]
2638    async fn daemon_connection_file_publishes_protocol_wire_version() {
2639        let (_dir, path) = temp_connection_file_path("wire-version");
2640        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2641        assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
2642        assert_eq!(
2643            connection_file::read(&path).unwrap().wire_version,
2644            Some(PROTOCOL_VERSION)
2645        );
2646
2647        drop(bound.listeners);
2648    }
2649
2650    #[tokio::test]
2651    async fn stale_unbound_connection_file_is_reclaimed() {
2652        let (_dir, path) = temp_connection_file_path("stale-reclaim");
2653        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2654        let stale_port = stale.local_addr().unwrap().port();
2655        drop(stale);
2656        let stale_info = make_connection_info(stale_port);
2657        write_atomic(&path, &stale_info).unwrap();
2658
2659        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2660        assert_ne!(bound.connection_info.key, stale_info.key);
2661        drop(bound.listeners);
2662    }
2663
2664    #[cfg(unix)]
2665    #[tokio::test]
2666    async fn ensure_singleton_reclaims_insecure_connection_file() {
2667        let (_dir, path) = temp_connection_file_path("insecure-reclaim");
2668        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2669        let stale_port = stale.local_addr().unwrap().port();
2670        drop(stale);
2671        let stale_info = make_connection_info(stale_port);
2672        write_atomic(&path, &stale_info).unwrap();
2673        fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
2674
2675        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2676        assert_ne!(bound.connection_info.key, stale_info.key);
2677        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2678        assert_owner_only_connection_file(&path);
2679
2680        drop(bound.listeners);
2681    }
2682
2683    #[tokio::test]
2684    async fn ensure_singleton_reclaims_non_loopback_connection_file() {
2685        let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
2686        let mut stale_info = make_connection_info(8757);
2687        stale_info.endpoints = vec![Endpoint {
2688            host: "192.0.2.10".to_owned(),
2689            port: 8757,
2690        }];
2691        write_atomic(&path, &stale_info).unwrap();
2692
2693        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2694        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
2695        assert_ne!(bound.connection_info.key, stale_info.key);
2696        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2697        assert!(bound
2698            .connection_info
2699            .endpoints
2700            .iter()
2701            .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
2702        assert_owner_only_connection_file(&path);
2703
2704        drop(bound.listeners);
2705    }
2706
2707    #[tokio::test]
2708    async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
2709        let mut unsupported_schema = make_connection_info(8757);
2710        unsupported_schema.schema = SCHEMA_VERSION + 1;
2711
2712        let mut empty_endpoints = make_connection_info(8757);
2713        empty_endpoints.endpoints.clear();
2714
2715        let mut short_key = make_connection_info(8757);
2716        short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
2717
2718        let cases = vec![
2719            (
2720                "unsupported-schema",
2721                serde_json::to_vec(&unsupported_schema).unwrap(),
2722                Some(unsupported_schema),
2723            ),
2724            (
2725                "empty-endpoints",
2726                serde_json::to_vec(&empty_endpoints).unwrap(),
2727                Some(empty_endpoints),
2728            ),
2729            (
2730                "short-key",
2731                serde_json::to_vec(&short_key).unwrap(),
2732                Some(short_key),
2733            ),
2734            ("invalid-json", b"{not valid connection json".to_vec(), None),
2735        ];
2736
2737        for (label, contents, old_info) in cases {
2738            let (_dir, path) = temp_connection_file_path(label);
2739            write_raw_owner_only_connection_file(&path, &contents);
2740
2741            let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2742            if let Some(old_info) = old_info {
2743                assert_ne!(bound.connection_info.key, old_info.key, "{label}");
2744                assert_ne!(
2745                    bound.connection_info.daemon_id, old_info.daemon_id,
2746                    "{label}"
2747                );
2748            }
2749            assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
2750            assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
2751            assert_owner_only_connection_file(&path);
2752
2753            drop(bound.listeners);
2754        }
2755    }
2756
2757    #[tokio::test]
2758    async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
2759        let (_dir, path) = temp_connection_file_path("foreign-reclaim");
2760        let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2761        let foreign_port = foreign.local_addr().unwrap().port();
2762        write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
2763        let foreign_task = tokio::spawn(async move {
2764            if let Ok((mut stream, _)) = foreign.accept().await {
2765                let mut buf = [0u8; 64];
2766                let _ = stream.read(&mut buf).await;
2767            }
2768        });
2769
2770        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2771        assert!(bound
2772            .connection_info
2773            .endpoints
2774            .iter()
2775            .all(|endpoint| endpoint.port != foreign_port));
2776
2777        drop(bound.listeners);
2778        let _ = foreign_task.await;
2779    }
2780
2781    #[tokio::test]
2782    async fn stale_start_lock_file_is_reclaimable() {
2783        let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
2784        let lock_path = start_lock_path(&path);
2785        drop(open_owner_only_lock(&lock_path).unwrap());
2786        assert!(lock_path.is_file());
2787
2788        let lock = StartLock::acquire(&path).await.unwrap();
2789        assert!(lock_path.is_file());
2790
2791        drop(lock);
2792        assert!(lock_path.is_file());
2793    }
2794
2795    #[tokio::test]
2796    async fn held_start_lock_blocks_second_acquire_until_release() {
2797        let (_dir, path) = temp_connection_file_path("start-lock-held");
2798        let lock_path = start_lock_path(&path);
2799        let first = StartLock::acquire(&path).await.unwrap();
2800
2801        let err = match StartLock::acquire(&path).await {
2802            Ok(_) => panic!("second acquire while held must stay busy"),
2803            Err(err) => err,
2804        };
2805        assert!(matches!(
2806            err,
2807            BootstrapError::StartLockBusy {
2808                ref path,
2809                attempts: START_LOCK_RETRIES,
2810            } if path == &lock_path
2811        ));
2812
2813        drop(first);
2814
2815        let second = StartLock::acquire(&path)
2816            .await
2817            .expect("released advisory lock should be reclaimable");
2818        drop(second);
2819    }
2820
2821    #[tokio::test]
2822    async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
2823        let (_dir, path) = temp_connection_file_path("bind-conflict");
2824        let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2825        let occupied_port = occupied.local_addr().unwrap().port();
2826
2827        let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
2828        assert!(matches!(
2829            err,
2830            BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
2831        ));
2832        assert!(err.to_string().contains("set the port in config"));
2833
2834        drop(occupied);
2835    }
2836
2837    #[tokio::test]
2838    async fn ephemeral_ipv6_collision_still_serves_ipv4() {
2839        let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
2840            let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
2841            let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
2842            drop(occupied);
2843            result
2844        })
2845        .await
2846        .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
2847        assert_eq!(endpoints.len(), 1);
2848        assert_eq!(endpoints[0].host, "127.0.0.1");
2849        TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
2850            .await
2851            .unwrap();
2852        drop(listeners);
2853    }
2854
2855    #[tokio::test]
2856    async fn unpublished_boot_holds_singleton_lock_until_publication() {
2857        let dir = ScratchDir::new("ul");
2858        let path = dir.join("c");
2859        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2860        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2861        assert!(!path.exists());
2862        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2863        assert!(
2864            matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
2865            "an unpublished boot must keep its singleton claim"
2866        );
2867        drop(bound);
2868        FileExt::try_lock(&contender).unwrap();
2869    }
2870
2871    #[tokio::test]
2872    async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
2873        let dir = ScratchDir::new("sl");
2874        let path = dir.join("c");
2875        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2876        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2877        // try_clone shares the open file description just as fork does,
2878        // without relying on the timing of another test's child spawn.
2879        let inherited = bound
2880            .publication_lock
2881            .as_ref()
2882            .unwrap()
2883            ._file
2884            .try_clone()
2885            .unwrap();
2886        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2887        assert!(matches!(
2888            FileExt::try_lock(&contender),
2889            Err(TryLockError::WouldBlock)
2890        ));
2891        drop(bound);
2892        FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
2893        drop(inherited);
2894    }
2895
2896    #[tokio::test]
2897    async fn readiness_probe_requires_an_authenticating_server() {
2898        let dir = ScratchDir::new("unserved-readiness");
2899        let bound = expect_bound(
2900            ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
2901                .await
2902                .unwrap(),
2903        );
2904        assert!(matches!(
2905            verify_serving(&bound.connection_info).await,
2906            Err(BootstrapError::StartupNotServing)
2907        ));
2908    }
2909
2910    #[cfg(unix)]
2911    #[tokio::test]
2912    async fn orphan_sweep_finishes_before_connection_file_publication() {
2913        use crate::live_children::{ExecutableIdentity, LiveChild};
2914        let dir = ScratchDir::new("pas");
2915        // A one-letter connection file name, and no long socket path: the sweep
2916        // this test covers runs before the local endpoint is bound, so a longer
2917        // name would only add a way for the test to fail on something else.
2918        let path = dir.join("c");
2919        let record = dir.join("live-children.json");
2920        let ready = dir.join("ready");
2921        let mut child = std::process::Command::new("/bin/sh")
2922            .args([
2923                "-c",
2924                "trap '' TERM; : > \"$0\"; read -r _",
2925                ready.to_str().unwrap(),
2926            ])
2927            .stdin(std::process::Stdio::piped())
2928            .env("XDG_DATA_HOME", dir.path())
2929            .env("XDG_RUNTIME_DIR", dir.path())
2930            .env("XDG_CONFIG_HOME", dir.path())
2931            .spawn()
2932            .unwrap();
2933        timeout(Duration::from_secs(5), async {
2934            while !ready.exists() {
2935                sleep(Duration::from_millis(5)).await;
2936            }
2937        })
2938        .await
2939        .unwrap();
2940        let observed = subc_os::Process::open(child.id())
2941            .unwrap()
2942            .unwrap()
2943            .observe()
2944            .unwrap();
2945        crate::live_children::write_record(
2946            &record,
2947            &[LiveChild {
2948                container: None,
2949                module_id: "old-sleep".into(),
2950                pid: child.id(),
2951                protocol: subc_control::ModuleProtocol::None,
2952                start_time: Some(observed.start_time),
2953                executable: observed.executable.map(ExecutableIdentity::from),
2954                cgroup_name: None,
2955            }],
2956        )
2957        .unwrap();
2958        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
2959            .with_live_children_record(&record);
2960        let daemon = tokio::spawn(run_with_config(config));
2961        timeout(Duration::from_secs(5), async {
2962            while !path.exists() {
2963                sleep(Duration::from_millis(5)).await;
2964            }
2965        })
2966        .await
2967        .unwrap();
2968        assert!(path.exists());
2969        let ended = child.try_wait().unwrap().is_some();
2970        if !ended {
2971            child.kill().unwrap();
2972        }
2973        child.wait().unwrap();
2974        if ended {
2975            assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
2976        }
2977        daemon.abort();
2978        let _ = daemon.await;
2979        assert!(
2980            ended,
2981            "the published daemon must not still owe its orphan sweep"
2982        );
2983    }
2984
2985    /// Publishing the connection file fsyncs, which can take seconds on a busy
2986    /// disk, so the daemon must do it off the async worker thread. The hook
2987    /// blocks inside the write until another task on the same single-threaded
2988    /// runtime has run. Writing on the worker thread would starve that task: the
2989    /// hook then gives up after its deadline and the test fails, as it does if
2990    /// startup publishes without going through the hooked write at all.
2991    #[tokio::test(flavor = "current_thread")]
2992    async fn connection_file_publication_leaves_the_async_runtime_free() {
2993        use std::sync::atomic::{AtomicBool, Ordering};
2994
2995        let (_dir, path) = temp_connection_file_path("publish-off-runtime");
2996        let write_started = Arc::new(AtomicBool::new(false));
2997        let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
2998        let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
2999        let hook_started = Arc::clone(&write_started);
3000        connection_file_write_hook::install(&path, move || {
3001            hook_started.store(true, Ordering::SeqCst);
3002            let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
3003            let _ = verdict_tx.send(progressed);
3004        });
3005        // Signals only after the write has started, so a message can only come
3006        // from this task running while the write is still blocked.
3007        let ticker = tokio::spawn(async move {
3008            loop {
3009                if write_started.load(Ordering::SeqCst) {
3010                    let _ = progress_tx.send(());
3011                    return;
3012                }
3013                sleep(Duration::from_millis(5)).await;
3014            }
3015        });
3016
3017        let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
3018        let published = timeout(Duration::from_secs(15), async {
3019            while !path.exists() && !daemon.is_finished() {
3020                sleep(Duration::from_millis(5)).await;
3021            }
3022        })
3023        .await;
3024        let daemon_exited = daemon.is_finished();
3025        daemon.abort();
3026        let daemon_result = daemon.await;
3027        ticker.abort();
3028
3029        assert!(
3030            published.is_ok() && !daemon_exited,
3031            "the daemon must start and publish its connection file: {daemon_result:?}"
3032        );
3033        match verdict_rx.try_recv() {
3034            Ok(true) => {}
3035            Ok(false) => panic!(
3036                "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
3037            ),
3038            Err(_) => panic!("startup published the connection file without the hooked write"),
3039        }
3040    }
3041
3042    /// A write that never reports back must stop startup with an error naming
3043    /// the connection file, never be taken as published.
3044    #[tokio::test]
3045    async fn a_panicking_connection_file_write_fails_startup() {
3046        let (_dir, path) = temp_connection_file_path("publish-panics");
3047        connection_file_write_hook::install(&path, || {
3048            panic!("deliberate panic inside the connection-file write")
3049        });
3050
3051        let error = timeout(
3052            Duration::from_secs(15),
3053            run_with_config(BootstrapConfig::new(&path, 0)),
3054        )
3055        .await
3056        .expect("startup must end once the write task fails")
3057        .expect_err("startup must fail when the connection-file write panics");
3058
3059        assert!(
3060            matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
3061            "expected a connection-file write error, got {error:?}"
3062        );
3063        let message = error.to_string();
3064        assert!(message.contains(&path.display().to_string()), "{message}");
3065        assert!(message.contains("panicked"), "{message}");
3066        assert!(!path.exists(), "no connection file may be published");
3067    }
3068
3069    #[cfg(target_os = "macos")]
3070    #[test]
3071    fn nofile_raise_worker() {
3072        let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
3073            return;
3074        };
3075        let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
3076        rlimit::Resource::NOFILE.set(256, hard).unwrap();
3077        raise_nofile_limit_with_ceiling(Some(10240));
3078        std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
3079    }
3080
3081    #[cfg(target_os = "macos")]
3082    #[test]
3083    fn macos_nofile_raise_clamps_to_kernel_ceiling() {
3084        let dir = ScratchDir::new("nofile-ceiling");
3085        let result = dir.join("soft");
3086        let status = std::process::Command::new(std::env::current_exe().unwrap())
3087            .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
3088            .env("SUBC_NOFILE_TEST_FILE", &result)
3089            .env("XDG_DATA_HOME", dir.path())
3090            .env("XDG_RUNTIME_DIR", dir.path())
3091            .env("XDG_CONFIG_HOME", dir.path())
3092            .status()
3093            .unwrap();
3094        assert!(status.success());
3095        let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
3096        assert!(
3097            soft > 256 && soft <= 10240,
3098            "the inherited limit must be raised within the kernel ceiling; got {soft}"
3099        );
3100    }
3101
3102    #[tokio::test]
3103    async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
3104        let (_dir, path) = temp_connection_file_path("key-rotation");
3105        let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3106        let old_info = first.connection_info.clone();
3107        let fixed_port = old_info.endpoints[0].port;
3108        drop(first.listeners);
3109
3110        let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
3111        let new_info = second.connection_info.clone();
3112        assert_ne!(old_info.key, new_info.key);
3113        assert_ne!(old_info.daemon_id, new_info.daemon_id);
3114        let server = start_server(second);
3115
3116        let mut old_stream = connect_from_info(&old_info).await.unwrap();
3117        let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
3118        assert!(
3119            old_auth.is_err(),
3120            "old key must not authenticate after restart"
3121        );
3122
3123        let reread = connection_file::read(&path).unwrap();
3124        let mut new_stream = connect_from_info(&reread).await.unwrap();
3125        authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
3126            .await
3127            .unwrap();
3128
3129        server.abort();
3130        let _ = server.await;
3131    }
3132
3133    #[cfg(unix)]
3134    #[tokio::test]
3135    async fn published_connection_file_permissions_are_owner_only() {
3136        let (_dir, path) = temp_connection_file_path("permissions");
3137        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3138
3139        let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
3140        assert_eq!(mode, 0o600);
3141
3142        drop(bound.listeners);
3143    }
3144
3145    /// A config with the socket in `dir`'s own short subdirectory.
3146    ///
3147    /// A Unix socket path is limited to just over a hundred bytes, and this
3148    /// harness's temporary directories use nearly all of that, so a test that
3149    /// boots a real daemon passes a short directory and a one-letter connection
3150    /// file name; anything longer makes the bind refuse. On Windows there is no
3151    /// socket path to keep short, so the config is returned unchanged.
3152    fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
3153        let dir = config
3154            .connection_file_path
3155            .parent()
3156            .map(Path::to_path_buf)
3157            .unwrap_or_else(|| PathBuf::from("."));
3158        #[cfg(unix)]
3159        let config = BootstrapConfig {
3160            connection_file_path: dir.join("c"),
3161            local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
3162            ..config
3163        };
3164        #[cfg(not(unix))]
3165        let _ = dir;
3166        config
3167    }
3168
3169    /// The shipped binary puts the socket in its own run directory. The file is
3170    /// named from the connection file's path, so a restart finds it again.
3171    #[cfg(unix)]
3172    #[test]
3173    fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
3174        let _env_lock = ENV_LOCK.lock().unwrap();
3175        let root = ScratchDir::new("binary-local-socket");
3176        let data_home = root.join("data");
3177        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3178        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3179        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3180
3181        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3182
3183        let run_dir = data_home.join("cortexkit").join("run");
3184        assert_eq!(
3185            config.local_listener.configured_dir(),
3186            Some(run_dir.as_path()),
3187            "the binary serves from its run directory"
3188        );
3189        let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
3190            .expect("the binary config names a directory");
3191        assert_eq!(socket.parent().unwrap(), run_dir);
3192        assert_eq!(
3193            socket.file_name().unwrap().to_string_lossy(),
3194            "subc.sock",
3195            "a fixed name: the run directory belongs to one data home"
3196        );
3197    }
3198
3199    /// An in-process daemon that names no directory serves TCP only: it publishes
3200    /// no local endpoint, creates nothing under the data home, and says nothing is
3201    /// unavailable, because nothing was asked for.
3202    /// The environment lock covers only the setup: the daemon under test reads
3203    /// its data home from the process environment while it boots, and holding
3204    /// the lock across that await would block every other test that sets one.
3205    #[cfg(unix)]
3206    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3207    async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
3208        let root = ScratchDir::new("in-process-no-local-socket");
3209        let data_home = root.join("data");
3210        fs::create_dir_all(&data_home).unwrap();
3211        let connection_path = root.join("c");
3212        let _data;
3213        let _config;
3214        let _port;
3215        {
3216            let _env_lock = ENV_LOCK.lock().unwrap();
3217            _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3218            _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3219            _port = EnvGuard::unset(SUBC_PORT_ENV);
3220        }
3221
3222        let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
3223
3224        assert!(bound.local_endpoint.is_none());
3225        assert!(bound.local_endpoint_unavailable.is_none());
3226        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3227        assert!(published.local_endpoint.is_none());
3228        assert!(
3229            local_listener::socket_path_for(None, &connection_path).is_none(),
3230            "no configured directory, no socket"
3231        );
3232        let mut leftovers = Vec::new();
3233        let mut pending = vec![data_home.clone()];
3234        while let Some(dir) = pending.pop() {
3235            for entry in fs::read_dir(&dir).unwrap() {
3236                let entry = entry.unwrap();
3237                let path = entry.path();
3238                if entry.file_type().unwrap().is_dir() {
3239                    pending.push(path.clone());
3240                }
3241                if entry.file_name().to_string_lossy().ends_with(".sock") {
3242                    leftovers.push(path);
3243                }
3244            }
3245        }
3246        assert!(
3247            leftovers.is_empty(),
3248            "an in-process daemon must not create a socket in the data home: {leftovers:?}"
3249        );
3250    }
3251
3252    /// A local endpoint that cannot be bound does not stop the daemon: it keeps
3253    /// serving TCP, publishes no local endpoint, and reports the reason where an
3254    /// operator reads daemon facts. Every module stays up for a feature nothing
3255    /// has to use.
3256    #[cfg(unix)]
3257    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3258    async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
3259        let dir = ScratchDir::new("tcp-only");
3260        let canonical = std::fs::canonicalize(&dir).unwrap();
3261        let connection_path = canonical.join("c");
3262        // The socket directory is deep enough that the socket's path is over the
3263        // Unix socket limit, so binding it must fail.
3264        let deep = canonical.join("x".repeat(80));
3265        fs::create_dir(&deep).unwrap();
3266        fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
3267        let config =
3268            BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
3269        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3270        assert!(
3271            bound.local_endpoint.is_none(),
3272            "a socket that cannot be bound is not published"
3273        );
3274        assert!(
3275            bound
3276                .local_endpoint_unavailable
3277                .as_deref()
3278                .is_some_and(|reason| reason.contains("too long")),
3279            "{:?}",
3280            bound.local_endpoint_unavailable
3281        );
3282        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3283        assert!(published.local_endpoint.is_none());
3284        assert!(
3285            !published.info.endpoints.is_empty(),
3286            "TCP is still published"
3287        );
3288
3289        // The daemon serves TCP as usual.
3290        let server = start_server(bound);
3291        let info = connection_file::read(&connection_path).unwrap();
3292        let mut client = connect_from_info(&info).await.unwrap();
3293        authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
3294            .await
3295            .expect("a TCP client still completes the handshake");
3296        server.abort();
3297        let _ = server.await;
3298    }
3299
3300    /// The published file names the local endpoint the daemon actually serves,
3301    /// and a client that trusts the file completes the handshake over it.
3302    #[cfg(unix)]
3303    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3304    async fn published_connection_file_names_the_served_local_endpoint() {
3305        let (_dir, path) = temp_connection_file_path("local-endpoint");
3306        // The socket goes in the system temporary directory rather than inside
3307        // the harness's own directory, whose long lockdown-sandbox path leaves
3308        // no room inside the Unix socket path limit.
3309        let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
3310        let config =
3311            BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
3312        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3313        let endpoint = connection_file::read_with_local_endpoint(&path)
3314            .unwrap()
3315            .local_endpoint
3316            .expect("the daemon serves a local endpoint, so the file names it");
3317        assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
3318        let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
3319            panic!("a Unix daemon publishes a socket path");
3320        };
3321        assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
3322        assert_eq!(
3323            fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
3324            0o600
3325        );
3326
3327        let router = Arc::new(Router::with_default_self_handler());
3328        let auth = auth_for(&bound.connection_info);
3329        let server = tokio::spawn(serve_local_listener(
3330            bound.local_listener.expect("the local listener is bound"),
3331            router,
3332            auth,
3333        ));
3334        let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
3335        authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
3336            .await
3337            .expect("a socket client completes the same handshake as a TCP client");
3338
3339        server.abort();
3340        drop(bound.listeners);
3341    }
3342
3343    /// A configured directory that is not a directory refuses at bind rather
3344    /// than publishing an endpoint nothing serves.
3345    #[cfg(unix)]
3346    #[test]
3347    fn a_socket_directory_that_is_a_file_refuses() {
3348        let dir = ScratchDir::new("local-listener-file-as-dir");
3349        let file = dir.join("not-a-directory");
3350        fs::write(&file, b"x").unwrap();
3351        let error = local_listener::bind(
3352            &LocalListenerConfig::in_socket_dir(file),
3353            &dir.join("connection.json"),
3354        )
3355        .expect_err("a path that is not a directory cannot hold the socket");
3356        assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
3357    }
3358
3359    /// The socket sits directly in the configured directory under one fixed
3360    /// name, and a directory that would put the path over the platform limit is
3361    /// refused rather than shortened around.
3362    #[cfg(unix)]
3363    #[test]
3364    fn the_socket_path_stays_within_the_unix_limit() {
3365        let run_dir = PathBuf::from("/tmp/short-run");
3366        assert!(
3367            local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
3368            "no configured directory means no socket"
3369        );
3370        let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
3371            .expect("a configured directory gets a socket");
3372        assert_eq!(socket, run_dir.join("subc.sock"));
3373        assert_eq!(
3374            socket,
3375            local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
3376            "the name does not come from the connection file"
3377        );
3378
3379        let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
3380        let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
3381            .expect("a configured directory gets a socket");
3382        assert!(
3383            long.as_os_str().len() < 104,
3384            "{} is {} bytes",
3385            long.display(),
3386            long.as_os_str().len()
3387        );
3388    }
3389}