Skip to main content

subc_daemon/
bootstrap.rs

1use std::{
2    collections::BTreeMap,
3    env,
4    error::Error,
5    ffi::OsString,
6    fmt, fs, io,
7    net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8    path::{Path, PathBuf},
9    process,
10    time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17    authenticate_client, connection_file, generate_daemon_id, generate_key,
18    write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
19    LocalEndpoint, SCHEMA_VERSION,
20};
21use tokio::{
22    net::{TcpListener, TcpStream},
23    task::{JoinError, JoinHandle},
24    time::{sleep, timeout},
25};
26use tracing::{error, info, warn};
27
28use crate::{
29    daemon_config::{self, ConfiguredModule, DaemonConfigError},
30    local_listener::{self, BoundLocalListener, LocalListenerConfig},
31    server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
32    supervise::HealthConfig,
33    ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
34    ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
35    SupervisorProcessLiveness,
36};
37use std::sync::Arc;
38
39#[cfg(windows)]
40mod windows_connection_copy;
41
42pub const DEFAULT_SUBC_PORT: u16 = 8757;
43pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
44use subc_transport::CONNECTION_FILE_NAME;
45const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
46const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
47const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
48const START_LOCK_RETRIES: usize = 40;
49const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum ConnectionFileSource {
53    XdgRuntimeDir,
54    TempDirFallback,
55    Explicit,
56}
57
58impl ConnectionFileSource {
59    fn reason(self) -> &'static str {
60        match self {
61            Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
62            Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
63            Self::Explicit => "configured path",
64        }
65    }
66}
67
68impl fmt::Display for ConnectionFileSource {
69    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
70        formatter.write_str(match self {
71            Self::XdgRuntimeDir => "xdg_runtime_dir",
72            Self::TempDirFallback => "temp_dir_fallback",
73            Self::Explicit => "explicit",
74        })
75    }
76}
77
78/// Runtime bootstrap configuration. Production uses the default fixed port and
79/// optional daemon-config override; tests pass port 0 to let the OS assign a free
80/// loopback port and discover it from the connection file.
81#[derive(Debug, Clone, Default)]
82struct AdmissionFactsConfig {
83    carrier_module_id: Option<String>,
84    targets: Option<Vec<String>>,
85}
86
87/// Controls where module cgroups are prepared.
88///
89/// In-process daemons default to [`Self::Disabled`] so they never derive a
90/// production location from the host process. The shipped daemon explicitly uses
91/// [`Self::Current`]. The `ck-subc` binary accepts
92/// `SUBC_CGROUP_PLACEMENT=disabled` for isolated test processes; an unset
93/// variable retains `Current`, and every other value is rejected at startup.
94/// Tests that exercise placement can own a [`Self::Root`].
95#[derive(Debug, Clone, Default, PartialEq, Eq)]
96pub enum CgroupPlacementConfig {
97    #[default]
98    Disabled,
99    Current,
100    Root(PathBuf),
101}
102
103#[derive(Debug, Clone)]
104pub struct BootstrapConfig {
105    #[cfg(feature = "test-support")]
106    shutdown_intent_window: Duration,
107    pub operator_queue_wait: Duration,
108    pub operator_timeout: Duration,
109    pub operator_stuck_grace: Duration,
110    #[cfg(feature = "test-support")]
111    pub operator_script: Option<PathBuf>,
112    #[cfg(feature = "test-support")]
113    pub operator_events: Option<PathBuf>,
114    pub connection_file_path: PathBuf,
115    pub port: u16,
116    pub daemon_ver: String,
117    configured_modules: Vec<ConfiguredModule>,
118    storage_config: Option<daemon_config::StorageConfig>,
119    admission_facts: AdmissionFactsConfig,
120    /// Module ids whose scopes may carry the attributes that grant authority.
121    scope_authority_owners: Vec<String>,
122    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
123    activity_config: daemon_config::ActivityConfig,
124    daemon_config_path: Option<PathBuf>,
125    configured_port: Option<u16>,
126    /// Daemon-wide route.bind relay budget in milliseconds (the fallback for
127    /// any module without a per-module override). `None` = built-in default
128    /// (12s — see `control::DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`).
129    route_bind_relay_default_ms: Option<u64>,
130    reserved_capabilities: BTreeMap<String, String>,
131    watchdog_config: DaemonSelfWatchdogConfig,
132    connection_file_source: ConnectionFileSource,
133    #[cfg(all(test, windows))]
134    profile_override: Option<PathBuf>,
135    /// Where module cgroups are prepared. Disabled unless a caller explicitly
136    /// opts in, because `Current` derives a host location from `/proc/self/cgroup`.
137    cgroup_placement: CgroupPlacementConfig,
138    /// Directory the supervisor writes per-module stdout/stderr capture files
139    /// into. `None` disables capture; the shipped binary supplies its real run
140    /// directory explicitly.
141    capture_logs_dir: Option<PathBuf>,
142    /// File the supervisor appends every module's terminal exits to, so exit
143    /// history survives a daemon restart. `None` keeps terminal history in
144    /// memory only (each module's ring). Absent by default for the same reason
145    /// as `capture_logs_dir`: an in-process daemon booted by a test must not
146    /// append its exits to the operator's real `terminals.jsonl`, where they
147    /// would show up in `ck module terminals`. The shipped binary supplies
148    /// `<run dir>/terminals.jsonl` explicitly.
149    terminal_journal_path: Option<PathBuf>,
150    /// Where the machine id is read from, or minted into when absent. `None`
151    /// serves no machine id. Absent by default for the same reason as
152    /// `capture_logs_dir`: an in-process daemon booted by a test must never
153    /// derive the operator's real data home and mint into it. The shipped binary
154    /// supplies `<data home>/cortexkit/machine-id` explicitly.
155    machine_id_path: Option<PathBuf>,
156
157    /// Where this daemon's same-user local listener lives (a Unix-domain socket
158    /// on macOS and Linux; nothing on Windows, where the pipe name is minted per
159    /// start). The default serves no local endpoint, so an in-process daemon
160    /// never touches the operator's run directory. The shipped binary names its
161    /// run directory; [`BootstrapConfig::with_local_listener_socket_dir`] names
162    /// any other, and an in-process daemon that wants the endpoint names one.
163    local_listener: LocalListenerConfig,
164    /// The live-children record: every supervised process this daemon has
165    /// running, kept so the next daemon can end the ones a crash left behind.
166    /// At startup, before any module is spawned, the previous daemon's record
167    /// here is swept. `None` keeps no record and sweeps nothing, for the same
168    /// reason as `capture_logs_dir`: an in-process daemon booted by a test
169    /// must never signal processes listed in the operator's real run
170    /// directory. The shipped binary supplies `<run dir>/live-children.json`.
171    live_children_path: Option<PathBuf>,
172    privacy_trampoline: Option<PathBuf>,
173}
174
175impl BootstrapConfig {
176    /// Shorten restart intent validity in daemon integration fixtures.
177    #[cfg(feature = "test-support")]
178    pub fn with_shutdown_intent_window(mut self, window: Duration) -> Self {
179        self.shutdown_intent_window = window;
180        self
181    }
182
183    pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
184        Self {
185            #[cfg(feature = "test-support")]
186            shutdown_intent_window: Duration::from_secs(120),
187            connection_file_path: connection_file_path.into(),
188            operator_queue_wait: Duration::from_secs(150),
189            operator_timeout: Duration::from_secs(120),
190            operator_stuck_grace: Duration::from_secs(10),
191            #[cfg(feature = "test-support")]
192            operator_script: None,
193            #[cfg(feature = "test-support")]
194            operator_events: None,
195            port,
196            daemon_ver: DAEMON_VERSION.to_owned(),
197            configured_modules: Vec::new(),
198            storage_config: None,
199            admission_facts: AdmissionFactsConfig::default(),
200            scope_authority_owners: daemon_config::default_scope_authority_owners(),
201            privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
202            activity_config: daemon_config::ActivityConfig::default(),
203            daemon_config_path: None,
204            configured_port: None,
205            route_bind_relay_default_ms: None,
206            reserved_capabilities: BTreeMap::new(),
207            watchdog_config: DaemonSelfWatchdogConfig::default(),
208            connection_file_source: ConnectionFileSource::Explicit,
209            #[cfg(all(test, windows))]
210            profile_override: None,
211            cgroup_placement: CgroupPlacementConfig::default(),
212            capture_logs_dir: None,
213            terminal_journal_path: None,
214            machine_id_path: None,
215            local_listener: LocalListenerConfig { socket_dir: None },
216            live_children_path: None,
217            privacy_trampoline: None,
218        }
219    }
220
221    /// Keep the local listener's socket in `dir`. Embedding daemons and tests
222    /// pass a path inside their own fixture tree.
223    pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
224        self.local_listener = LocalListenerConfig::in_socket_dir(dir);
225        self
226    }
227
228    /// Set where the local listener lives, as [`LocalListenerConfig`] describes.
229    pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
230        self.local_listener = config;
231        self
232    }
233
234    /// Keep the live-children record at `path`, and at startup end the
235    /// processes a previous daemon recorded there that are still running.
236    /// Embedding daemons and tests pass a path inside their own fixture tree.
237    pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
238        self.live_children_path = Some(path.into());
239        self
240    }
241
242    /// macOS module launches require an explicit executable implementing the
243    /// subc-os trampoline entry point. Other systems ignore this setting.
244    pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
245        self.privacy_trampoline = Some(path.into());
246        self
247    }
248
249    /// Serve the machine id stored at `path`, minting it there at startup when
250    /// the file is absent. Embedding daemons and tests pass a path inside their
251    /// own fixture tree.
252    pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
253        self.machine_id_path = Some(path.into());
254        self
255    }
256
257    /// Selects module cgroup placement. The default is disabled.
258    pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
259        self.cgroup_placement = placement;
260        self
261    }
262
263    /// Redirects per-module stdout/stderr capture files out of the real run
264    /// directory. Tests that start an in-process daemon must call this with a
265    /// path inside their fixture tree.
266    pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
267        self.capture_logs_dir = Some(dir.into());
268        self
269    }
270
271    /// Redirects the daemon-private journal, allowing embedded daemons and tests
272    /// to keep their observations out of the operator's live run directory.
273    pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
274        self.terminal_journal_path = Some(path.into());
275        self
276    }
277
278    pub fn from_env() -> Result<Self, BootstrapError> {
279        Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
280    }
281
282    /// The SHIPPED BINARY's config, which is the only caller that should capture
283    /// child output into the operator's real run directory.
284    ///
285    /// Kept separate from `from_env` deliberately: see `capture_logs_dir` and
286    /// `terminal_journal_path` on this struct for why an absent value must mean
287    /// NO CAPTURE and NO JOURNAL rather than the operator's real run directory.
288    /// A run directory that cannot be resolved (a relative data home) refuses
289    /// startup instead of landing under the working directory.
290    pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
291        let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
292        let machine_id_path =
293            crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
294        let config = Self::from_env()?
295            .with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
296            .with_capture_logs_dir(run_dir.join("logs"))
297            .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
298            .with_live_children_record(crate::live_children::record_path(&run_dir))
299            .with_machine_id_path(machine_id_path);
300        #[cfg(feature = "test-support")]
301        let config = {
302            let mut config = config;
303            config.operator_script =
304                std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
305            config.operator_events =
306                std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
307            for (name, value) in [
308                (
309                    "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
310                    &mut config.operator_queue_wait,
311                ),
312                (
313                    "SUBC_TEST_OPERATOR_TIMEOUT_MS",
314                    &mut config.operator_timeout,
315                ),
316                (
317                    "SUBC_TEST_OPERATOR_STUCK_MS",
318                    &mut config.operator_stuck_grace,
319                ),
320            ] {
321                if let Ok(ms) = std::env::var(name) {
322                    if let Ok(ms) = ms.parse::<u64>() {
323                        *value = Duration::from_millis(ms);
324                    }
325                }
326            }
327            config
328        };
329        Ok(config)
330    }
331
332    pub fn from_env_with_daemon_config_path(
333        daemon_config_path: impl AsRef<Path>,
334    ) -> Result<Self, BootstrapError> {
335        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
336        let (daemon_config, activity_config) =
337            daemon_config::load_with_activity(&daemon_config_path)
338                .map_err(BootstrapError::DaemonConfig)?
339                .map(|(config, activity)| (Some(config), activity))
340                .unwrap_or_default();
341        let config_port = daemon_config.as_ref().and_then(|config| config.port);
342        let storage_config = daemon_config
343            .as_ref()
344            .and_then(|config| config.storage.clone());
345        let admission_facts_carrier_module_id = daemon_config
346            .as_ref()
347            .and_then(|config| config.admission_facts_carrier_module_id.clone());
348        let admission_facts_targets = daemon_config
349            .as_ref()
350            .and_then(|config| config.admission_facts_targets.clone());
351        let scope_authority_owners = daemon_config
352            .as_ref()
353            .map(|config| config.scope_authority_owners.clone())
354            .unwrap_or_else(daemon_config::default_scope_authority_owners);
355        let privileged_operator_prompt = daemon_config
356            .as_ref()
357            .map(|config| config.privileged_operator_prompt)
358            .unwrap_or_default();
359        let route_bind_relay_default_ms = daemon_config
360            .as_ref()
361            .and_then(|config| config.route_bind_relay_timeout_ms);
362        let reserved_capabilities = daemon_config
363            .as_ref()
364            .map(|config| config.reserved_capabilities.clone())
365            .unwrap_or_default();
366        let configured_modules = daemon_config
367            .map(|config| config.modules)
368            .unwrap_or_default();
369
370        let port = match env::var(SUBC_PORT_ENV) {
371            Ok(raw) if !raw.trim().is_empty() => {
372                let port = raw
373                    .parse::<u16>()
374                    .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
375                if let Some(config_port) = config_port {
376                    info!(
377                        env = SUBC_PORT_ENV,
378                        env_port = port,
379                        config_port,
380                        "SUBC_PORT overrides daemon config port"
381                    );
382                }
383                port
384            }
385            Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
386        };
387
388        let (connection_file_path, connection_file_source) =
389            connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
390        Ok(Self::new(connection_file_path, port)
391            .with_configured_modules(configured_modules)
392            .with_storage_config(storage_config)
393            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
394            .with_scope_authority_owners(scope_authority_owners)
395            .with_privileged_operator_prompt(privileged_operator_prompt)
396            .with_activity_config(activity_config)
397            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
398            .with_reserved_capabilities(reserved_capabilities)
399            .with_daemon_config_source(daemon_config_path, config_port)
400            .with_connection_file_source(connection_file_source))
401    }
402
403    pub fn with_daemon_config_path(
404        self,
405        daemon_config_path: impl AsRef<Path>,
406    ) -> Result<Self, BootstrapError> {
407        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
408        let (daemon_config, activity_config) =
409            daemon_config::load_with_activity(&daemon_config_path)
410                .map_err(BootstrapError::DaemonConfig)?
411                .map(|(config, activity)| (Some(config), activity))
412                .unwrap_or_default();
413        let configured_port = daemon_config.as_ref().and_then(|config| config.port);
414        let storage_config = daemon_config
415            .as_ref()
416            .and_then(|config| config.storage.clone());
417        let admission_facts_carrier_module_id = daemon_config
418            .as_ref()
419            .and_then(|config| config.admission_facts_carrier_module_id.clone());
420        let admission_facts_targets = daemon_config
421            .as_ref()
422            .and_then(|config| config.admission_facts_targets.clone());
423        let scope_authority_owners = daemon_config
424            .as_ref()
425            .map(|config| config.scope_authority_owners.clone())
426            .unwrap_or_else(daemon_config::default_scope_authority_owners);
427        let privileged_operator_prompt = daemon_config
428            .as_ref()
429            .map(|config| config.privileged_operator_prompt)
430            .unwrap_or_default();
431        let route_bind_relay_default_ms = daemon_config
432            .as_ref()
433            .and_then(|config| config.route_bind_relay_timeout_ms);
434        let reserved_capabilities = daemon_config
435            .as_ref()
436            .map(|config| config.reserved_capabilities.clone())
437            .unwrap_or_default();
438        let configured_modules = daemon_config
439            .map(|config| config.modules)
440            .unwrap_or_default();
441        Ok(self
442            .with_configured_modules(configured_modules)
443            .with_storage_config(storage_config)
444            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
445            .with_scope_authority_owners(scope_authority_owners)
446            .with_privileged_operator_prompt(privileged_operator_prompt)
447            .with_activity_config(activity_config)
448            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
449            .with_reserved_capabilities(reserved_capabilities)
450            .with_daemon_config_source(daemon_config_path, configured_port))
451    }
452
453    pub fn with_configured_modules(
454        mut self,
455        modules: impl IntoIterator<Item = ConfiguredModule>,
456    ) -> Self {
457        self.configured_modules = modules.into_iter().collect();
458        self.configured_modules
459            .sort_by(|left, right| left.module_id.cmp(&right.module_id));
460        self
461    }
462
463    pub fn with_storage_config(
464        mut self,
465        storage_config: Option<daemon_config::StorageConfig>,
466    ) -> Self {
467        self.storage_config = storage_config;
468        self
469    }
470
471    pub fn with_admission_facts_config(
472        mut self,
473        carrier_module_id: Option<String>,
474        targets: Option<Vec<String>>,
475    ) -> Self {
476        self.admission_facts = AdmissionFactsConfig {
477            carrier_module_id,
478            targets,
479        };
480        self
481    }
482
483    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
484    /// Absent from a config, this is `daemon_config::default_scope_authority_owners`.
485    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
486        self.scope_authority_owners = owners;
487        self
488    }
489
490    /// Choose the privileged-operation policy for the daemon's entire lifetime.
491    pub fn with_privileged_operator_prompt(
492        mut self,
493        mode: daemon_config::PrivilegedOperatorPrompt,
494    ) -> Self {
495        self.privileged_operator_prompt = mode;
496        self
497    }
498
499    pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
500        self.activity_config = config;
501        self
502    }
503
504    /// Set the daemon-wide route.bind relay default (the fallback for any
505    /// module without a per-module override). `None` preserves the built-in
506    /// default (12s). `serve_bound_daemon` reads this at startup and threads
507    /// it into the control handler's daemon-wide field.
508    pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
509        self.route_bind_relay_default_ms = ms;
510        self
511    }
512
513    pub fn with_reserved_capabilities(
514        mut self,
515        reserved_capabilities: BTreeMap<String, String>,
516    ) -> Self {
517        self.reserved_capabilities = reserved_capabilities;
518        self
519    }
520
521    fn with_daemon_config_source(
522        mut self,
523        daemon_config_path: PathBuf,
524        configured_port: Option<u16>,
525    ) -> Self {
526        self.daemon_config_path = Some(daemon_config_path);
527        self.configured_port = configured_port;
528        self
529    }
530
531    fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
532        self.connection_file_source = source;
533        self
534    }
535
536    pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
537        self.watchdog_config = watchdog_config;
538        self
539    }
540}
541
542/// Result of singleton discovery.
543// Built once per daemon start and immediately matched, so the size gap between
544// the variants costs nothing worth a box on the public shape.
545#[allow(clippy::large_enum_variant)]
546#[derive(Debug)]
547pub enum Outcome {
548    /// A live daemon authenticated from the connection file; this invocation should exit 0.
549    AlreadyRunning,
550    /// This process won the singleton race, owns bound loopback listener(s), and
551    /// has published a fresh connection file.
552    Bound(BoundDaemon),
553}
554
555#[derive(Debug)]
556pub struct BoundDaemon {
557    pub listeners: Vec<TcpListener>,
558    /// The same-user local listener, bound and not yet served. `None` when
559    /// binding it failed, in which case `local_endpoint` is `None` and
560    /// `local_endpoint_unavailable` says why.
561    pub local_listener: Option<BoundLocalListener>,
562    /// Why the daemon serves no local endpoint, when binding it failed. Reported
563    /// by `server.describe` so an operator can see it.
564    pub local_endpoint_unavailable: Option<String>,
565    /// What the connection file publishes under `local_endpoint`, so a client can
566    /// find the local listener without knowing how it was bound.
567    pub local_endpoint: Option<LocalEndpoint>,
568    pub connection_info: ConnectionInfo,
569    pub connection_file_path: PathBuf,
570    pub connection_file_source: ConnectionFileSource,
571    #[cfg(windows)]
572    connection_file_copy: Option<PathBuf>,
573    /// The machine id this daemon serves, established before any connection is
574    /// accepted. `None` when the config named no machine id path.
575    pub machine_id: Option<crate::machine_id::MachineId>,
576    /// Ownership of the run directory, held until the daemon exits so no other
577    /// daemon sweeps or writes this one's run state. `None` when the config
578    /// keeps no live-children record, and so has no run directory to own.
579    run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
580    /// The singleton start lock, kept when binding deferred publishing the
581    /// connection file. It stays held until the daemon has verified it is serving
582    /// and written the connection file, so a second daemon starting meanwhile
583    /// cannot claim the singleton too. `None` when the connection
584    /// file was already written at bind time.
585    publication_lock: Option<StartLock>,
586}
587
588/// Resolve subc's per-user TCP connection-file path.
589///
590/// `$XDG_RUNTIME_DIR/subc-connection.json` is preferred because the runtime
591/// directory is already per-user on Unix desktops. Without it, subc falls back
592/// to the system temp dir with a per-user token in the filename so different OS
593/// users do not collide on shared temp directories.
594pub fn connection_file_path() -> PathBuf {
595    connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
596}
597
598fn connection_file_path_with_source(
599    runtime_dir: Option<OsString>,
600) -> (PathBuf, ConnectionFileSource) {
601    if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
602        return (
603            PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
604            ConnectionFileSource::XdgRuntimeDir,
605        );
606    }
607
608    (
609        env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
610        ConnectionFileSource::TempDirFallback,
611    )
612}
613
614/// Resolve, claim, and serve the per-user daemon singleton.
615///
616/// A second invocation is successful: if a live daemon authenticates from the
617/// existing connection file, this returns `Ok(())` after logging and the caller
618/// exits with status 0.
619pub async fn run() -> Result<(), BootstrapError> {
620    // `run` is a binary entry point, so it opts into the production cgroup and
621    // child-log locations. In-process callers keep both features disabled by default.
622    run_with_config(
623        BootstrapConfig::from_env_for_daemon_binary()?
624            .with_cgroup_placement(CgroupPlacementConfig::Current),
625    )
626    .await
627}
628
629/// Serve a daemon from an explicit config. This is the entry point the twelve
630/// sibling repos use to boot an in-process daemon in their integration tests.
631///
632/// # THIS INSTALLS NO TRACING SUBSCRIBER, SO THE DAEMON IS SILENT BY DEFAULT
633///
634/// The daemon's own diagnostics go through `tracing`, and `tracing` DISCARDS
635/// every event when no subscriber is installed. The shipped binary installs one
636/// in `main` (`init_tracing`); this function deliberately does not, because a
637/// library that installs a global subscriber fights with whatever the host
638/// process already set up.
639///
640/// The consequence for a test harness is not "less verbose": it is that the
641/// daemon has NOTHING TO SAY about any failure, and a missing instrument reads
642/// exactly like a clean one. PLEX found this on 2026-09-18 while trying to
643/// capture daemon logs beside an intermittent bind failure, and discovered the
644/// daemon had been silent in every conformance run that repo had ever done --
645/// so the one client-side error string was all the evidence that could exist,
646/// and they had spent a real investigation on a failure whose second source was
647/// never being recorded.
648///
649/// Install one in the harness before calling this, and assert it did something
650/// (they measured 236 daemon lines with the subscriber installed, 0 with the
651/// call commented out) -- otherwise the fix is itself unverified.
652pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
653    #[cfg(feature = "test-support")]
654    let shutdown_intent_window = config.shutdown_intent_window;
655    let forwarding = Arc::new(ForwardingTable::default());
656    forwarding.operator_confirms().configure_bootstrap(&config);
657    let configured_modules = config.configured_modules.clone();
658    let storage_config = config.storage_config.clone();
659    let admission_facts = config.admission_facts.clone();
660    let scope_authority_owners = config.scope_authority_owners.clone();
661    let privileged_operator_prompt = config.privileged_operator_prompt;
662    let activity_config = config.activity_config;
663    let daemon_config_path = config.daemon_config_path.clone();
664    let configured_port = config.configured_port;
665    let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
666    let reserved_capabilities = config.reserved_capabilities.clone();
667    let watchdog_config = config.watchdog_config.clone();
668    let cgroup_placement_config = config.cgroup_placement.clone();
669    let capture_logs_dir = config.capture_logs_dir.clone();
670    let terminal_journal_path = config.terminal_journal_path.clone();
671    let live_children_path = config.live_children_path.clone();
672    let privacy_trampoline = config.privacy_trampoline.clone();
673    match ensure_singleton_inner(config, false).await? {
674        Outcome::AlreadyRunning => {
675            info!("subc daemon already running");
676            Ok(())
677        }
678        Outcome::Bound(bound) => {
679            #[cfg(target_os = "linux")]
680            let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
681            #[cfg(not(target_os = "linux"))]
682            let _ = cgroup_placement_config;
683            serve_bound_daemon(
684                bound,
685                forwarding,
686                configured_modules,
687                storage_config,
688                admission_facts,
689                scope_authority_owners,
690                privileged_operator_prompt,
691                activity_config,
692                daemon_config_path,
693                configured_port,
694                route_bind_relay_default_ms,
695                reserved_capabilities,
696                watchdog_config,
697                capture_logs_dir,
698                terminal_journal_path,
699                live_children_path,
700                privacy_trampoline,
701                #[cfg(feature = "test-support")]
702                shutdown_intent_window,
703                #[cfg(target_os = "linux")]
704                cgroup_placement,
705            )
706            .await
707        }
708    }
709}
710
711#[cfg(target_os = "linux")]
712fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
713    let result = match config {
714        CgroupPlacementConfig::Disabled => return None,
715        CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
716        CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
717    };
718
719    match result {
720        Ok(Some(placement)) => Some(placement),
721        Ok(None) => {
722            warn!(
723                placement = ?config,
724                "module cgroup placement is disabled: configured cgroup root is not delegated"
725            );
726            None
727        }
728        Err(error) => {
729            warn!(
730                placement = ?config,
731                error = %error,
732                "module cgroup placement is disabled by an unexpected cgroup probe error"
733            );
734            None
735        }
736    }
737}
738
739/// Target soft limit for open file descriptors, applied to the daemon before any
740/// module is spawned so children inherit it. Multi-root modules (one process
741/// aggregating every project root's sqlite stores, index caches, watchers, and
742/// LSP pipes) trivially exceed the macOS default soft limit of 256; a launchd
743/// user agent does not pass login-shell ulimits through, so the raise must
744/// happen in-process.
745#[cfg(unix)]
746const NOFILE_TARGET: u64 = 65536;
747
748/// Raise RLIMIT_NOFILE to `NOFILE_TARGET` (clamped to the hard limit).
749/// Best-effort: failure is logged and never fatal, since the daemon can run
750/// under the inherited limit — modules with few roots just have less headroom.
751#[cfg(unix)]
752fn raise_nofile_limit() {
753    #[cfg(target_os = "macos")]
754    let ceiling = std::process::Command::new("/usr/sbin/sysctl")
755        .args(["-n", "kern.maxfilesperproc"])
756        .output()
757        .ok()
758        .filter(|output| output.status.success())
759        .and_then(|output| String::from_utf8(output.stdout).ok())
760        .and_then(|value| value.trim().parse::<u64>().ok())
761        .filter(|value| *value > 0);
762    #[cfg(not(target_os = "macos"))]
763    let ceiling = None;
764    raise_nofile_limit_with_ceiling(ceiling);
765}
766
767#[cfg(unix)]
768fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
769    match rlimit::Resource::NOFILE.get() {
770        Ok((soft, hard)) => {
771            // On XNU an infinite hard limit does not remove the separate
772            // per-process kernel ceiling. Exceeding it makes setrlimit fail
773            // without raising the inherited (often 256) soft limit at all.
774            let target = NOFILE_TARGET
775                .min(hard)
776                .min(kernel_ceiling.unwrap_or(u64::MAX));
777            if soft >= target {
778                return;
779            }
780            match rlimit::Resource::NOFILE.set(target, hard) {
781                Ok(()) => info!(
782                    previous_soft = soft,
783                    new_soft = target,
784                    hard,
785                    "raised open-file soft limit for daemon and module children"
786                ),
787                Err(err) => warn!(
788                    soft,
789                    hard,
790                    error = %err,
791                    "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
792                ),
793            }
794        }
795        Err(err) => warn!(error = %err, "could not read open-file limit"),
796    }
797}
798
799/// CRT stdio-stream target on Windows (the `_setmaxstdio` maximum). Win32
800/// HANDLEs — what Rust `File`, tokio sockets, and SQLite's Win32 VFS actually
801/// consume — have a per-process quota in the millions and need no raise; the
802/// C-runtime stream table (default 512) is the only low ceiling, and it is
803/// per-process rather than inherited, so supervised modules linking the CRT
804/// must raise their own. Raising it here covers the daemon itself.
805#[cfg(windows)]
806fn raise_nofile_limit() {
807    const MAXSTDIO_TARGET: u32 = 8192;
808    let current = rlimit::getmaxstdio();
809    if current >= MAXSTDIO_TARGET {
810        return;
811    }
812    match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
813        Ok(new_max) => info!(
814            previous = current,
815            new_max, "raised CRT stdio-stream limit for daemon"
816        ),
817        Err(err) => warn!(
818            current,
819            error = %err,
820            "could not raise CRT stdio-stream limit"
821        ),
822    }
823}
824
825#[cfg(not(any(unix, windows)))]
826fn raise_nofile_limit() {}
827
828pub async fn run_with_daemon_config_path(
829    config: BootstrapConfig,
830    daemon_config_path: impl AsRef<Path>,
831) -> Result<(), BootstrapError> {
832    run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
833}
834
835#[allow(clippy::too_many_arguments)]
836async fn serve_bound_daemon(
837    bound: BoundDaemon,
838    forwarding: Arc<ForwardingTable>,
839    configured_modules: Vec<ConfiguredModule>,
840    storage_config: Option<daemon_config::StorageConfig>,
841    admission_facts: AdmissionFactsConfig,
842    scope_authority_owners: Vec<String>,
843    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
844    activity_config: daemon_config::ActivityConfig,
845    daemon_config_path: Option<PathBuf>,
846    configured_port: Option<u16>,
847    route_bind_relay_default_ms: Option<u64>,
848    reserved_capabilities: BTreeMap<String, String>,
849    watchdog_config: DaemonSelfWatchdogConfig,
850    capture_logs_dir: Option<PathBuf>,
851    terminal_journal_path: Option<PathBuf>,
852    live_children_path: Option<PathBuf>,
853    privacy_trampoline: Option<PathBuf>,
854    #[cfg(feature = "test-support")] shutdown_intent_window: Duration,
855    #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
856) -> Result<(), BootstrapError> {
857    #[cfg(unix)]
858    let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
859        .map_err(BootstrapError::Signal)?;
860    // Windows has no SIGTERM. Ctrl-C is a different event, not an equivalent
861    // service-stop contract, so this shutdown handler is intentionally Unix-only.
862    raise_nofile_limit();
863    #[cfg(windows)]
864    if let Err(error) = subc_jobobject::ensure_root_job() {
865        // Keep the control plane available. The root-job API caches setup
866        // failures, so later module spawns refuse instead of running unprotected.
867        warn!(%error, "module launches disabled: Windows root job setup failed");
868    }
869
870    info!(
871        connection_file = %bound.connection_file_path.display(),
872        connection_file_source = %bound.connection_file_source,
873        connection_file_source_reason = bound.connection_file_source.reason(),
874        endpoints = ?bound.connection_info.endpoints,
875        configured_modules = configured_modules.len(),
876        machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
877        "subc daemon starting"
878    );
879
880    // Before anything can spawn a module (the configured modules below, or a
881    // client's start request once the listeners are served): a previous daemon
882    // that died without its shutdown stop may have left children running, and
883    // a fresh copy beside one would fight it for its port and stores. The
884    // record is not a running daemon's because this daemon holds the lock on
885    // the run directory the record lives in, and a running daemon holds that
886    // lock for its whole life. (Claiming the singleton is not enough: it is
887    // keyed on the connection file, which can live in a different runtime
888    // directory from the run directory.) The lock stays held until this
889    // function returns, which is when the daemon exits.
890    let run_dir_lock = bound.run_dir_lock;
891    let publication_lock = bound.publication_lock;
892    if let Some(owner) = &run_dir_lock {
893        #[cfg(target_os = "linux")]
894        crate::live_children::sweep_orphans_with_placement(
895            owner,
896            &crate::live_children::AdoptedPids::none(),
897            crate::live_children::SweepBounds::default(),
898            cgroup_placement.as_ref(),
899        )
900        .await;
901        #[cfg(not(target_os = "linux"))]
902        crate::live_children::sweep_orphans(
903            owner,
904            &crate::live_children::AdoptedPids::none(),
905            crate::live_children::SweepBounds::default(),
906        )
907        .await;
908    }
909
910    let registry = Arc::new(Registry::default());
911    let process_liveness = Arc::new(SupervisorProcessLiveness::new());
912    let supervisor_handle = SupervisorHandle::new();
913    #[cfg(feature = "test-support")]
914    supervisor_handle.set_shutdown_intent_window(shutdown_intent_window);
915    let connected_clients = ConnectedClients::new();
916    let daemon_incarnation = format!(
917        "{:032x}",
918        u128::from_be_bytes(bound.connection_info.daemon_id)
919    );
920    let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
921        .with_process_liveness(process_liveness.clone())
922        .with_forwarding(Arc::clone(&forwarding))
923        .with_handle(supervisor_handle.clone())
924        .with_connection_file_path(bound.connection_file_path.clone())
925        .with_daemon_incarnation(daemon_incarnation.clone());
926    let supervisor = match privacy_trampoline {
927        Some(path) => supervisor.with_privacy_trampoline(path),
928        None => supervisor,
929    };
930    // ABSENT MEANS NO CAPTURE AND NO JOURNAL, NOT "THE REAL RUN DIRECTORY", and
931    // the difference is a production-corruption hazard rather than a preference.
932    // Both fields below follow the same rule: `None` for `capture_logs_dir`
933    // means supervised output is not captured, and `None` for
934    // `terminal_journal_path` means terminal history lives only in each
935    // module's in-memory ring (`supervisor.terminals` still answers, with the
936    // journal counters at zero).
937    //
938    // The terminal journal used to fall back to
939    // `daemon_run_dir().join("terminals.jsonl")`, so an in-process test daemon
940    // appended its fixture exits to the operator's journal, where they then
941    // appeared in `ck module terminals`.
942    //
943    // The capture line used to be `unwrap_or_else(|| daemon_run_dir().join("logs"))`,
944    // so ANY caller that did not set the field captured supervised children into
945    // the operator's live `~/.local/share/cortexkit/run/logs/`. That is twelve
946    // sibling repos whose integration tests boot an in-process daemon through
947    // `run_with_config` -- none of which asked for it, and none of which can see
948    // it from their side.
949    //
950    // Harmless while fixture module ids are fixture-shaped: this host carries 20
951    // zero-byte files from subc's own tests (good-aft, missing-aft,
952    // preview-consumer...). THE HAZARD IS A COLLISION. A fixture named "broca"
953    // or "aft" appends to a PRODUCTION capture file that operators read
954    // forensically and that placement gates count lines in -- with no residue to
955    // notice, because the file legitimately exists and legitimately grows.
956    //
957    // Found by BROCA (2026-09-19) from the other side: their rigs spawn the
958    // SHIPPED ck-subc and set XDG_CONFIG_HOME + XDG_RUNTIME_DIR but not
959    // XDG_DATA_HOME, so every local rig run supervised a module named "broca"
960    // and captured it into production's broca.stderr.log -- the same file I
961    // count seal lines in before and after placing their binaries.
962    //
963    // The supervisor already treats `None` as no-capture and no-journal, so
964    // this only removes invented defaults. The binary keeps capturing and
965    // journaling via `BootstrapConfig::from_env_for_daemon_binary`.
966
967    // Keep both ledgers in the explicitly selected run directory. An embedded
968    // daemon without a journal path must not write to the operator's live ledger.
969    let activity_run_dir = terminal_journal_path
970        .as_ref()
971        .and_then(|path| path.parent())
972        .map(Path::to_path_buf);
973    let supervisor = match terminal_journal_path {
974        Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
975        None => supervisor,
976    };
977    let supervisor = match capture_logs_dir {
978        Some(dir) => supervisor.with_capture_logs_dir(dir),
979        None => supervisor,
980    };
981    let supervisor = match live_children_path {
982        Some(path) => supervisor.with_live_children_record(path),
983        None => supervisor,
984    };
985    #[cfg(target_os = "linux")]
986    let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
987    // Collect per-module route.bind relay overrides BEFORE handing the
988    // `configured_modules` vector to the supervisor (which only needs each
989    // module's `drain_timeout_ms`). Each entry was filled in by parse-time
990    // resolution (per-module > daemon-wide > absent), so modules with no
991    // override are absent from this map and the daemon-wide default applies.
992    let route_bind_relay_timeouts = configured_modules
993        .iter()
994        .filter_map(|module| {
995            module
996                .route_bind_relay_timeout_ms
997                .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
998        })
999        .collect::<std::collections::BTreeMap<_, _>>();
1000    let control_start_clock = crate::clock::StartClock::capture();
1001    let mut control = ControlHandler::with_forwarding(Arc::clone(&registry), forwarding)
1002        .with_process_liveness(process_liveness)
1003        .with_supervisor(supervisor_handle)
1004        // The same flag `begin_daemon_shutdown` sets below, so requirement
1005        // changes while modules exit during the shutdown log at INFO.
1006        .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
1007        .with_connected_clients(connected_clients.clone())
1008        .with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
1009        .with_storage_config(storage_config)
1010        .with_machine_id(bound.machine_id.clone())
1011        .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
1012        .with_scope_authority_owners(scope_authority_owners)
1013        .with_privileged_operator_prompt(privileged_operator_prompt)
1014        .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
1015        .with_daemon_provenance(
1016            bound.connection_info.pid,
1017            control_start_clock.started_at_ms(),
1018            std::env::current_exe().ok(),
1019            normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
1020            normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
1021        )
1022        .with_daemon_start_clock(control_start_clock)
1023        .with_capability_config(
1024            configured_modules
1025                .iter()
1026                .map(|module| (module.module_id.clone(), module.enabled)),
1027            reserved_capabilities,
1028        );
1029    if let Some(run_dir) = activity_run_dir {
1030        control =
1031            control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
1032    }
1033    if let Some(ms) = route_bind_relay_default_ms {
1034        // A daemon-wide config value overrides the built-in default; a
1035        // `None` here leaves the ControlHandler's 12s default in place.
1036        control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
1037    }
1038    if let Some(config_path) = daemon_config_path {
1039        control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
1040    }
1041    let control = Arc::new(control);
1042    let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
1043    let auth = ServerAuth::new(
1044        bound.connection_info.key.clone(),
1045        bound.connection_info.daemon_id,
1046        bound.connection_info.daemon_ver.clone(),
1047    )
1048    .with_connected_clients(connected_clients);
1049
1050    // Discovery must expose every accepted configured module, including disabled
1051    // ones. Registration creates status handles but neither starts monitors
1052    // nor launches processes: children need the published file for `--subc`.
1053    let mut registered_modules = Vec::with_capacity(configured_modules.len());
1054    for configured in configured_modules {
1055        let enabled = configured.enabled;
1056        let health = configured.health.clone();
1057        let module_id = configured.module_id.clone();
1058        let registered = match supervisor.register_configured_with_health(
1059            configured.module_spec(),
1060            enabled,
1061            health.clone(),
1062            configured.drain_timeout_ms,
1063            configured.restart,
1064            configured.register_timeout,
1065        ) {
1066            Ok(registered) => registered,
1067            Err(err) => {
1068                error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
1069                continue;
1070            }
1071        };
1072        registered_modules.push(registered);
1073        // A relaxed threshold can silently outlive the temporary allowance
1074        // that required it, so report the extra tolerance on every boot.
1075        let default_threshold = HealthConfig::default().failure_threshold;
1076        if enabled && health.failure_threshold > default_threshold {
1077            warn!(
1078                module_id = %module_id,
1079                failure_threshold = health.failure_threshold,
1080                default_threshold,
1081                tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1082                "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1083            );
1084        }
1085        info!(module_id = %module_id, enabled, "configured module registered");
1086    }
1087    control.refresh_capability_requirements();
1088
1089    // The local endpoint is served beside TCP and answers the same handshake.
1090    // It serves clients only, which the server enforces per connection.
1091    let local_serve_task = bound.local_listener.map(|local_listener| {
1092        AbortOnDrop::new(tokio::spawn(serve_local_listener(
1093            local_listener,
1094            Arc::clone(&router),
1095            auth.clone(),
1096        )))
1097    });
1098    // On Unix the task is dropped below to stop accepting at shutdown. Windows
1099    // leaves it to the process exit; naming it keeps the binding uniform.
1100    #[cfg(windows)]
1101    let _local_serve_task = local_serve_task;
1102    let mut serve_task =
1103        AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1104    verify_serving(&bound.connection_info).await?;
1105    // A published endpoint promises that authentication can complete now.
1106    // Orphan cleanup and control-plane construction can take longer than an
1107    // existing-daemon probe's deadline, so finish both before discovery.
1108    // The write fsyncs, so it runs on tokio's blocking pool (see
1109    // `publish_connection_file`). The singleton lock travels with it and comes
1110    // back only once the file is in place.
1111    let (published, publication_lock) = publish_connection_file(
1112        &bound.connection_file_path,
1113        #[cfg(windows)]
1114        bound.connection_file_copy.as_deref(),
1115        &bound.connection_info,
1116        bound.local_endpoint.as_ref(),
1117        publication_lock,
1118    )
1119    .await;
1120    published?;
1121    #[cfg(test)]
1122    startup_publication_hook::run(&bound.connection_file_path).await;
1123    drop(publication_lock);
1124    let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1125    // Off the startup path: it runs `systemctl`, and only ever logs.
1126    #[cfg(target_os = "linux")]
1127    let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1128        crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1129    ));
1130    let _watchdog_task = AbortOnDrop::new(
1131        DaemonSelfWatchdog::new(
1132            bound.connection_info.clone(),
1133            bound.connection_file_path.clone(),
1134        )
1135        .with_config(watchdog_config)
1136        .spawn(),
1137    );
1138
1139    for registered in registered_modules {
1140        match supervisor.start_registered_configured(registered, None, true) {
1141            Ok(module) => {
1142                info!(module_id = %module.module_id(), "configured module supervised");
1143            }
1144            Err(err) => {
1145                error!(error = %err, "failed to start configured module; continuing daemon startup");
1146            }
1147        }
1148    }
1149
1150    control.refresh_capability_requirements();
1151    Arc::clone(&control).spawn_capability_deadline_loop();
1152    Arc::clone(&control).spawn_scope_expiry_loop();
1153
1154    #[cfg(unix)]
1155    {
1156        tokio::select! {
1157            result = serve_task.join() => {
1158                return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1159            }
1160            _ = terminate.recv() => {}
1161        }
1162        // First, before the notice, the drain, or any connection close: from
1163        // here on a module exit is recorded as `daemon_shutdown` and never
1164        // respawned. Also before allowing a second signal to cut the bounded
1165        // wait short, so the journal marker is always written.
1166        supervisor.begin_daemon_shutdown();
1167        // Stop the self-watchdog before closing the listener. Its next tick would
1168        // connect to that listener, fail, and log an ERROR indistinguishable from
1169        // a wedged daemon, once for every interval a planned stop lasts.
1170        drop(_watchdog_task);
1171        // Dropping the listener stops new accepts, not established connections:
1172        // their detached tasks must remain live throughout notice and drain. The
1173        // local endpoint stops accepting the same way.
1174        drop(serve_task);
1175        drop(local_serve_task);
1176        let second_sigterm = async {
1177            terminate.recv().await;
1178            std::time::Instant::now()
1179        };
1180        tokio::pin!(second_sigterm);
1181        let mut escalation_started = tokio::select! {
1182            biased;
1183            at = second_sigterm.as_mut() => {
1184                info!("second SIGTERM: abandoning daemon shutdown wait");
1185                Some(at)
1186            }
1187            result = supervisor.drain_for_daemon_shutdown() => {
1188                if let Err(error) = result {
1189                    warn!(%error, "daemon shutdown drain failed; exiting anyway");
1190                }
1191                None
1192            }
1193        };
1194        // Supervised modules lead their own process groups, so a service
1195        // manager's kill of this process's group does not reach them. (A
1196        // systemd unit with KillMode=control-group kills by cgroup instead and
1197        // does reach them; see `systemd_kill_mode`.) The daemon ends them
1198        // itself: EOF (or SIGTERM for a protocol none child) first, then
1199        // signals at each child's own drain deadline.
1200        let already_escalated = escalation_started.is_some();
1201        supervisor
1202            .end_children_for_daemon_shutdown(already_escalated, async {
1203                if already_escalated {
1204                    std::future::pending::<()>().await;
1205                } else {
1206                    escalation_started = Some(second_sigterm.await);
1207                }
1208            })
1209            .await;
1210        if let Some(at) = escalation_started {
1211            info!(
1212                second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1213                "daemon shutdown wait ended after second SIGTERM"
1214            );
1215        }
1216        info!("daemon exiting: SIGTERM shutdown complete");
1217        Ok(())
1218    }
1219    #[cfg(not(unix))]
1220    serve_task
1221        .join()
1222        .await
1223        .map_err(BootstrapError::ServeJoin)?
1224        .map_err(BootstrapError::Serve)
1225}
1226
1227fn normalized_build_provenance(value: &str) -> Option<String> {
1228    match value.trim() {
1229        "" | "unavailable" => None,
1230        value => Some(value.to_string()),
1231    }
1232}
1233
1234/// Find an existing daemon or atomically bind loopback TCP for this daemon.
1235///
1236/// The algorithm is intentionally connect-first: an endpoint from the connection
1237/// file is treated as live only after the TCP+key server-proof authenticates for
1238/// that file's key and daemon_id. Stale or foreign connection files are reclaimed
1239/// only while holding the per-user start lock; the TCP port is never the
1240/// singleton primitive.
1241pub async fn ensure_singleton(
1242    connection_file_path: impl AsRef<Path>,
1243    port: u16,
1244) -> Result<Outcome, BootstrapError> {
1245    ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1246}
1247
1248pub async fn ensure_singleton_with_config(
1249    config: BootstrapConfig,
1250) -> Result<Outcome, BootstrapError> {
1251    ensure_singleton_inner(config, true).await
1252}
1253
1254async fn ensure_singleton_inner(
1255    config: BootstrapConfig,
1256    publish: bool,
1257) -> Result<Outcome, BootstrapError> {
1258    #[cfg(windows)]
1259    let copy_path = windows_connection_copy::resolve(&config);
1260    let path = config.connection_file_path;
1261
1262    if stable_peer(&path).await? {
1263        return Ok(Outcome::AlreadyRunning);
1264    }
1265
1266    let lock = StartLock::acquire(&path).await?;
1267
1268    // Re-probe after acquiring the start lock so a peer that won the race between
1269    // our first failed probe and the lock acquisition is observed instead of
1270    // overwritten.
1271    if stable_peer(&path).await? {
1272        return Ok(Outcome::AlreadyRunning);
1273    }
1274
1275    // The start lock and the probe above only rule out a daemon using this
1276    // connection file. A daemon started with another runtime directory but
1277    // the same data home shares the run directory while passing both, so the
1278    // run directory needs its own owner: refuse at once if a live daemon holds
1279    // it, before anything below reads or writes run or data-home state.
1280    let run_dir_lock = config
1281        .live_children_path
1282        .as_deref()
1283        .map(crate::run_dir_lock::RunDirLock::acquire)
1284        .transpose()?;
1285
1286    remove_stale_connection_file_if_present(&path)?;
1287    #[cfg(windows)]
1288    let copy_path = windows_connection_copy::reclaim(&path, copy_path).await;
1289
1290    // Established under the start lock, and under the run-directory lock when
1291    // there is one, and before binding: a corrupt file stops boot before
1292    // anything is published, and two daemons racing to start cannot both mint.
1293    //
1294    // Minting a new machine id fsyncs its file, so loading or minting runs on
1295    // tokio's blocking pool. Both locks travel with that work, so neither is
1296    // released while a mint is in flight.
1297    let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1298        None => (None, lock, run_dir_lock),
1299        Some(machine_id_path) => {
1300            let task_path = machine_id_path.clone();
1301            match run_blocking_holding((lock, run_dir_lock), move || {
1302                crate::machine_id::load_or_mint(&task_path)
1303            })
1304            .await
1305            {
1306                Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1307                Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1308                Err(join_error) => {
1309                    return Err(BootstrapError::MachineId(
1310                        crate::machine_id::MachineIdFileError::Write {
1311                            path: machine_id_path,
1312                            source: io::Error::other(describe_blocking_task_failure(
1313                                "loading or minting the machine id",
1314                                &join_error,
1315                            )),
1316                        },
1317                    ))
1318                }
1319            }
1320        }
1321    };
1322
1323    let (listeners, endpoints) = bind_loopback(config.port).await?;
1324    // The local endpoint is an addition to the daemon, not a condition for it:
1325    // a bind failure leaves clients on TCP only, with the connection file naming
1326    // no local endpoint, and the reason reported by `server.describe`. A daemon
1327    // that refused to start would take every module down for a feature nothing
1328    // has to use yet.
1329    let (local_listener, local_endpoint, local_endpoint_unavailable) =
1330        if config.local_listener.configured_dir().is_some() {
1331            match local_listener::bind(&config.local_listener, &path) {
1332                Ok(local_listener) => {
1333                    let endpoint = local_listener.endpoint();
1334                    (Some(local_listener), Some(endpoint), None)
1335                }
1336                Err(error) => {
1337                    let reason = error.to_string();
1338                    warn!(
1339                        error = %reason,
1340                        "the local endpoint is unavailable; serving loopback TCP only"
1341                    );
1342                    (None, None, Some(reason))
1343                }
1344            }
1345        } else {
1346            (None, None, None)
1347        };
1348    let connection_info = ConnectionInfo {
1349        schema: SCHEMA_VERSION,
1350        wire_version: Some(PROTOCOL_VERSION),
1351        endpoints,
1352        key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1353        daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1354        pid: process::id(),
1355        daemon_ver: config.daemon_ver,
1356    };
1357
1358    // The low-level binding API historically publishes for callers that serve
1359    // these listeners themselves. The daemon entry point defers publication
1360    // until its own authentication server is running.
1361    let publication_lock = if publish {
1362        let (published, lock) = publish_connection_file(
1363            &path,
1364            #[cfg(windows)]
1365            copy_path.as_deref(),
1366            &connection_info,
1367            local_endpoint.as_ref(),
1368            lock,
1369        )
1370        .await;
1371        if let Err(error) = published {
1372            if let Some(local_listener) = local_listener.as_ref() {
1373                local_listener::release(local_listener);
1374            }
1375            drop(listeners);
1376            return Err(error);
1377        }
1378        drop(lock);
1379        None
1380    } else {
1381        Some(lock)
1382    };
1383
1384    Ok(Outcome::Bound(BoundDaemon {
1385        listeners,
1386        local_listener,
1387        local_endpoint,
1388        local_endpoint_unavailable,
1389        connection_info,
1390        connection_file_path: path,
1391        connection_file_source: config.connection_file_source,
1392        #[cfg(windows)]
1393        connection_file_copy: copy_path,
1394        machine_id,
1395        run_dir_lock,
1396        publication_lock,
1397    }))
1398}
1399
1400#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1401enum Probe {
1402    Live,
1403    StaleOrAbsent,
1404}
1405
1406async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1407    if !matches!(probe_existing(path).await?, Probe::Live) {
1408        return Ok(false);
1409    }
1410    // A terminating peer can still authenticate while its listeners drain.
1411    // Require another proof after a short settling interval before reporting
1412    // a successful duplicate start. If the peer left, proceed with startup;
1413    // any remaining bind/ownership conflict is an error, not exit 0 (which
1414    // would suppress launchd's restart-on-failure policy).
1415    tokio::time::sleep(Duration::from_millis(1500)).await;
1416    Ok(matches!(probe_existing(path).await?, Probe::Live))
1417}
1418
1419async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1420    let info = match connection_file::read(path) {
1421        Ok(info) => info,
1422        Err(source) if is_absent_or_stale_connection_file(&source) => {
1423            return Ok(Probe::StaleOrAbsent)
1424        }
1425        Err(source) => {
1426            return Err(BootstrapError::ConnectionFileRead {
1427                path: path.to_path_buf(),
1428                source,
1429            })
1430        }
1431    };
1432
1433    for endpoint in &info.endpoints {
1434        if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1435            return Ok(Probe::Live);
1436        }
1437    }
1438
1439    Ok(Probe::StaleOrAbsent)
1440}
1441
1442async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1443    let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1444        return Probe::StaleOrAbsent;
1445    };
1446    if !ip.is_loopback() {
1447        return Probe::StaleOrAbsent;
1448    }
1449    let addr = SocketAddr::new(ip, endpoint.port);
1450
1451    let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1452        Ok(Ok(stream)) => stream,
1453        Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1454    };
1455
1456    match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1457        Ok(()) => Probe::Live,
1458        Err(AuthError::DaemonIdMismatch)
1459        | Err(AuthError::InvalidServerProof)
1460        | Err(AuthError::UnexpectedEof { .. })
1461        | Err(AuthError::Timeout { .. })
1462        | Err(AuthError::JsonEncode { .. })
1463        | Err(AuthError::JsonDecode { .. })
1464        | Err(AuthError::Io { .. })
1465        | Err(AuthError::MessageTooLarge { .. })
1466        | Err(AuthError::KeyTooShort { .. })
1467        | Err(AuthError::Random(_))
1468        | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1469    }
1470}
1471
1472async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1473    // Scheduling a server task is not a readiness barrier. Authenticate once
1474    // against the mandatory IPv4 listener before exposing discovery to peers.
1475    // This uses the normal probe deadline, with no retries or widened budget.
1476    if let Some(endpoint) = info.endpoints.first() {
1477        if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1478            return Ok(());
1479        }
1480    }
1481    Err(BootstrapError::StartupNotServing)
1482}
1483
1484/// Run blocking filesystem work (an fsync, a rename) on tokio's blocking-thread
1485/// pool rather than on the async worker thread that awaits it. On a slow or
1486/// busy disk an fsync can take seconds, and a runtime has few worker threads
1487/// (an in-process test daemon may share two with everything else), so doing it
1488/// inline stalls unrelated tasks for that long.
1489///
1490/// `guards` (lock handles) move into the blocking task and are handed back
1491/// with the work's result. If the awaiting future is dropped first, the work
1492/// still runs to completion on its thread and the guards are released only
1493/// after it ends, so a lock is never given up while the work it protects is
1494/// still writing.
1495///
1496/// `Err` means the task panicked, or was cancelled before it started because
1497/// the runtime is shutting down; the guards have then been released already.
1498async fn run_blocking_holding<G, T>(
1499    guards: G,
1500    work: impl FnOnce() -> T + Send + 'static,
1501) -> Result<(T, G), JoinError>
1502where
1503    G: Send + 'static,
1504    T: Send + 'static,
1505{
1506    tokio::task::spawn_blocking(move || {
1507        let output = work();
1508        (output, guards)
1509    })
1510    .await
1511}
1512
1513/// Explain, for a startup error, why blocking work produced no result.
1514fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1515    if join_error.is_panic() {
1516        format!("the background thread {work} panicked before finishing: {join_error}")
1517    } else {
1518        format!(
1519            "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1520        )
1521    }
1522}
1523
1524/// Publish the connection file: owner-only permissions, written to a temporary
1525/// file, fsynced and renamed over `path` (all done by `write_atomic`). The
1526/// write runs on tokio's blocking pool through [`run_blocking_holding`], so the
1527/// fsync never holds an async worker thread.
1528///
1529/// `guard` is the singleton start lock (or whatever the caller must keep held
1530/// across publication); it is returned with the result. It comes back `None`
1531/// only when the write task panicked or was cancelled, in which case the
1532/// result is an error and the file must be treated as not published.
1533///
1534/// On Windows, `copy_path` asks for the profile copy as well. Its start lock is
1535/// taken before the primary write. Once `path` is written, an error returned
1536/// while preparing or writing the copy only logs a warning, and the result is
1537/// still success. The two writes share one blocking task but aren't one atomic
1538/// step, and a panic or cancellation of that task is still a publication
1539/// error.
1540async fn publish_connection_file<G: Send + 'static>(
1541    path: &Path,
1542    #[cfg(windows)] copy_path: Option<&Path>,
1543    info: &ConnectionInfo,
1544    local_endpoint: Option<&LocalEndpoint>,
1545    guard: G,
1546) -> (Result<(), BootstrapError>, Option<G>) {
1547    let task_path = path.to_path_buf();
1548    let task_info = info.clone();
1549    let task_endpoint = local_endpoint.cloned();
1550    #[cfg(windows)]
1551    let copy = windows_connection_copy::claim_for_publication(path, copy_path, info).await;
1552    let outcome = run_blocking_holding(guard, move || {
1553        write_connection_file(&task_path, &task_info, task_endpoint.as_ref())?;
1554        #[cfg(windows)]
1555        if let Some((copy_path, _lock)) = copy {
1556            if let Err(error) =
1557                write_connection_file(&copy_path, &task_info, task_endpoint.as_ref())
1558            {
1559                windows_connection_copy::warn_unavailable(&task_path, Some(&copy_path), &error);
1560            }
1561        }
1562        Ok(())
1563    })
1564    .await;
1565    match outcome {
1566        Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1567        Ok((Err(source), guard)) => (
1568            Err(BootstrapError::ConnectionFileWrite {
1569                path: path.to_path_buf(),
1570                source,
1571            }),
1572            Some(guard),
1573        ),
1574        Err(join_error) => (
1575            Err(BootstrapError::ConnectionFileWrite {
1576                path: path.to_path_buf(),
1577                source: ConnectionFileError::Io {
1578                    op: "write",
1579                    path: path.to_path_buf(),
1580                    source: io::Error::other(describe_blocking_task_failure(
1581                        "writing the connection file",
1582                        &join_error,
1583                    )),
1584                },
1585            }),
1586            None,
1587        ),
1588    }
1589}
1590
1591/// The synchronous connection-file write, including its fsync. Call it only
1592/// through [`publish_connection_file`], which keeps it off the async workers.
1593///
1594/// `local_endpoint` is written through subc-transport's endpoint-aware writer,
1595/// so the file carries `local_endpoint` when there is a local listener and is
1596/// byte-identical to before when there is none.
1597fn write_connection_file(
1598    path: &Path,
1599    info: &ConnectionInfo,
1600    local_endpoint: Option<&LocalEndpoint>,
1601) -> Result<(), ConnectionFileError> {
1602    #[cfg(test)]
1603    connection_file_write_hook::run(path);
1604    write_atomic_with_local_endpoint(path, info, local_endpoint)
1605}
1606
1607/// Lets a test run code inside the connection-file write, on whichever thread
1608/// performs it, for example to block there and check what else can still run.
1609#[cfg(test)]
1610mod connection_file_write_hook {
1611    use std::{
1612        path::{Path, PathBuf},
1613        sync::{Mutex, PoisonError},
1614    };
1615
1616    type Hook = Box<dyn FnOnce() + Send>;
1617
1618    /// Keyed by connection-file path, so a hook fires only for the test that
1619    /// installed it while other tests publish their own files in parallel.
1620    static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
1621
1622    /// Run `hook` once, at the start of the next write of `path`.
1623    pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
1624        HOOKS
1625            .lock()
1626            .unwrap_or_else(PoisonError::into_inner)
1627            .push((path.to_path_buf(), Box::new(hook)));
1628    }
1629
1630    pub(super) fn run(path: &Path) {
1631        let hook = {
1632            let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
1633            hooks
1634                .iter()
1635                .position(|(hooked, _)| hooked == path)
1636                .map(|index| hooks.swap_remove(index).1)
1637        };
1638        if let Some(hook) = hook {
1639            hook();
1640        }
1641    }
1642}
1643
1644/// A path-scoped barrier after publication and before any configured activation.
1645/// The async wait leaves the listeners free to answer discovery-time requests.
1646#[cfg(test)]
1647mod startup_publication_hook {
1648    use std::{
1649        path::{Path, PathBuf},
1650        sync::{Mutex, PoisonError},
1651    };
1652    use tokio::sync::oneshot;
1653
1654    type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
1655    static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
1656
1657    pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
1658        let (entered_tx, entered_rx) = oneshot::channel();
1659        let (release_tx, release_rx) = oneshot::channel();
1660        BARRIERS
1661            .lock()
1662            .unwrap_or_else(PoisonError::into_inner)
1663            .push((path.to_path_buf(), (entered_tx, release_rx)));
1664        (entered_rx, release_tx)
1665    }
1666
1667    pub(super) async fn run(path: &Path) {
1668        let barrier = {
1669            let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
1670            barriers
1671                .iter()
1672                .position(|(hooked, _)| hooked == path)
1673                .map(|index| barriers.swap_remove(index).1)
1674        };
1675        if let Some((entered, release)) = barrier {
1676            let _ = entered.send(());
1677            let _ = release.await;
1678        }
1679    }
1680}
1681
1682fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
1683    match err {
1684        ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
1685        ConnectionFileError::JsonRead { .. }
1686        | ConnectionFileError::UnsupportedSchema { .. }
1687        | ConnectionFileError::Invalid { .. }
1688        | ConnectionFileError::KeyTooShort { .. }
1689        // A live daemon always publishes the file owner-only (0600), so a file
1690        // with insecure permissions is never a daemon we should defer to: treat it
1691        // as stale and take over (which republishes a correct 0600 file).
1692        | ConnectionFileError::InsecurePermissions { .. } => true,
1693        ConnectionFileError::MissingParent { .. }
1694        | ConnectionFileError::MissingFileName { .. }
1695        // A writable ancestor is an operator misconfiguration, never evidence
1696        // about whether a daemon is live. Reclaiming the file would republish key
1697        // material into the same directory the refusal is about.
1698        | ConnectionFileError::InsecureParentDirectory { .. }
1699        | ConnectionFileError::Io { .. }
1700        | ConnectionFileError::JsonWrite { .. }
1701        | ConnectionFileError::Random(_)
1702        // A wire mismatch may identify a newer live daemon, so never reclaim its
1703        // connection file merely because this binary cannot speak its envelope.
1704        | ConnectionFileError::WireVersionMismatch { .. } => false,
1705    }
1706}
1707
1708fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
1709    match fs::remove_file(path) {
1710        Ok(()) => Ok(()),
1711        Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
1712        Err(source) => Err(BootstrapError::RemoveStale {
1713            path: path.to_path_buf(),
1714            source,
1715        }),
1716    }
1717}
1718
1719async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
1720    bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
1721}
1722
1723async fn bind_loopback_with<F, Fut>(
1724    port: u16,
1725    mut bind_v6: F,
1726) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
1727where
1728    F: FnMut(u16) -> Fut,
1729    Fut: std::future::Future<Output = io::Result<TcpListener>>,
1730{
1731    let v4_host = Ipv4Addr::LOCALHOST;
1732    let v4 = TcpListener::bind((v4_host, port))
1733        .await
1734        .map_err(|source| BootstrapError::Bind {
1735            host: v4_host.to_string(),
1736            port,
1737            source,
1738        })?;
1739    let actual_port = v4
1740        .local_addr()
1741        .map_err(|source| BootstrapError::LocalAddr {
1742            host: v4_host.to_string(),
1743            source,
1744        })?
1745        .port();
1746
1747    let mut listeners = vec![v4];
1748    let mut endpoints = vec![Endpoint {
1749        host: v4_host.to_string(),
1750        port: actual_port,
1751    }];
1752
1753    let v6_host = Ipv6Addr::LOCALHOST;
1754    match bind_v6(actual_port).await {
1755        Ok(v6) => {
1756            listeners.push(v6);
1757            endpoints.push(Endpoint {
1758                host: v6_host.to_string(),
1759                port: actual_port,
1760            });
1761        }
1762        Err(err)
1763            if ipv6_loopback_unavailable(&err)
1764                || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
1765        {
1766            warn!(
1767                port = actual_port,
1768                error = %err,
1769                "IPv6 loopback unavailable; serving only IPv4 loopback"
1770            );
1771        }
1772        Err(source) => {
1773            drop(listeners);
1774            return Err(BootstrapError::Bind {
1775                host: v6_host.to_string(),
1776                port: actual_port,
1777                source,
1778            });
1779        }
1780    }
1781
1782    Ok((listeners, endpoints))
1783}
1784
1785fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
1786    matches!(
1787        err.kind(),
1788        io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
1789    ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
1790}
1791
1792struct AbortOnDrop<T> {
1793    handle: JoinHandle<T>,
1794}
1795
1796impl<T> AbortOnDrop<T> {
1797    fn new(handle: JoinHandle<T>) -> Self {
1798        Self { handle }
1799    }
1800
1801    async fn join(&mut self) -> Result<T, JoinError> {
1802        (&mut self.handle).await
1803    }
1804}
1805
1806impl<T> Drop for AbortOnDrop<T> {
1807    fn drop(&mut self) {
1808        if !self.handle.is_finished() {
1809            self.handle.abort();
1810        }
1811    }
1812}
1813
1814#[derive(Debug)]
1815struct StartLock {
1816    // Keep the locked file handle alive for the duration of bootstrap.
1817    _file: fs::File,
1818}
1819
1820impl Drop for StartLock {
1821    fn drop(&mut self) {
1822        // Closing alone leaves the lock held if a concurrent fork inherited
1823        // the open file description and has not exec'd yet. Unlock explicitly
1824        // so a restart can reclaim the singleton as soon as bootstrap ends.
1825        if let Err(error) = FileExt::unlock(&self._file) {
1826            tracing::warn!(%error, "failed to release the singleton start lock");
1827        }
1828    }
1829}
1830
1831impl StartLock {
1832    async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
1833        let path = start_lock_path(connection_file_path);
1834        for _ in 0..START_LOCK_RETRIES {
1835            let file = match open_owner_only_lock(&path) {
1836                Ok(file) => file,
1837                Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
1838            };
1839            match FileExt::try_lock(&file) {
1840                Ok(()) => return Ok(Self { _file: file }),
1841                Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
1842                Err(TryLockError::Error(source)) => {
1843                    return Err(BootstrapError::StartLockCreate { path, source });
1844                }
1845            }
1846        }
1847
1848        Err(BootstrapError::StartLockBusy {
1849            path,
1850            attempts: START_LOCK_RETRIES,
1851        })
1852    }
1853}
1854
1855pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
1856    let mut options = fs::OpenOptions::new();
1857    options.read(true).write(true).create(true);
1858    #[cfg(unix)]
1859    {
1860        use std::os::unix::fs::OpenOptionsExt;
1861        options.mode(0o600);
1862    }
1863    options.open(path)
1864}
1865
1866fn start_lock_path(connection_file_path: &Path) -> PathBuf {
1867    let file_name = connection_file_path
1868        .file_name()
1869        .map(|name| name.to_string_lossy())
1870        .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
1871    let lock_name = format!("{file_name}.start-lock");
1872    connection_file_path
1873        .parent()
1874        .filter(|parent| !parent.as_os_str().is_empty())
1875        .unwrap_or_else(|| Path::new("."))
1876        .join(lock_name)
1877}
1878
1879fn non_empty_os_var(key: &str) -> Option<OsString> {
1880    let value = env::var_os(key)?;
1881    if value.is_empty() {
1882        None
1883    } else {
1884        Some(value)
1885    }
1886}
1887
1888/// Bootstrap-layer errors are deliberately typed so startup never panics for
1889/// ordinary daemon-discovery races or stale filesystem state.
1890#[derive(Debug)]
1891pub enum BootstrapError {
1892    /// The bound server could not authenticate its own readiness probe.
1893    StartupNotServing,
1894    #[cfg(unix)]
1895    Signal(io::Error),
1896    InvalidPort {
1897        raw: String,
1898        source: std::num::ParseIntError,
1899    },
1900    ConnectionFileRead {
1901        path: PathBuf,
1902        source: ConnectionFileError,
1903    },
1904    ConnectionFileWrite {
1905        path: PathBuf,
1906        source: ConnectionFileError,
1907    },
1908    GenerateConnectionFile(ConnectionFileError),
1909    StartLockCreate {
1910        path: PathBuf,
1911        source: io::Error,
1912    },
1913    StartLockBusy {
1914        path: PathBuf,
1915        attempts: usize,
1916    },
1917    /// The run-directory lock file could not be created, opened or locked.
1918    RunDirLockCreate {
1919        path: PathBuf,
1920        source: io::Error,
1921    },
1922    /// Another live process, almost certainly a daemon started with a
1923    /// different runtime directory over the same data home, owns the run
1924    /// directory. The daemon does not start. `holder_pid` is read from the
1925    /// lock file when possible and is informational only.
1926    RunDirBusy {
1927        path: PathBuf,
1928        holder_pid: Option<u32>,
1929    },
1930    RemoveStale {
1931        path: PathBuf,
1932        source: io::Error,
1933    },
1934    Bind {
1935        host: String,
1936        port: u16,
1937        source: io::Error,
1938    },
1939    LocalAddr {
1940        host: String,
1941        source: io::Error,
1942    },
1943    DaemonConfig(DaemonConfigError),
1944    /// The machine id could not be established: its file is corrupt, unreadable
1945    /// or unwritable, or the data home is relative. The daemon does not start.
1946    MachineId(crate::machine_id::MachineIdFileError),
1947    /// The daemon run directory could not be resolved because the data home is
1948    /// relative. The daemon does not start.
1949    RunDir(daemon_config::DaemonRunDirError),
1950    Serve(ServerError),
1951    ServeJoin(tokio::task::JoinError),
1952}
1953
1954impl From<ServerError> for BootstrapError {
1955    fn from(error: ServerError) -> Self {
1956        Self::Serve(error)
1957    }
1958}
1959
1960impl fmt::Display for BootstrapError {
1961    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1962        match self {
1963            Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
1964            #[cfg(unix)]
1965            Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
1966            Self::InvalidPort { raw, source } => {
1967                write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
1968            }
1969            Self::ConnectionFileRead { path, source } => write!(
1970                f,
1971                "failed to read connection file {}: {source}",
1972                path.display()
1973            ),
1974            Self::ConnectionFileWrite { path, source } => write!(
1975                f,
1976                "failed to publish connection file {}: {source}",
1977                path.display()
1978            ),
1979            Self::GenerateConnectionFile(err) => {
1980                write!(f, "failed to generate connection-file auth material: {err}")
1981            }
1982            Self::StartLockCreate { path, source } => {
1983                write!(
1984                    f,
1985                    "failed to create start lock {}: {source}",
1986                    path.display()
1987                )
1988            }
1989            Self::StartLockBusy { path, attempts } => write!(
1990                f,
1991                "start lock {} remained busy after {attempts} attempts",
1992                path.display()
1993            ),
1994            Self::RunDirLockCreate { path, source } => write!(
1995                f,
1996                "refusing to start: failed to lock run directory via {}: {source}",
1997                path.display()
1998            ),
1999            Self::RunDirBusy { path, holder_pid } => {
2000                write!(
2001                    f,
2002                    "refusing to start: run directory lock {} is held by another daemon",
2003                    path.display()
2004                )?;
2005                if let Some(pid) = holder_pid {
2006                    write!(f, " (pid {pid})")?;
2007                }
2008                write!(
2009                    f,
2010                    "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
2011                )
2012            }
2013            Self::RemoveStale { path, source } => write!(
2014                f,
2015                "failed to remove stale connection file {}: {source}",
2016                path.display()
2017            ),
2018            Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
2019                write!(
2020                    f,
2021                    "port {port} in use on loopback {host}: {source}; set the port in config"
2022                )
2023            }
2024            Self::Bind { host, port, source } => {
2025                write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
2026            }
2027            Self::LocalAddr { host, source } => {
2028                write!(f, "failed to read local address for {host}: {source}")
2029            }
2030            Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
2031            Self::MachineId(err) => write!(f, "refusing to start: {err}"),
2032            Self::RunDir(err) => write!(f, "refusing to start: {err}"),
2033            Self::Serve(err) => write!(f, "daemon server failed: {err}"),
2034            Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
2035        }
2036    }
2037}
2038
2039impl Error for BootstrapError {
2040    fn source(&self) -> Option<&(dyn Error + 'static)> {
2041        match self {
2042            Self::StartupNotServing => None,
2043            #[cfg(unix)]
2044            Self::Signal(source) => Some(source),
2045            Self::InvalidPort { source, .. } => Some(source),
2046            Self::ConnectionFileRead { source, .. }
2047            | Self::ConnectionFileWrite { source, .. }
2048            | Self::GenerateConnectionFile(source) => Some(source),
2049            Self::StartLockCreate { source, .. }
2050            | Self::RunDirLockCreate { source, .. }
2051            | Self::RemoveStale { source, .. }
2052            | Self::Bind { source, .. }
2053            | Self::LocalAddr { source, .. } => Some(source),
2054            Self::DaemonConfig(err) => Some(err),
2055            Self::MachineId(err) => Some(err),
2056            Self::RunDir(err) => Some(err),
2057            Self::Serve(err) => Some(err),
2058            Self::ServeJoin(err) => Some(err),
2059            Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
2060        }
2061    }
2062}
2063
2064#[cfg(test)]
2065mod tests {
2066    mod activity {
2067        include!("activity_bootstrap_tests.rs");
2068    }
2069    mod startup_order {
2070        include!("startup_order_tests.rs");
2071    }
2072    use super::*;
2073    use crate::server::ServerAuth;
2074    use cortexkit_test_support::ScratchDir;
2075    #[cfg(target_os = "linux")]
2076    use std::collections::BTreeSet;
2077    use std::sync::Mutex;
2078    #[cfg(target_os = "linux")]
2079    use subc_control::ModuleProtocol;
2080    use subc_transport::{write_atomic, MIN_KEY_LEN};
2081    use tokio::io::AsyncReadExt;
2082    use tokio::task::JoinHandle;
2083
2084    #[cfg(unix)]
2085    use std::os::unix::fs::PermissionsExt;
2086
2087    static ENV_LOCK: Mutex<()> = Mutex::new(());
2088
2089    #[test]
2090    fn normalized_build_provenance_preserves_real_values() {
2091        assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
2092    }
2093
2094    #[test]
2095    fn normalized_build_provenance_omits_unavailable_and_empty_values() {
2096        assert_eq!(normalized_build_provenance("unavailable"), None);
2097        assert_eq!(normalized_build_provenance(""), None);
2098    }
2099
2100    #[cfg(target_os = "linux")]
2101    fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
2102        let cgroups = fs::read_to_string("/proc/self/cgroup")?;
2103        let relative = cgroups
2104            .lines()
2105            .find_map(|line| line.strip_prefix("0::"))
2106            .ok_or_else(|| {
2107                io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
2108            })?;
2109        Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
2110    }
2111
2112    #[cfg(target_os = "linux")]
2113    fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
2114        let modules = current_cgroup_path_for_test()?.join("subc-modules");
2115        let entries = match fs::read_dir(modules) {
2116            Ok(entries) => entries,
2117            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
2118            Err(error) => return Err(error),
2119        };
2120        let mut directories = BTreeSet::new();
2121        for entry in entries {
2122            let entry = entry?;
2123            if entry.file_type()?.is_dir() {
2124                directories.insert(entry.file_name());
2125            }
2126        }
2127        Ok(directories)
2128    }
2129
2130    #[cfg(target_os = "linux")]
2131    async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
2132        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2133        while !path.exists() && tokio::time::Instant::now() < deadline {
2134            assert!(
2135                !task.is_finished(),
2136                "daemon exited before creating {}",
2137                path.display()
2138            );
2139            sleep(Duration::from_millis(10)).await;
2140        }
2141        assert!(path.exists(), "daemon did not create {}", path.display());
2142    }
2143
2144    /// An in-process daemon with the default (disabled) cgroup placement must
2145    /// leave the host's live module cgroup tree exactly as it found it.
2146    /// Red-checking this test (making it fail) creates a
2147    /// `cgroup-isolation-probe-*` directory in the live cgroup tree, which must be removed by hand.
2148    #[cfg(target_os = "linux")]
2149    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2150    async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
2151        let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
2152        let module_id = format!("cgroup-isolation-probe-{}", process::id());
2153        let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
2154        assert!(
2155            !before.contains(&OsString::from(&module_id)),
2156            "isolation probe cgroup already exists before this daemon starts"
2157        );
2158        let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
2159        let module = ConfiguredModule {
2160            carry_containment: false,
2161            module_id,
2162            program: PathBuf::from("sh"),
2163            args: vec!["-c".to_string(), "sleep 30".to_string()],
2164            env: Vec::new(),
2165            log: None,
2166            enabled: true,
2167            reserved: false,
2168            reserved_prefixes: Vec::new(),
2169            protocol: ModuleProtocol::None,
2170            overlap: Default::default(),
2171            health: HealthConfig::default(),
2172            drain_timeout_ms: None,
2173            register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2174            route_bind_relay_timeout_ms: None,
2175            restart: RestartPolicy::default(),
2176        };
2177        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2178            .with_configured_modules([module])
2179            .with_capture_logs_dir(temp.join("logs"))
2180            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2181        let task = tokio::spawn(run_with_config(config));
2182
2183        wait_for_path(&capture, &task).await;
2184        let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2185
2186        task.abort();
2187        assert!(task
2188            .await
2189            .expect_err("aborted daemon task must cancel")
2190            .is_cancelled());
2191        assert_eq!(
2192            before, after,
2193            "an in-process daemon must not create or reconcile ambient module cgroups"
2194        );
2195    }
2196
2197    #[cfg(target_os = "linux")]
2198    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2199    async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2200        let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2201        let cgroup_root = temp.join("cgroup");
2202        fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2203        fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2204        let modules = cgroup_root.join("subc-modules");
2205        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2206            .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2207            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2208        let task = tokio::spawn(run_with_config(config));
2209
2210        wait_for_path(&modules, &task).await;
2211
2212        task.abort();
2213        assert!(task
2214            .await
2215            .expect_err("aborted daemon task must cancel")
2216            .is_cancelled());
2217        assert!(
2218            fs::read_dir(&modules)
2219                .expect("read prepared modules directory")
2220                .next()
2221                .is_none(),
2222            "the delegation probe must clean up after itself"
2223        );
2224    }
2225
2226    struct EnvGuard {
2227        key: &'static str,
2228        previous: Option<OsString>,
2229    }
2230
2231    impl EnvGuard {
2232        fn set(key: &'static str, value: &Path) -> Self {
2233            let previous = env::var_os(key);
2234            env::set_var(key, value);
2235            Self { key, previous }
2236        }
2237
2238        fn set_str(key: &'static str, value: &str) -> Self {
2239            let previous = env::var_os(key);
2240            env::set_var(key, value);
2241            Self { key, previous }
2242        }
2243
2244        fn unset(key: &'static str) -> Self {
2245            let previous = env::var_os(key);
2246            env::remove_var(key);
2247            Self { key, previous }
2248        }
2249    }
2250
2251    impl Drop for EnvGuard {
2252        fn drop(&mut self) {
2253            match &self.previous {
2254                Some(value) => env::set_var(self.key, value),
2255                None => env::remove_var(self.key),
2256            }
2257        }
2258    }
2259
2260    fn unique_temp_dir(name: &str) -> ScratchDir {
2261        ScratchDir::new(name)
2262    }
2263
2264    fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2265        let dir = unique_temp_dir(name);
2266        let path = dir.join("conn.json");
2267        (dir, path)
2268    }
2269
2270    fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2271        ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2272    }
2273
2274    fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2275        let auth = auth_for(&bound.connection_info);
2276        tokio::spawn(serve_listeners(
2277            bound.listeners,
2278            Arc::new(Router::with_default_self_handler()),
2279            auth,
2280        ))
2281    }
2282
2283    fn expect_bound(outcome: Outcome) -> BoundDaemon {
2284        match outcome {
2285            Outcome::Bound(bound) => bound,
2286            Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2287        }
2288    }
2289
2290    async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2291        let endpoint = conn
2292            .endpoints
2293            .first()
2294            .expect("test connection file should have an endpoint");
2295        let ip: IpAddr = endpoint.host.parse().unwrap();
2296        TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2297    }
2298
2299    fn make_connection_info(port: u16) -> ConnectionInfo {
2300        ConnectionInfo {
2301            schema: SCHEMA_VERSION,
2302            wire_version: Some(PROTOCOL_VERSION),
2303            endpoints: vec![Endpoint {
2304                host: "127.0.0.1".to_owned(),
2305                port,
2306            }],
2307            key: generate_key().unwrap(),
2308            daemon_id: generate_daemon_id().unwrap(),
2309            pid: process::id(),
2310            daemon_ver: "test-subc".to_owned(),
2311        }
2312    }
2313
2314    fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2315        fs::write(path, contents).unwrap();
2316        #[cfg(unix)]
2317        fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2318    }
2319
2320    fn assert_owner_only_connection_file(path: &Path) {
2321        // `path` is only inspected on Unix (mode bits); on Windows the owner-only
2322        // guarantee comes from the inherited %TEMP% ACL, nothing to assert here.
2323        #[cfg(unix)]
2324        {
2325            let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2326            assert_eq!(mode, 0o600);
2327        }
2328        #[cfg(not(unix))]
2329        let _ = path;
2330    }
2331
2332    #[test]
2333    fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2334        let _env_lock = ENV_LOCK.lock().unwrap();
2335        let runtime_dir = unique_temp_dir("xdg-runtime");
2336        let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2337
2338        assert_eq!(
2339            connection_file_path(),
2340            runtime_dir.join(CONNECTION_FILE_NAME)
2341        );
2342    }
2343
2344    #[test]
2345    fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2346        let runtime_dir = OsString::from("/run/user/1000");
2347
2348        let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2349
2350        assert_eq!(
2351            path,
2352            PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2353        );
2354        assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2355    }
2356
2357    #[test]
2358    fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2359        let _env_lock = ENV_LOCK.lock().unwrap();
2360        let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2361
2362        assert_eq!(
2363            connection_file_path(),
2364            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2365        );
2366    }
2367
2368    #[test]
2369    fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2370        let (path, source) = connection_file_path_with_source(None);
2371
2372        assert_eq!(
2373            path,
2374            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2375        );
2376        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2377    }
2378
2379    /// Concurrent callers must derive one token. The former temp-file uid probe
2380    /// could fail transiently (same-tick name collision, fd exhaustion) and send
2381    /// the loser down the env-derived fallback with a different identity for the
2382    /// same user; this fence keeps identity independent of filesystem luck.
2383    #[test]
2384    fn user_connection_token_is_stable_under_concurrent_callers() {
2385        let expected = user_connection_token();
2386        let workers: Vec<_> = (0..32)
2387            .map(|_| {
2388                std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2389            })
2390            .collect();
2391        for worker in workers {
2392            for token in worker.join().expect("probe thread") {
2393                assert_eq!(token, expected, "token diverged under concurrent probes");
2394            }
2395        }
2396    }
2397
2398    #[test]
2399    fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
2400        let (path, source) = connection_file_path_with_source(Some(OsString::new()));
2401
2402        assert_eq!(
2403            path,
2404            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2405        );
2406        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2407    }
2408
2409    /// The shipped binary is the only caller that journals terminal exits into
2410    /// the real run directory, so its constructor must supply that path itself;
2411    /// the in-process default leaves it unset.
2412    #[test]
2413    fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
2414        let _env_lock = ENV_LOCK.lock().unwrap();
2415        let root = unique_temp_dir("daemon-binary-config");
2416        let data_home = root.join("data");
2417        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
2418        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2419        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2420
2421        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
2422
2423        let run_dir = data_home.join("cortexkit").join("run");
2424        assert_eq!(
2425            config.terminal_journal_path,
2426            Some(run_dir.join("terminals.jsonl"))
2427        );
2428        assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
2429        assert_eq!(
2430            BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
2431            None,
2432            "an in-process config must not journal anywhere unless asked to"
2433        );
2434    }
2435
2436    #[test]
2437    fn daemon_binary_config_refuses_a_relative_data_home() {
2438        let _env_lock = ENV_LOCK.lock().unwrap();
2439        let root = unique_temp_dir("daemon-binary-relative-data");
2440        let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
2441        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2442        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2443
2444        let error = BootstrapConfig::from_env_for_daemon_binary()
2445            .expect_err("a relative data home must refuse the daemon binary's config");
2446        assert!(
2447            matches!(error, BootstrapError::RunDir(_)),
2448            "expected a run-directory refusal, got {error}"
2449        );
2450        assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
2451    }
2452
2453    #[test]
2454    fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
2455        let _env_lock = ENV_LOCK.lock().unwrap();
2456        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2457        let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
2458        let config_path = conn_path.with_file_name("subc.jsonc");
2459        for (doc, expected) in [
2460            (None, daemon_config::PrivilegedOperatorPrompt::Log),
2461            (
2462                Some(r#"{"version":1}"#),
2463                daemon_config::PrivilegedOperatorPrompt::Log,
2464            ),
2465            (
2466                Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
2467                daemon_config::PrivilegedOperatorPrompt::Log,
2468            ),
2469            (
2470                Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
2471                daemon_config::PrivilegedOperatorPrompt::Enforce,
2472            ),
2473        ] {
2474            if let Some(doc) = doc {
2475                fs::write(&config_path, doc).unwrap();
2476            }
2477            assert_eq!(
2478                BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2479                    .unwrap()
2480                    .privileged_operator_prompt,
2481                expected
2482            );
2483            assert_eq!(
2484                BootstrapConfig::new(&conn_path, 0)
2485                    .with_daemon_config_path(&config_path)
2486                    .unwrap()
2487                    .privileged_operator_prompt,
2488                expected
2489            );
2490        }
2491    }
2492
2493    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2494    async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
2495        let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
2496        let connection_path = dir.join("connection.json");
2497        let config_path = dir.join("subc.jsonc");
2498        fs::write(
2499            &config_path,
2500            r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
2501        )
2502        .unwrap();
2503        let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
2504            .with_daemon_config_path(&config_path)
2505            .unwrap();
2506        let connection_path = config.connection_file_path.clone();
2507        let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
2508        tokio::time::timeout(Duration::from_secs(5), async {
2509            while !connection_path.exists() {
2510                assert!(
2511                    !daemon.handle.is_finished(),
2512                    "daemon exited before publishing"
2513                );
2514                tokio::task::yield_now().await;
2515            }
2516        })
2517        .await
2518        .unwrap();
2519        let info = connection_file::read(&connection_path).unwrap();
2520        let mut client = connect_from_info(&info).await.unwrap();
2521        authenticate_client(&mut client, &info, Duration::from_secs(5))
2522            .await
2523            .unwrap();
2524        for (correlation, mode, expected) in [
2525            (1, "enforce", Vec::new()),
2526            (2, "log", vec!["privileged_operator_prompt"]),
2527            (3, "log", vec!["privileged_operator_prompt"]),
2528        ] {
2529            fs::write(
2530                &config_path,
2531                serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
2532            )
2533            .unwrap();
2534            let frame = crate::Frame::build(
2535                subc_protocol::FrameType::Request,
2536                subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
2537                0,
2538                0,
2539                correlation,
2540                serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
2541                    preview: true,
2542                })
2543                .unwrap(),
2544            )
2545            .unwrap();
2546            crate::write_frame(&mut client, &frame).await.unwrap();
2547            let reply =
2548                tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
2549                    .await
2550                    .unwrap()
2551                    .unwrap()
2552                    .unwrap();
2553            assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
2554            assert_eq!(reply.header.corr, correlation);
2555            let subc_control::ClientControlResponse::SupervisorRescan { result } =
2556                serde_json::from_slice(&reply.body).unwrap()
2557            else {
2558                panic!("expected a rescan response");
2559            };
2560            assert_eq!(result.restart_required, expected);
2561        }
2562    }
2563
2564    #[test]
2565    fn configured_port_uses_default_config_and_env_override() {
2566        let _env_lock = ENV_LOCK.lock().unwrap();
2567        let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
2568        let config_path = conn_path.with_file_name("subc.jsonc");
2569
2570        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2571        assert_eq!(
2572            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2573                .unwrap()
2574                .port,
2575            DEFAULT_SUBC_PORT
2576        );
2577
2578        fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
2579        assert_eq!(
2580            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2581                .unwrap()
2582                .port,
2583            8123
2584        );
2585
2586        let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
2587        assert_eq!(
2588            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2589                .unwrap()
2590                .port,
2591            9012
2592        );
2593    }
2594
2595    #[tokio::test]
2596    async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
2597        let (_dir, path) = temp_connection_file_path("departing-peer");
2598        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2599        let info = make_connection_info(listener.local_addr().unwrap().port());
2600        write_connection_file(&path, &info, None).unwrap();
2601        let peer = tokio::spawn(async move {
2602            let (mut stream, _) = listener.accept().await.unwrap();
2603            subc_transport::authenticate_server(
2604                &mut stream,
2605                &info.key,
2606                &info.daemon_id,
2607                &info.daemon_ver,
2608                PROBE_AUTH_DEADLINE,
2609            )
2610            .await
2611            .unwrap();
2612        });
2613        let outcome = ensure_singleton(&path, 0).await.unwrap();
2614        peer.await.unwrap();
2615        assert!(
2616            matches!(outcome, Outcome::Bound(_)),
2617            "a departing peer is not AlreadyRunning"
2618        );
2619    }
2620
2621    #[tokio::test]
2622    async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
2623        let (_dir, path) = temp_connection_file_path("already-running");
2624
2625        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2626        let server = start_server(bound);
2627
2628        let second = ensure_singleton(&path, 0).await.unwrap();
2629        assert!(matches!(second, Outcome::AlreadyRunning));
2630
2631        server.abort();
2632        let _ = server.await;
2633    }
2634
2635    #[tokio::test]
2636    async fn daemon_connection_file_publishes_protocol_wire_version() {
2637        let (_dir, path) = temp_connection_file_path("wire-version");
2638        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2639        assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
2640        assert_eq!(
2641            connection_file::read(&path).unwrap().wire_version,
2642            Some(PROTOCOL_VERSION)
2643        );
2644
2645        drop(bound.listeners);
2646    }
2647
2648    #[tokio::test]
2649    async fn stale_unbound_connection_file_is_reclaimed() {
2650        let (_dir, path) = temp_connection_file_path("stale-reclaim");
2651        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2652        let stale_port = stale.local_addr().unwrap().port();
2653        drop(stale);
2654        let stale_info = make_connection_info(stale_port);
2655        write_atomic(&path, &stale_info).unwrap();
2656
2657        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2658        assert_ne!(bound.connection_info.key, stale_info.key);
2659        drop(bound.listeners);
2660    }
2661
2662    #[cfg(unix)]
2663    #[tokio::test]
2664    async fn ensure_singleton_reclaims_insecure_connection_file() {
2665        let (_dir, path) = temp_connection_file_path("insecure-reclaim");
2666        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2667        let stale_port = stale.local_addr().unwrap().port();
2668        drop(stale);
2669        let stale_info = make_connection_info(stale_port);
2670        write_atomic(&path, &stale_info).unwrap();
2671        fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
2672
2673        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2674        assert_ne!(bound.connection_info.key, stale_info.key);
2675        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2676        assert_owner_only_connection_file(&path);
2677
2678        drop(bound.listeners);
2679    }
2680
2681    #[tokio::test]
2682    async fn ensure_singleton_reclaims_non_loopback_connection_file() {
2683        let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
2684        let mut stale_info = make_connection_info(8757);
2685        stale_info.endpoints = vec![Endpoint {
2686            host: "192.0.2.10".to_owned(),
2687            port: 8757,
2688        }];
2689        write_atomic(&path, &stale_info).unwrap();
2690
2691        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2692        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
2693        assert_ne!(bound.connection_info.key, stale_info.key);
2694        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2695        assert!(bound
2696            .connection_info
2697            .endpoints
2698            .iter()
2699            .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
2700        assert_owner_only_connection_file(&path);
2701
2702        drop(bound.listeners);
2703    }
2704
2705    #[tokio::test]
2706    async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
2707        let mut unsupported_schema = make_connection_info(8757);
2708        unsupported_schema.schema = SCHEMA_VERSION + 1;
2709
2710        let mut empty_endpoints = make_connection_info(8757);
2711        empty_endpoints.endpoints.clear();
2712
2713        let mut short_key = make_connection_info(8757);
2714        short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
2715
2716        let cases = vec![
2717            (
2718                "unsupported-schema",
2719                serde_json::to_vec(&unsupported_schema).unwrap(),
2720                Some(unsupported_schema),
2721            ),
2722            (
2723                "empty-endpoints",
2724                serde_json::to_vec(&empty_endpoints).unwrap(),
2725                Some(empty_endpoints),
2726            ),
2727            (
2728                "short-key",
2729                serde_json::to_vec(&short_key).unwrap(),
2730                Some(short_key),
2731            ),
2732            ("invalid-json", b"{not valid connection json".to_vec(), None),
2733        ];
2734
2735        for (label, contents, old_info) in cases {
2736            let (_dir, path) = temp_connection_file_path(label);
2737            write_raw_owner_only_connection_file(&path, &contents);
2738
2739            let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2740            if let Some(old_info) = old_info {
2741                assert_ne!(bound.connection_info.key, old_info.key, "{label}");
2742                assert_ne!(
2743                    bound.connection_info.daemon_id, old_info.daemon_id,
2744                    "{label}"
2745                );
2746            }
2747            assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
2748            assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
2749            assert_owner_only_connection_file(&path);
2750
2751            drop(bound.listeners);
2752        }
2753    }
2754
2755    #[tokio::test]
2756    async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
2757        let (_dir, path) = temp_connection_file_path("foreign-reclaim");
2758        let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2759        let foreign_port = foreign.local_addr().unwrap().port();
2760        write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
2761        let foreign_task = tokio::spawn(async move {
2762            if let Ok((mut stream, _)) = foreign.accept().await {
2763                let mut buf = [0u8; 64];
2764                let _ = stream.read(&mut buf).await;
2765            }
2766        });
2767
2768        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2769        assert!(bound
2770            .connection_info
2771            .endpoints
2772            .iter()
2773            .all(|endpoint| endpoint.port != foreign_port));
2774
2775        drop(bound.listeners);
2776        let _ = foreign_task.await;
2777    }
2778
2779    #[tokio::test]
2780    async fn stale_start_lock_file_is_reclaimable() {
2781        let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
2782        let lock_path = start_lock_path(&path);
2783        drop(open_owner_only_lock(&lock_path).unwrap());
2784        assert!(lock_path.is_file());
2785
2786        let lock = StartLock::acquire(&path).await.unwrap();
2787        assert!(lock_path.is_file());
2788
2789        drop(lock);
2790        assert!(lock_path.is_file());
2791    }
2792
2793    #[tokio::test]
2794    async fn held_start_lock_blocks_second_acquire_until_release() {
2795        let (_dir, path) = temp_connection_file_path("start-lock-held");
2796        let lock_path = start_lock_path(&path);
2797        let first = StartLock::acquire(&path).await.unwrap();
2798
2799        let err = match StartLock::acquire(&path).await {
2800            Ok(_) => panic!("second acquire while held must stay busy"),
2801            Err(err) => err,
2802        };
2803        assert!(matches!(
2804            err,
2805            BootstrapError::StartLockBusy {
2806                ref path,
2807                attempts: START_LOCK_RETRIES,
2808            } if path == &lock_path
2809        ));
2810
2811        drop(first);
2812
2813        let second = StartLock::acquire(&path)
2814            .await
2815            .expect("released advisory lock should be reclaimable");
2816        drop(second);
2817    }
2818
2819    #[tokio::test]
2820    async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
2821        let (_dir, path) = temp_connection_file_path("bind-conflict");
2822        let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2823        let occupied_port = occupied.local_addr().unwrap().port();
2824
2825        let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
2826        assert!(matches!(
2827            err,
2828            BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
2829        ));
2830        assert!(err.to_string().contains("set the port in config"));
2831
2832        drop(occupied);
2833    }
2834
2835    #[tokio::test]
2836    async fn ephemeral_ipv6_collision_still_serves_ipv4() {
2837        let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
2838            let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
2839            let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
2840            drop(occupied);
2841            result
2842        })
2843        .await
2844        .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
2845        assert_eq!(endpoints.len(), 1);
2846        assert_eq!(endpoints[0].host, "127.0.0.1");
2847        TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
2848            .await
2849            .unwrap();
2850        drop(listeners);
2851    }
2852
2853    #[tokio::test]
2854    async fn unpublished_boot_holds_singleton_lock_until_publication() {
2855        let dir = ScratchDir::new("ul");
2856        let path = dir.join("c");
2857        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2858        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2859        assert!(!path.exists());
2860        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2861        assert!(
2862            matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
2863            "an unpublished boot must keep its singleton claim"
2864        );
2865        drop(bound);
2866        FileExt::try_lock(&contender).unwrap();
2867    }
2868
2869    #[tokio::test]
2870    async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
2871        let dir = ScratchDir::new("sl");
2872        let path = dir.join("c");
2873        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2874        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2875        // try_clone shares the open file description just as fork does,
2876        // without relying on the timing of another test's child spawn.
2877        let inherited = bound
2878            .publication_lock
2879            .as_ref()
2880            .unwrap()
2881            ._file
2882            .try_clone()
2883            .unwrap();
2884        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2885        assert!(matches!(
2886            FileExt::try_lock(&contender),
2887            Err(TryLockError::WouldBlock)
2888        ));
2889        drop(bound);
2890        FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
2891        drop(inherited);
2892    }
2893
2894    #[tokio::test]
2895    async fn readiness_probe_requires_an_authenticating_server() {
2896        let dir = ScratchDir::new("unserved-readiness");
2897        let bound = expect_bound(
2898            ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
2899                .await
2900                .unwrap(),
2901        );
2902        assert!(matches!(
2903            verify_serving(&bound.connection_info).await,
2904            Err(BootstrapError::StartupNotServing)
2905        ));
2906    }
2907
2908    #[cfg(unix)]
2909    #[tokio::test]
2910    async fn orphan_sweep_finishes_before_connection_file_publication() {
2911        use crate::live_children::{ExecutableIdentity, LiveChild};
2912        let dir = ScratchDir::new("pas");
2913        // A one-letter connection file name, and no long socket path: the sweep
2914        // this test covers runs before the local endpoint is bound, so a longer
2915        // name would only add a way for the test to fail on something else.
2916        let path = dir.join("c");
2917        let record = dir.join("live-children.json");
2918        let ready = dir.join("ready");
2919        let mut child = std::process::Command::new("/bin/sh")
2920            .args([
2921                "-c",
2922                "trap '' TERM; : > \"$0\"; read -r _",
2923                ready.to_str().unwrap(),
2924            ])
2925            .stdin(std::process::Stdio::piped())
2926            .env("XDG_DATA_HOME", dir.path())
2927            .env("XDG_RUNTIME_DIR", dir.path())
2928            .env("XDG_CONFIG_HOME", dir.path())
2929            .spawn()
2930            .unwrap();
2931        timeout(Duration::from_secs(5), async {
2932            while !ready.exists() {
2933                sleep(Duration::from_millis(5)).await;
2934            }
2935        })
2936        .await
2937        .unwrap();
2938        let observed = subc_os::Process::open(child.id())
2939            .unwrap()
2940            .unwrap()
2941            .observe()
2942            .unwrap();
2943        crate::live_children::write_record(
2944            &record,
2945            &[LiveChild {
2946                container: None,
2947                module_id: "old-sleep".into(),
2948                pid: child.id(),
2949                protocol: subc_control::ModuleProtocol::None,
2950                start_time: Some(observed.start_time),
2951                executable: observed.executable.map(ExecutableIdentity::from),
2952                cgroup_name: None,
2953            }],
2954        )
2955        .unwrap();
2956        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
2957            .with_live_children_record(&record);
2958        let daemon = tokio::spawn(run_with_config(config));
2959        timeout(Duration::from_secs(5), async {
2960            while !path.exists() {
2961                sleep(Duration::from_millis(5)).await;
2962            }
2963        })
2964        .await
2965        .unwrap();
2966        assert!(path.exists());
2967        let ended = child.try_wait().unwrap().is_some();
2968        if !ended {
2969            child.kill().unwrap();
2970        }
2971        child.wait().unwrap();
2972        if ended {
2973            assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
2974        }
2975        daemon.abort();
2976        let _ = daemon.await;
2977        assert!(
2978            ended,
2979            "the published daemon must not still owe its orphan sweep"
2980        );
2981    }
2982
2983    /// Publishing the connection file fsyncs, which can take seconds on a busy
2984    /// disk, so the daemon must do it off the async worker thread. The hook
2985    /// blocks inside the write until another task on the same single-threaded
2986    /// runtime has run. Writing on the worker thread would starve that task: the
2987    /// hook then gives up after its deadline and the test fails, as it does if
2988    /// startup publishes without going through the hooked write at all.
2989    #[tokio::test(flavor = "current_thread")]
2990    async fn connection_file_publication_leaves_the_async_runtime_free() {
2991        use std::sync::atomic::{AtomicBool, Ordering};
2992
2993        let (_dir, path) = temp_connection_file_path("publish-off-runtime");
2994        let write_started = Arc::new(AtomicBool::new(false));
2995        let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
2996        let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
2997        let hook_started = Arc::clone(&write_started);
2998        connection_file_write_hook::install(&path, move || {
2999            hook_started.store(true, Ordering::SeqCst);
3000            let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
3001            let _ = verdict_tx.send(progressed);
3002        });
3003        // Signals only after the write has started, so a message can only come
3004        // from this task running while the write is still blocked.
3005        let ticker = tokio::spawn(async move {
3006            loop {
3007                if write_started.load(Ordering::SeqCst) {
3008                    let _ = progress_tx.send(());
3009                    return;
3010                }
3011                sleep(Duration::from_millis(5)).await;
3012            }
3013        });
3014
3015        let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
3016        let published = timeout(Duration::from_secs(15), async {
3017            while !path.exists() && !daemon.is_finished() {
3018                sleep(Duration::from_millis(5)).await;
3019            }
3020        })
3021        .await;
3022        let daemon_exited = daemon.is_finished();
3023        daemon.abort();
3024        let daemon_result = daemon.await;
3025        ticker.abort();
3026
3027        assert!(
3028            published.is_ok() && !daemon_exited,
3029            "the daemon must start and publish its connection file: {daemon_result:?}"
3030        );
3031        match verdict_rx.try_recv() {
3032            Ok(true) => {}
3033            Ok(false) => panic!(
3034                "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
3035            ),
3036            Err(_) => panic!("startup published the connection file without the hooked write"),
3037        }
3038    }
3039
3040    /// A write that never reports back must stop startup with an error naming
3041    /// the connection file, never be taken as published.
3042    #[tokio::test]
3043    async fn a_panicking_connection_file_write_fails_startup() {
3044        let (_dir, path) = temp_connection_file_path("publish-panics");
3045        connection_file_write_hook::install(&path, || {
3046            panic!("deliberate panic inside the connection-file write")
3047        });
3048
3049        let error = timeout(
3050            Duration::from_secs(15),
3051            run_with_config(BootstrapConfig::new(&path, 0)),
3052        )
3053        .await
3054        .expect("startup must end once the write task fails")
3055        .expect_err("startup must fail when the connection-file write panics");
3056
3057        assert!(
3058            matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
3059            "expected a connection-file write error, got {error:?}"
3060        );
3061        let message = error.to_string();
3062        assert!(message.contains(&path.display().to_string()), "{message}");
3063        assert!(message.contains("panicked"), "{message}");
3064        assert!(!path.exists(), "no connection file may be published");
3065    }
3066
3067    #[cfg(target_os = "macos")]
3068    #[test]
3069    fn nofile_raise_worker() {
3070        let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
3071            return;
3072        };
3073        let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
3074        rlimit::Resource::NOFILE.set(256, hard).unwrap();
3075        raise_nofile_limit_with_ceiling(Some(10240));
3076        std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
3077    }
3078
3079    #[cfg(target_os = "macos")]
3080    #[test]
3081    fn macos_nofile_raise_clamps_to_kernel_ceiling() {
3082        let dir = ScratchDir::new("nofile-ceiling");
3083        let result = dir.join("soft");
3084        let status = std::process::Command::new(std::env::current_exe().unwrap())
3085            .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
3086            .env("SUBC_NOFILE_TEST_FILE", &result)
3087            .env("XDG_DATA_HOME", dir.path())
3088            .env("XDG_RUNTIME_DIR", dir.path())
3089            .env("XDG_CONFIG_HOME", dir.path())
3090            .status()
3091            .unwrap();
3092        assert!(status.success());
3093        let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
3094        assert!(
3095            soft > 256 && soft <= 10240,
3096            "the inherited limit must be raised within the kernel ceiling; got {soft}"
3097        );
3098    }
3099
3100    #[tokio::test]
3101    async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
3102        let (_dir, path) = temp_connection_file_path("key-rotation");
3103        let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3104        let old_info = first.connection_info.clone();
3105        let fixed_port = old_info.endpoints[0].port;
3106        drop(first.listeners);
3107
3108        let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
3109        let new_info = second.connection_info.clone();
3110        assert_ne!(old_info.key, new_info.key);
3111        assert_ne!(old_info.daemon_id, new_info.daemon_id);
3112        let server = start_server(second);
3113
3114        let mut old_stream = connect_from_info(&old_info).await.unwrap();
3115        let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
3116        assert!(
3117            old_auth.is_err(),
3118            "old key must not authenticate after restart"
3119        );
3120
3121        let reread = connection_file::read(&path).unwrap();
3122        let mut new_stream = connect_from_info(&reread).await.unwrap();
3123        authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
3124            .await
3125            .unwrap();
3126
3127        server.abort();
3128        let _ = server.await;
3129    }
3130
3131    #[cfg(unix)]
3132    #[tokio::test]
3133    async fn published_connection_file_permissions_are_owner_only() {
3134        let (_dir, path) = temp_connection_file_path("permissions");
3135        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3136
3137        let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
3138        assert_eq!(mode, 0o600);
3139
3140        drop(bound.listeners);
3141    }
3142
3143    /// A config with the socket in `dir`'s own short subdirectory.
3144    ///
3145    /// A Unix socket path is limited to just over a hundred bytes, and this
3146    /// harness's temporary directories use nearly all of that, so a test that
3147    /// boots a real daemon passes a short directory and a one-letter connection
3148    /// file name; anything longer makes the bind refuse. On Windows there is no
3149    /// socket path to keep short, so the config is returned unchanged.
3150    fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
3151        let dir = config
3152            .connection_file_path
3153            .parent()
3154            .map(Path::to_path_buf)
3155            .unwrap_or_else(|| PathBuf::from("."));
3156        #[cfg(unix)]
3157        let config = BootstrapConfig {
3158            connection_file_path: dir.join("c"),
3159            local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
3160            ..config
3161        };
3162        #[cfg(not(unix))]
3163        let _ = dir;
3164        config
3165    }
3166
3167    /// The shipped binary puts the socket in its own run directory. The file is
3168    /// named from the connection file's path, so a restart finds it again.
3169    #[cfg(unix)]
3170    #[test]
3171    fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
3172        let _env_lock = ENV_LOCK.lock().unwrap();
3173        let root = ScratchDir::new("binary-local-socket");
3174        let data_home = root.join("data");
3175        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3176        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3177        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3178
3179        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3180
3181        let run_dir = data_home.join("cortexkit").join("run");
3182        assert_eq!(
3183            config.local_listener.configured_dir(),
3184            Some(run_dir.as_path()),
3185            "the binary serves from its run directory"
3186        );
3187        let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
3188            .expect("the binary config names a directory");
3189        assert_eq!(socket.parent().unwrap(), run_dir);
3190        assert_eq!(
3191            socket.file_name().unwrap().to_string_lossy(),
3192            "subc.sock",
3193            "a fixed name: the run directory belongs to one data home"
3194        );
3195    }
3196
3197    /// An in-process daemon that names no directory serves TCP only: it publishes
3198    /// no local endpoint, creates nothing under the data home, and says nothing is
3199    /// unavailable, because nothing was asked for.
3200    /// The environment lock covers only the setup: the daemon under test reads
3201    /// its data home from the process environment while it boots, and holding
3202    /// the lock across that await would block every other test that sets one.
3203    #[cfg(unix)]
3204    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3205    async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
3206        let root = ScratchDir::new("in-process-no-local-socket");
3207        let data_home = root.join("data");
3208        fs::create_dir_all(&data_home).unwrap();
3209        let connection_path = root.join("c");
3210        let _data;
3211        let _config;
3212        let _port;
3213        {
3214            let _env_lock = ENV_LOCK.lock().unwrap();
3215            _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3216            _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3217            _port = EnvGuard::unset(SUBC_PORT_ENV);
3218        }
3219
3220        let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
3221
3222        assert!(bound.local_endpoint.is_none());
3223        assert!(bound.local_endpoint_unavailable.is_none());
3224        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3225        assert!(published.local_endpoint.is_none());
3226        assert!(
3227            local_listener::socket_path_for(None, &connection_path).is_none(),
3228            "no configured directory, no socket"
3229        );
3230        let mut leftovers = Vec::new();
3231        let mut pending = vec![data_home.clone()];
3232        while let Some(dir) = pending.pop() {
3233            for entry in fs::read_dir(&dir).unwrap() {
3234                let entry = entry.unwrap();
3235                let path = entry.path();
3236                if entry.file_type().unwrap().is_dir() {
3237                    pending.push(path.clone());
3238                }
3239                if entry.file_name().to_string_lossy().ends_with(".sock") {
3240                    leftovers.push(path);
3241                }
3242            }
3243        }
3244        assert!(
3245            leftovers.is_empty(),
3246            "an in-process daemon must not create a socket in the data home: {leftovers:?}"
3247        );
3248    }
3249
3250    /// A local endpoint that cannot be bound does not stop the daemon: it keeps
3251    /// serving TCP, publishes no local endpoint, and reports the reason where an
3252    /// operator reads daemon facts. Every module stays up for a feature nothing
3253    /// has to use.
3254    #[cfg(unix)]
3255    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3256    async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
3257        let dir = ScratchDir::new("tcp-only");
3258        let canonical = std::fs::canonicalize(&dir).unwrap();
3259        let connection_path = canonical.join("c");
3260        // The socket directory is deep enough that the socket's path is over the
3261        // Unix socket limit, so binding it must fail.
3262        let deep = canonical.join("x".repeat(80));
3263        fs::create_dir(&deep).unwrap();
3264        fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
3265        let config =
3266            BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
3267        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3268        assert!(
3269            bound.local_endpoint.is_none(),
3270            "a socket that cannot be bound is not published"
3271        );
3272        assert!(
3273            bound
3274                .local_endpoint_unavailable
3275                .as_deref()
3276                .is_some_and(|reason| reason.contains("too long")),
3277            "{:?}",
3278            bound.local_endpoint_unavailable
3279        );
3280        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3281        assert!(published.local_endpoint.is_none());
3282        assert!(
3283            !published.info.endpoints.is_empty(),
3284            "TCP is still published"
3285        );
3286
3287        // The daemon serves TCP as usual.
3288        let server = start_server(bound);
3289        let info = connection_file::read(&connection_path).unwrap();
3290        let mut client = connect_from_info(&info).await.unwrap();
3291        authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
3292            .await
3293            .expect("a TCP client still completes the handshake");
3294        server.abort();
3295        let _ = server.await;
3296    }
3297
3298    /// The published file names the local endpoint the daemon actually serves,
3299    /// and a client that trusts the file completes the handshake over it.
3300    #[cfg(unix)]
3301    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3302    async fn published_connection_file_names_the_served_local_endpoint() {
3303        let (_dir, path) = temp_connection_file_path("local-endpoint");
3304        // The socket goes in the system temporary directory rather than inside
3305        // the harness's own directory, whose long lockdown-sandbox path leaves
3306        // no room inside the Unix socket path limit.
3307        let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
3308        let config =
3309            BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
3310        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3311        let endpoint = connection_file::read_with_local_endpoint(&path)
3312            .unwrap()
3313            .local_endpoint
3314            .expect("the daemon serves a local endpoint, so the file names it");
3315        assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
3316        let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
3317            panic!("a Unix daemon publishes a socket path");
3318        };
3319        assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
3320        assert_eq!(
3321            fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
3322            0o600
3323        );
3324
3325        let router = Arc::new(Router::with_default_self_handler());
3326        let auth = auth_for(&bound.connection_info);
3327        let server = tokio::spawn(serve_local_listener(
3328            bound.local_listener.expect("the local listener is bound"),
3329            router,
3330            auth,
3331        ));
3332        let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
3333        authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
3334            .await
3335            .expect("a socket client completes the same handshake as a TCP client");
3336
3337        server.abort();
3338        drop(bound.listeners);
3339    }
3340
3341    /// A configured directory that is not a directory refuses at bind rather
3342    /// than publishing an endpoint nothing serves.
3343    #[cfg(unix)]
3344    #[test]
3345    fn a_socket_directory_that_is_a_file_refuses() {
3346        let dir = ScratchDir::new("local-listener-file-as-dir");
3347        let file = dir.join("not-a-directory");
3348        fs::write(&file, b"x").unwrap();
3349        let error = local_listener::bind(
3350            &LocalListenerConfig::in_socket_dir(file),
3351            &dir.join("connection.json"),
3352        )
3353        .expect_err("a path that is not a directory cannot hold the socket");
3354        assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
3355    }
3356
3357    /// The socket sits directly in the configured directory under one fixed
3358    /// name, and a directory that would put the path over the platform limit is
3359    /// refused rather than shortened around.
3360    #[cfg(unix)]
3361    #[test]
3362    fn the_socket_path_stays_within_the_unix_limit() {
3363        let run_dir = PathBuf::from("/tmp/short-run");
3364        assert!(
3365            local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
3366            "no configured directory means no socket"
3367        );
3368        let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
3369            .expect("a configured directory gets a socket");
3370        assert_eq!(socket, run_dir.join("subc.sock"));
3371        assert_eq!(
3372            socket,
3373            local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
3374            "the name does not come from the connection file"
3375        );
3376
3377        let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
3378        let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
3379            .expect("a configured directory gets a socket");
3380        assert!(
3381            long.as_os_str().len() < 104,
3382            "{} is {} bytes",
3383            long.display(),
3384            long.as_os_str().len()
3385        );
3386    }
3387}