Skip to main content

subc_daemon/
bootstrap.rs

1use std::{
2    collections::BTreeMap,
3    env,
4    error::Error,
5    ffi::OsString,
6    fmt, fs, io,
7    net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8    path::{Path, PathBuf},
9    process,
10    time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17    authenticate_client, connection_file, generate_daemon_id, generate_key,
18    write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
19    LocalEndpoint, SCHEMA_VERSION,
20};
21use tokio::{
22    net::{TcpListener, TcpStream},
23    task::{JoinError, JoinHandle},
24    time::{sleep, timeout},
25};
26use tracing::{error, info, warn};
27
28use crate::{
29    daemon_config::{self, ConfiguredModule, DaemonConfigError},
30    local_listener::{self, BoundLocalListener, LocalListenerConfig},
31    server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
32    supervise::HealthConfig,
33    ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
34    ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
35    SupervisorProcessLiveness,
36};
37use std::sync::Arc;
38
39#[cfg(windows)]
40mod windows_connection_copy;
41
42pub const DEFAULT_SUBC_PORT: u16 = 8757;
43pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
44use subc_transport::CONNECTION_FILE_NAME;
45const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
46const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
47const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
48const START_LOCK_RETRIES: usize = 40;
49const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum ConnectionFileSource {
53    XdgRuntimeDir,
54    TempDirFallback,
55    Explicit,
56}
57
58impl ConnectionFileSource {
59    fn reason(self) -> &'static str {
60        match self {
61            Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
62            Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
63            Self::Explicit => "configured path",
64        }
65    }
66}
67
68impl fmt::Display for ConnectionFileSource {
69    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
70        formatter.write_str(match self {
71            Self::XdgRuntimeDir => "xdg_runtime_dir",
72            Self::TempDirFallback => "temp_dir_fallback",
73            Self::Explicit => "explicit",
74        })
75    }
76}
77
78/// Runtime bootstrap configuration. Production uses the default fixed port and
79/// optional daemon-config override; tests pass port 0 to let the OS assign a free
80/// loopback port and discover it from the connection file.
81#[derive(Debug, Clone, Default)]
82struct AdmissionFactsConfig {
83    carrier_module_id: Option<String>,
84    targets: Option<Vec<String>>,
85}
86
87/// Controls where module cgroups are prepared.
88///
89/// In-process daemons default to [`Self::Disabled`] so they never derive a
90/// production location from the host process. The shipped daemon explicitly uses
91/// [`Self::Current`]. The `ck-subc` binary accepts
92/// `SUBC_CGROUP_PLACEMENT=disabled` for isolated test processes; an unset
93/// variable retains `Current`, and every other value is rejected at startup.
94/// Tests that exercise placement can own a [`Self::Root`].
95#[derive(Debug, Clone, Default, PartialEq, Eq)]
96pub enum CgroupPlacementConfig {
97    #[default]
98    Disabled,
99    Current,
100    Root(PathBuf),
101}
102
103#[derive(Debug, Clone)]
104pub struct BootstrapConfig {
105    #[cfg(feature = "test-support")]
106    shutdown_intent_window: Duration,
107    pub operator_queue_wait: Duration,
108    pub operator_timeout: Duration,
109    pub operator_stuck_grace: Duration,
110    #[cfg(feature = "test-support")]
111    pub operator_script: Option<PathBuf>,
112    #[cfg(feature = "test-support")]
113    pub operator_events: Option<PathBuf>,
114    pub connection_file_path: PathBuf,
115    pub port: u16,
116    pub daemon_ver: String,
117    configured_modules: Vec<ConfiguredModule>,
118    storage_config: Option<daemon_config::StorageConfig>,
119    admission_facts: AdmissionFactsConfig,
120    /// Module ids whose scopes may carry the attributes that grant authority.
121    scope_authority_owners: Vec<String>,
122    federated_scope_owner: String,
123    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
124    activity_config: daemon_config::ActivityConfig,
125    daemon_config_path: Option<PathBuf>,
126    /// Only a service-definition argument may name this authority store. An
127    /// embedded daemon with no explicit root never opens an installation store.
128    config_store_root: Option<PathBuf>,
129    defer_file_load: bool,
130    // Test-only stand-in for an additional field supplied by the file loader.
131    // Authority selection must carry it without knowing its name.
132    #[cfg(test)]
133    file_derived_test_probe: Option<Vec<String>>,
134    #[cfg(all(test, feature = "test-support"))]
135    test_store_factory:
136        Option<fn(crate::config_store::StoreRoot) -> crate::config_store::ConfigStore>,
137    #[cfg(all(test, feature = "test-support"))]
138    test_control: Option<fn(ControlHandler) -> ControlHandler>,
139    #[cfg(all(test, feature = "test-support"))]
140    test_listener_bind: Option<fn(&mut BootstrapConfig)>,
141    configured_port: Option<u16>,
142    /// Daemon-wide route.bind relay budget in milliseconds (the fallback for
143    /// any module without a per-module override). `None` = built-in default
144    /// (12s — see `control::DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`).
145    route_bind_relay_default_ms: Option<u64>,
146    reserved_capabilities: BTreeMap<String, String>,
147    watchdog_config: DaemonSelfWatchdogConfig,
148    connection_file_source: ConnectionFileSource,
149    #[cfg(all(test, windows))]
150    profile_override: Option<PathBuf>,
151    /// Where module cgroups are prepared. Disabled unless a caller explicitly
152    /// opts in, because `Current` derives a host location from `/proc/self/cgroup`.
153    cgroup_placement: CgroupPlacementConfig,
154    /// Directory the supervisor writes per-module stdout/stderr capture files
155    /// into. `None` disables capture; the shipped binary supplies its real run
156    /// directory explicitly.
157    capture_logs_dir: Option<PathBuf>,
158    /// File the supervisor appends every module's terminal exits to, so exit
159    /// history survives a daemon restart. `None` keeps terminal history in
160    /// memory only (each module's ring). Absent by default for the same reason
161    /// as `capture_logs_dir`: an in-process daemon booted by a test must not
162    /// append its exits to the operator's real `terminals.jsonl`, where they
163    /// would show up in `ck module terminals`. The shipped binary supplies
164    /// `<run dir>/terminals.jsonl` explicitly.
165    terminal_journal_path: Option<PathBuf>,
166    /// Where the machine id is read from, or minted into when absent. `None`
167    /// serves no machine id. Absent by default for the same reason as
168    /// `capture_logs_dir`: an in-process daemon booted by a test must never
169    /// derive the operator's real data home and mint into it. The shipped binary
170    /// supplies `<data home>/cortexkit/machine-id` explicitly.
171    machine_id_path: Option<PathBuf>,
172
173    /// Where this daemon's same-user local listener lives (a Unix-domain socket
174    /// on macOS and Linux; nothing on Windows, where the pipe name is minted per
175    /// start). The default serves no local endpoint, so an in-process daemon
176    /// never touches the operator's run directory. The shipped binary names its
177    /// run directory; [`BootstrapConfig::with_local_listener_socket_dir`] names
178    /// any other, and an in-process daemon that wants the endpoint names one.
179    local_listener: LocalListenerConfig,
180    /// The live-children record: every supervised process this daemon has
181    /// running, kept so the next daemon can end the ones a crash left behind.
182    /// At startup, before any module is spawned, the previous daemon's record
183    /// here is swept. `None` keeps no record and sweeps nothing, for the same
184    /// reason as `capture_logs_dir`: an in-process daemon booted by a test
185    /// must never signal processes listed in the operator's real run
186    /// directory. The shipped binary supplies `<run dir>/live-children.json`.
187    live_children_path: Option<PathBuf>,
188    privacy_trampoline: Option<PathBuf>,
189}
190
191impl BootstrapConfig {
192    pub fn with_config_store_root(mut self, root: PathBuf) -> Result<Self, BootstrapError> {
193        crate::config_store::StoreRoot::from_service_definition(root.clone())
194            .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?;
195        let parent = root.parent().ok_or_else(|| {
196            BootstrapError::ConfigStore("config store root needs a parent directory".into())
197        })?;
198        std::fs::read_dir(parent).map_err(|e| {
199            BootstrapError::ConfigStore(format!("config store parent is unreadable: {e}"))
200        })?;
201        self.config_store_root = Some(root);
202        Ok(self)
203    }
204    /// Shorten restart intent validity in daemon integration fixtures.
205    #[cfg(feature = "test-support")]
206    pub fn with_shutdown_intent_window(mut self, window: Duration) -> Self {
207        self.shutdown_intent_window = window;
208        self
209    }
210
211    pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
212        Self {
213            #[cfg(feature = "test-support")]
214            shutdown_intent_window: Duration::from_secs(120),
215            connection_file_path: connection_file_path.into(),
216            operator_queue_wait: Duration::from_secs(150),
217            operator_timeout: Duration::from_secs(120),
218            operator_stuck_grace: Duration::from_secs(10),
219            #[cfg(feature = "test-support")]
220            operator_script: None,
221            #[cfg(feature = "test-support")]
222            operator_events: None,
223            port,
224            daemon_ver: DAEMON_VERSION.to_owned(),
225            configured_modules: Vec::new(),
226            storage_config: None,
227            admission_facts: AdmissionFactsConfig::default(),
228            scope_authority_owners: daemon_config::default_scope_authority_owners(),
229            federated_scope_owner: daemon_config::default_federated_scope_owner(),
230            privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
231            activity_config: daemon_config::ActivityConfig::default(),
232            daemon_config_path: None,
233            config_store_root: None,
234            defer_file_load: false,
235            #[cfg(test)]
236            file_derived_test_probe: None,
237            #[cfg(all(test, feature = "test-support"))]
238            test_store_factory: None,
239            #[cfg(all(test, feature = "test-support"))]
240            test_control: None,
241            #[cfg(all(test, feature = "test-support"))]
242            test_listener_bind: None,
243            configured_port: None,
244            route_bind_relay_default_ms: None,
245            reserved_capabilities: BTreeMap::new(),
246            watchdog_config: DaemonSelfWatchdogConfig::default(),
247            connection_file_source: ConnectionFileSource::Explicit,
248            #[cfg(all(test, windows))]
249            profile_override: None,
250            cgroup_placement: CgroupPlacementConfig::default(),
251            capture_logs_dir: None,
252            terminal_journal_path: None,
253            machine_id_path: None,
254            local_listener: LocalListenerConfig { socket_dir: None },
255            live_children_path: None,
256            privacy_trampoline: None,
257        }
258    }
259
260    /// Keep the local listener's socket in `dir`. Embedding daemons and tests
261    /// pass a path inside their own fixture tree.
262    pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
263        self.local_listener = LocalListenerConfig::in_socket_dir(dir);
264        self
265    }
266
267    /// Set where the local listener lives, as [`LocalListenerConfig`] describes.
268    pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
269        self.local_listener = config;
270        self
271    }
272
273    /// Keep the live-children record at `path`, and at startup end the
274    /// processes a previous daemon recorded there that are still running.
275    /// Embedding daemons and tests pass a path inside their own fixture tree.
276    pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
277        self.live_children_path = Some(path.into());
278        self
279    }
280
281    /// macOS module launches require an explicit executable implementing the
282    /// subc-os trampoline entry point. Other systems ignore this setting.
283    pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
284        self.privacy_trampoline = Some(path.into());
285        self
286    }
287
288    /// Serve the machine id stored at `path`, minting it there at startup when
289    /// the file is absent. Embedding daemons and tests pass a path inside their
290    /// own fixture tree.
291    pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
292        self.machine_id_path = Some(path.into());
293        self
294    }
295
296    /// Selects module cgroup placement. The default is disabled.
297    pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
298        self.cgroup_placement = placement;
299        self
300    }
301
302    /// Redirects per-module stdout/stderr capture files out of the real run
303    /// directory. Tests that start an in-process daemon must call this with a
304    /// path inside their fixture tree.
305    pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
306        self.capture_logs_dir = Some(dir.into());
307        self
308    }
309
310    /// Redirects the daemon-private journal, allowing embedded daemons and tests
311    /// to keep their observations out of the operator's live run directory.
312    pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
313        self.terminal_journal_path = Some(path.into());
314        self
315    }
316
317    pub fn from_env() -> Result<Self, BootstrapError> {
318        Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
319    }
320
321    /// The SHIPPED BINARY's config, which is the only caller that should capture
322    /// child output into the operator's real run directory.
323    ///
324    /// Kept separate from `from_env` deliberately: see `capture_logs_dir` and
325    /// `terminal_journal_path` on this struct for why an absent value must mean
326    /// NO CAPTURE and NO JOURNAL rather than the operator's real run directory.
327    /// A run directory that cannot be resolved (a relative data home) refuses
328    /// startup instead of landing under the working directory.
329    pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
330        let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
331        let machine_id_path =
332            crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
333        // The service supplies its store root after constructing this value.
334        // Do not parse live files until that store has selected recovery mode.
335        let (connection_file, source) =
336            connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
337        let mut config = Self::new(connection_file, DEFAULT_SUBC_PORT)
338            .with_connection_file_source(source)
339            .with_daemon_config_source(daemon_config::default_config_path(), None)
340            .with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
341            .with_capture_logs_dir(run_dir.join("logs"))
342            .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
343            .with_live_children_record(crate::live_children::record_path(&run_dir))
344            .with_machine_id_path(machine_id_path);
345        config.defer_file_load = true;
346        #[cfg(feature = "test-support")]
347        let config = {
348            let mut config = config;
349            config.operator_script =
350                std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
351            config.operator_events =
352                std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
353            for (name, value) in [
354                (
355                    "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
356                    &mut config.operator_queue_wait,
357                ),
358                (
359                    "SUBC_TEST_OPERATOR_TIMEOUT_MS",
360                    &mut config.operator_timeout,
361                ),
362                (
363                    "SUBC_TEST_OPERATOR_STUCK_MS",
364                    &mut config.operator_stuck_grace,
365                ),
366            ] {
367                if let Ok(ms) = std::env::var(name) {
368                    if let Ok(ms) = ms.parse::<u64>() {
369                        *value = Duration::from_millis(ms);
370                    }
371                }
372            }
373            config
374        };
375        Ok(config)
376    }
377
378    /// Load live-file values for an unstamped embedder. If a store root is
379    /// attached later, these values are provisional and replaced at run time.
380    /// An invalid live file can still fail this eager constructor; the binary
381    /// defers loading until store authority has been selected instead.
382    pub fn from_env_with_daemon_config_path(
383        daemon_config_path: impl AsRef<Path>,
384    ) -> Result<Self, BootstrapError> {
385        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
386        let (daemon_config, activity_config) =
387            daemon_config::load_with_activity(&daemon_config_path)
388                .map_err(BootstrapError::DaemonConfig)?
389                .map(|(config, activity)| (Some(config), activity))
390                .unwrap_or_default();
391        let config_port = daemon_config.as_ref().and_then(|config| config.port);
392        let storage_config = daemon_config
393            .as_ref()
394            .and_then(|config| config.storage.clone());
395        let admission_facts_carrier_module_id = daemon_config
396            .as_ref()
397            .and_then(|config| config.admission_facts_carrier_module_id.clone());
398        let admission_facts_targets = daemon_config
399            .as_ref()
400            .and_then(|config| config.admission_facts_targets.clone());
401        let scope_authority_owners = daemon_config
402            .as_ref()
403            .map(|config| config.scope_authority_owners.clone())
404            .unwrap_or_else(daemon_config::default_scope_authority_owners);
405        let federated_scope_owner = daemon_config
406            .as_ref()
407            .map(|config| config.federated_scope_owner.clone())
408            .unwrap_or_else(daemon_config::default_federated_scope_owner);
409        let privileged_operator_prompt = daemon_config
410            .as_ref()
411            .map(|config| config.privileged_operator_prompt)
412            .unwrap_or_default();
413        let route_bind_relay_default_ms = daemon_config
414            .as_ref()
415            .and_then(|config| config.route_bind_relay_timeout_ms);
416        let reserved_capabilities = daemon_config
417            .as_ref()
418            .map(|config| config.reserved_capabilities.clone())
419            .unwrap_or_default();
420        let configured_modules = daemon_config
421            .map(|config| config.modules)
422            .unwrap_or_default();
423
424        let port = match env::var(SUBC_PORT_ENV) {
425            Ok(raw) if !raw.trim().is_empty() => {
426                let port = raw
427                    .parse::<u16>()
428                    .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
429                if let Some(config_port) = config_port {
430                    info!(
431                        env = SUBC_PORT_ENV,
432                        env_port = port,
433                        config_port,
434                        "SUBC_PORT overrides daemon config port"
435                    );
436                }
437                port
438            }
439            Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
440        };
441
442        let (connection_file_path, connection_file_source) =
443            connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
444        let config = Self::new(connection_file_path, port)
445            .with_configured_modules(configured_modules)
446            .with_storage_config(storage_config)
447            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
448            .with_scope_authority_owners(scope_authority_owners)
449            .with_federated_scope_owner(federated_scope_owner)
450            .with_privileged_operator_prompt(privileged_operator_prompt)
451            .with_activity_config(activity_config)
452            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
453            .with_reserved_capabilities(reserved_capabilities)
454            .with_daemon_config_source(daemon_config_path, config_port)
455            .with_connection_file_source(connection_file_source);
456        #[cfg(test)]
457        let config = {
458            let mut config = config;
459            config.file_derived_test_probe = Some(config.scope_authority_owners.clone());
460            config
461        };
462        Ok(config)
463    }
464
465    /// Load live-file values unless a store root is already attached. Attaching
466    /// a root later makes eager values provisional: run time replaces them with
467    /// approved authority. An invalid live file can fail the eager constructor;
468    /// the binary's deferred path does not parse it before selecting authority.
469    pub fn with_daemon_config_path(
470        mut self,
471        daemon_config_path: impl AsRef<Path>,
472    ) -> Result<Self, BootstrapError> {
473        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
474        if self.config_store_root.is_some() {
475            self.defer_file_load = true;
476            return Ok(self.with_daemon_config_source(daemon_config_path, None));
477        }
478        let (daemon_config, activity_config) =
479            daemon_config::load_with_activity(&daemon_config_path)
480                .map_err(BootstrapError::DaemonConfig)?
481                .map(|(config, activity)| (Some(config), activity))
482                .unwrap_or_default();
483        let configured_port = daemon_config.as_ref().and_then(|config| config.port);
484        Ok(self
485            .with_parsed_daemon_config(daemon_config, activity_config)
486            .with_daemon_config_source(daemon_config_path, configured_port))
487    }
488
489    fn with_parsed_daemon_config(
490        self,
491        daemon_config: Option<daemon_config::DaemonConfig>,
492        activity_config: daemon_config::ActivityConfig,
493    ) -> Self {
494        let storage_config = daemon_config
495            .as_ref()
496            .and_then(|config| config.storage.clone());
497        let admission_facts_carrier_module_id = daemon_config
498            .as_ref()
499            .and_then(|config| config.admission_facts_carrier_module_id.clone());
500        let admission_facts_targets = daemon_config
501            .as_ref()
502            .and_then(|config| config.admission_facts_targets.clone());
503        let scope_authority_owners = daemon_config
504            .as_ref()
505            .map(|config| config.scope_authority_owners.clone())
506            .unwrap_or_else(daemon_config::default_scope_authority_owners);
507        let federated_scope_owner = daemon_config
508            .as_ref()
509            .map(|config| config.federated_scope_owner.clone())
510            .unwrap_or_else(daemon_config::default_federated_scope_owner);
511        let privileged_operator_prompt = daemon_config
512            .as_ref()
513            .map(|config| config.privileged_operator_prompt)
514            .unwrap_or_default();
515        let route_bind_relay_default_ms = daemon_config
516            .as_ref()
517            .and_then(|config| config.route_bind_relay_timeout_ms);
518        let reserved_capabilities = daemon_config
519            .as_ref()
520            .map(|config| config.reserved_capabilities.clone())
521            .unwrap_or_default();
522        let configured_modules = daemon_config
523            .map(|config| config.modules)
524            .unwrap_or_default();
525        self.with_configured_modules(configured_modules)
526            .with_storage_config(storage_config)
527            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
528            .with_scope_authority_owners(scope_authority_owners)
529            .with_federated_scope_owner(federated_scope_owner)
530            .with_privileged_operator_prompt(privileged_operator_prompt)
531            .with_activity_config(activity_config)
532            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
533            .with_reserved_capabilities(reserved_capabilities)
534    }
535
536    pub fn with_configured_modules(
537        mut self,
538        modules: impl IntoIterator<Item = ConfiguredModule>,
539    ) -> Self {
540        self.configured_modules = modules.into_iter().collect();
541        self.configured_modules
542            .sort_by(|left, right| left.module_id.cmp(&right.module_id));
543        self
544    }
545
546    pub fn with_storage_config(
547        mut self,
548        storage_config: Option<daemon_config::StorageConfig>,
549    ) -> Self {
550        self.storage_config = storage_config;
551        self
552    }
553
554    pub fn with_admission_facts_config(
555        mut self,
556        carrier_module_id: Option<String>,
557        targets: Option<Vec<String>>,
558    ) -> Self {
559        self.admission_facts = AdmissionFactsConfig {
560            carrier_module_id,
561            targets,
562        };
563        self
564    }
565
566    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
567    /// Absent from a config, this is `daemon_config::default_scope_authority_owners`.
568    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
569        self.scope_authority_owners = owners;
570        self
571    }
572
573    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
574        self.federated_scope_owner = owner;
575        self
576    }
577
578    /// Choose the privileged-operation policy for the daemon's entire lifetime.
579    pub fn with_privileged_operator_prompt(
580        mut self,
581        mode: daemon_config::PrivilegedOperatorPrompt,
582    ) -> Self {
583        self.privileged_operator_prompt = mode;
584        self
585    }
586
587    pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
588        self.activity_config = config;
589        self
590    }
591
592    /// Set the daemon-wide route.bind relay default (the fallback for any
593    /// module without a per-module override). `None` preserves the built-in
594    /// default (12s). `serve_bound_daemon` reads this at startup and threads
595    /// it into the control handler's daemon-wide field.
596    pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
597        self.route_bind_relay_default_ms = ms;
598        self
599    }
600
601    pub fn with_reserved_capabilities(
602        mut self,
603        reserved_capabilities: BTreeMap<String, String>,
604    ) -> Self {
605        self.reserved_capabilities = reserved_capabilities;
606        self
607    }
608
609    fn with_daemon_config_source(
610        mut self,
611        daemon_config_path: PathBuf,
612        configured_port: Option<u16>,
613    ) -> Self {
614        self.daemon_config_path = Some(daemon_config_path);
615        self.configured_port = configured_port;
616        self
617    }
618
619    fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
620        self.connection_file_source = source;
621        self
622    }
623
624    pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
625        self.watchdog_config = watchdog_config;
626        self
627    }
628}
629
630/// Result of singleton discovery.
631// Built once per daemon start and immediately matched, so the size gap between
632// the variants costs nothing worth a box on the public shape.
633#[allow(clippy::large_enum_variant)]
634#[derive(Debug)]
635pub enum Outcome {
636    /// A live daemon authenticated from the connection file; this invocation should exit 0.
637    AlreadyRunning,
638    /// This process won the singleton race, owns bound loopback listener(s), and
639    /// has published a fresh connection file.
640    Bound(BoundDaemon),
641}
642
643#[derive(Debug)]
644pub struct BoundDaemon {
645    pub listeners: Vec<TcpListener>,
646    /// The same-user local listener, bound and not yet served. `None` when
647    /// binding it failed, in which case `local_endpoint` is `None` and
648    /// `local_endpoint_unavailable` says why.
649    pub local_listener: Option<BoundLocalListener>,
650    /// Why the daemon serves no local endpoint, when binding it failed. Reported
651    /// by `server.describe` so an operator can see it.
652    pub local_endpoint_unavailable: Option<String>,
653    /// What the connection file publishes under `local_endpoint`, so a client can
654    /// find the local listener without knowing how it was bound.
655    pub local_endpoint: Option<LocalEndpoint>,
656    pub connection_info: ConnectionInfo,
657    pub connection_file_path: PathBuf,
658    pub connection_file_source: ConnectionFileSource,
659    #[cfg(windows)]
660    connection_file_copy: Option<PathBuf>,
661    /// The machine id this daemon serves, established before any connection is
662    /// accepted. `None` when the config named no machine id path.
663    pub machine_id: Option<crate::machine_id::MachineId>,
664    /// Ownership of the run directory, held until the daemon exits so no other
665    /// daemon sweeps or writes this one's run state. `None` when the config
666    /// keeps no live-children record, and so has no run directory to own.
667    run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
668    /// The singleton start lock, kept when binding deferred publishing the
669    /// connection file. It stays held until the daemon has verified it is serving
670    /// and written the connection file, so a second daemon starting meanwhile
671    /// cannot claim the singleton too. `None` when the connection
672    /// file was already written at bind time.
673    publication_lock: Option<StartLock>,
674}
675
676/// Resolve subc's per-user TCP connection-file path.
677///
678/// `$XDG_RUNTIME_DIR/subc-connection.json` is preferred because the runtime
679/// directory is already per-user on Unix desktops. Without it, subc falls back
680/// to the system temp dir with a per-user token in the filename so different OS
681/// users do not collide on shared temp directories.
682pub fn connection_file_path() -> PathBuf {
683    connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
684}
685
686fn connection_file_path_with_source(
687    runtime_dir: Option<OsString>,
688) -> (PathBuf, ConnectionFileSource) {
689    if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
690        return (
691            PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
692            ConnectionFileSource::XdgRuntimeDir,
693        );
694    }
695
696    (
697        env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
698        ConnectionFileSource::TempDirFallback,
699    )
700}
701
702/// Resolve, claim, and serve the per-user daemon singleton.
703///
704/// A second invocation is successful: if a live daemon authenticates from the
705/// existing connection file, this returns `Ok(())` after logging and the caller
706/// exits with status 0.
707pub async fn run() -> Result<(), BootstrapError> {
708    // `run` is a binary entry point, so it opts into the production cgroup and
709    // child-log locations. In-process callers keep both features disabled by default.
710    run_with_config(
711        BootstrapConfig::from_env_for_daemon_binary()?
712            .with_cgroup_placement(CgroupPlacementConfig::Current),
713    )
714    .await
715}
716
717#[cfg(feature = "test-support")]
718mod config_test_seam {
719    #[cfg(not(test))]
720    use super::BootstrapError;
721
722    pub(super) fn enabled() -> bool {
723        std::env::var_os("SUBC_TEST_CONFIG_GATE").as_deref() == Some(std::ffi::OsStr::new("1"))
724            && std::env::current_exe().ok().is_some_and(|path| {
725                path.file_name()
726                    .is_some_and(|name| name.to_string_lossy().starts_with("ckdev-"))
727            })
728    }
729
730    #[cfg(not(test))]
731    pub(super) fn open(
732        root: crate::config_store::StoreRoot,
733    ) -> Result<crate::config_store::ConfigStore, BootstrapError> {
734        if !enabled() {
735            return Ok(crate::config_store::ConfigStore::open(root));
736        }
737        // The service's isolated store root contains every seam artifact, so a
738        // test cannot accidentally open the operator's keychain or anchor.
739        crate::config_store::testing::TestSupport::new(
740            root.path().to_path_buf(),
741            Some(root.path().join("test-key")),
742        )
743        .and_then(|hooks| {
744            hooks
745                .with_anchor_failure_file(root.path().join("fail-anchor"))
746                .open()
747        })
748        .map_err(|error| BootstrapError::ConfigStore(error.to_string()))
749    }
750}
751
752/// Serve a daemon from an explicit config. This is the entry point the twelve
753/// sibling repos use to boot an in-process daemon in their integration tests.
754///
755/// # THIS INSTALLS NO TRACING SUBSCRIBER, SO THE DAEMON IS SILENT BY DEFAULT
756///
757/// The daemon's own diagnostics go through `tracing`, and `tracing` DISCARDS
758/// every event when no subscriber is installed. The shipped binary installs one
759/// in `main` (`init_tracing`); this function deliberately does not, because a
760/// library that installs a global subscriber fights with whatever the host
761/// process already set up.
762///
763/// The consequence for a test harness is not "less verbose": it is that the
764/// daemon has NOTHING TO SAY about any failure, and a missing instrument reads
765/// exactly like a clean one. PLEX found this on 2026-09-18 while trying to
766/// capture daemon logs beside an intermittent bind failure, and discovered the
767/// daemon had been silent in every conformance run that repo had ever done --
768/// so the one client-side error string was all the evidence that could exist,
769/// and they had spent a real investigation on a failure whose second source was
770/// never being recorded.
771///
772/// Install one in the harness before calling this, and assert it did something
773/// (they measured 236 daemon lines with the subscriber installed, 0 with the
774/// call commented out) -- otherwise the fix is itself unverified.
775pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
776    let Some(start_lock) = singleton_start_lock(&config.connection_file_path).await? else {
777        info!("subc daemon already running");
778        return Ok(());
779    };
780    let (config, opened_store, start_lock) = tokio::task::spawn_blocking(move || {
781        let mut opened_store = config
782            .config_store_root
783            .as_ref()
784            .map(|root| {
785                let (root, lock) = crate::config_store::StoreWriterLock::acquire(root)
786                    .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?;
787                #[cfg(all(test, feature = "test-support"))]
788                let store = if let Some(factory) = config.test_store_factory {
789                    crate::config_store::testing::TestSupport::new(root.path().to_path_buf(), None)
790                        .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?;
791                    factory(root)
792                } else {
793                    crate::config_store::ConfigStore::open(root)
794                };
795                #[cfg(all(feature = "test-support", not(test)))]
796                let store = config_test_seam::open(root)?;
797                #[cfg(not(feature = "test-support"))]
798                let store = crate::config_store::ConfigStore::open(root);
799                Ok::<_, BootstrapError>((store, lock))
800            })
801            .transpose()?;
802        let config =
803            load_bootstrap_authority(config, opened_store.as_mut().map(|(store, _)| store))?;
804        #[cfg(all(test, feature = "test-support"))]
805        let config = {
806            let mut config = config;
807            if let Some(bind) = config.test_listener_bind {
808                // Tests observe authority's port selection but change only the
809                // bind, so they never take a running daemon's production port.
810                bind(&mut config);
811            }
812            config
813        };
814        Ok::<_, BootstrapError>((config, opened_store, start_lock))
815    })
816    .await
817    .map_err(|e| BootstrapError::ConfigStore(e.to_string()))??;
818    #[cfg(feature = "test-support")]
819    let shutdown_intent_window = config.shutdown_intent_window;
820    let forwarding = Arc::new(ForwardingTable::default());
821    forwarding.operator_confirms().configure_bootstrap(&config);
822    let configured_modules = config.configured_modules.clone();
823    let storage_config = config.storage_config.clone();
824    let admission_facts = config.admission_facts.clone();
825    let scope_authority_owners = config.scope_authority_owners.clone();
826    let federated_scope_owner = config.federated_scope_owner.clone();
827    let privileged_operator_prompt = config.privileged_operator_prompt;
828    let activity_config = config.activity_config;
829    let daemon_config_path = config.daemon_config_path.clone();
830    let configured_port = config.configured_port;
831    let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
832    let reserved_capabilities = config.reserved_capabilities.clone();
833    let watchdog_config = config.watchdog_config.clone();
834    let cgroup_placement_config = config.cgroup_placement.clone();
835    let capture_logs_dir = config.capture_logs_dir.clone();
836    let terminal_journal_path = config.terminal_journal_path.clone();
837    let live_children_path = config.live_children_path.clone();
838    let privacy_trampoline = config.privacy_trampoline.clone();
839    #[cfg(all(test, feature = "test-support"))]
840    let test_control = config.test_control;
841    match ensure_singleton_with_start_lock(config, false, start_lock).await? {
842        Outcome::AlreadyRunning => {
843            info!("subc daemon already running");
844            Ok(())
845        }
846        Outcome::Bound(bound) => {
847            #[cfg(target_os = "linux")]
848            let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
849            #[cfg(not(target_os = "linux"))]
850            let _ = cgroup_placement_config;
851            serve_bound_daemon(
852                bound,
853                forwarding,
854                configured_modules,
855                storage_config,
856                admission_facts,
857                scope_authority_owners,
858                federated_scope_owner,
859                privileged_operator_prompt,
860                activity_config,
861                daemon_config_path,
862                opened_store,
863                configured_port,
864                route_bind_relay_default_ms,
865                reserved_capabilities,
866                watchdog_config,
867                capture_logs_dir,
868                terminal_journal_path,
869                live_children_path,
870                privacy_trampoline,
871                #[cfg(all(test, feature = "test-support"))]
872                test_control,
873                #[cfg(feature = "test-support")]
874                shutdown_intent_window,
875                #[cfg(target_os = "linux")]
876                cgroup_placement,
877            )
878            .await
879        }
880    }
881}
882
883// Keep authority selection before binding listeners or constructing control
884// authorization. Recovery must remain reachable even with invalid live files.
885fn load_bootstrap_authority(
886    config: BootstrapConfig,
887    mut store: Option<&mut crate::config_store::ConfigStore>,
888) -> Result<BootstrapConfig, BootstrapError> {
889    let store_state = store.as_mut().map(|store| store.status().state);
890    let recovery = matches!(
891        store_state,
892        Some(
893            crate::config_store::StoreState::StoreUnverified
894                | crate::config_store::StoreState::StoreMissing
895        )
896    );
897    let authority = if recovery {
898        BootstrapConfig::new(&config.connection_file_path, DEFAULT_SUBC_PORT)
899    } else if store_state == Some(crate::config_store::StoreState::Stamped) {
900        let store = store.expect("stamped store was just checked");
901        match stamped_authority(&config.connection_file_path, store) {
902            Ok(authority) => authority,
903            Err(reason) => {
904                // A stamped daemon never falls back to files, and exiting would
905                // leave the operator no way to reach recovery.
906                tracing::error!(%reason, "approved configuration unusable; starting in recovery");
907                store.enter_recovery(&reason);
908                BootstrapConfig::new(&config.connection_file_path, DEFAULT_SUBC_PORT)
909            }
910        }
911    } else if config.defer_file_load {
912        BootstrapConfig::from_env_with_daemon_config_path(
913            config
914                .daemon_config_path
915                .as_ref()
916                .expect("binary config path"),
917        )?
918    } else {
919        return Ok(config);
920    };
921    // Only installation paths and explicit binary arguments override authority.
922    // New file-derived fields are inherited whole from the selected authority.
923    Ok(BootstrapConfig {
924        connection_file_path: config.connection_file_path,
925        connection_file_source: config.connection_file_source,
926        daemon_ver: config.daemon_ver,
927        daemon_config_path: config.daemon_config_path,
928        config_store_root: config.config_store_root,
929        defer_file_load: config.defer_file_load,
930        operator_queue_wait: config.operator_queue_wait,
931        operator_timeout: config.operator_timeout,
932        operator_stuck_grace: config.operator_stuck_grace,
933        watchdog_config: config.watchdog_config,
934        cgroup_placement: config.cgroup_placement,
935        capture_logs_dir: config.capture_logs_dir,
936        terminal_journal_path: config.terminal_journal_path,
937        machine_id_path: config.machine_id_path,
938        local_listener: config.local_listener,
939        live_children_path: config.live_children_path,
940        privacy_trampoline: config.privacy_trampoline,
941        #[cfg(feature = "test-support")]
942        shutdown_intent_window: config.shutdown_intent_window,
943        #[cfg(feature = "test-support")]
944        operator_script: config.operator_script,
945        #[cfg(feature = "test-support")]
946        operator_events: config.operator_events,
947        #[cfg(all(test, windows))]
948        profile_override: config.profile_override,
949        #[cfg(all(test, feature = "test-support"))]
950        test_store_factory: config.test_store_factory,
951        #[cfg(all(test, feature = "test-support"))]
952        test_control: config.test_control,
953        #[cfg(all(test, feature = "test-support"))]
954        test_listener_bind: config.test_listener_bind,
955        ..authority
956    })
957}
958
959/// Authority from the approved daemon generation the store selects for this
960/// start. Every failure is a reason to enter recovery, never to exit.
961fn stamped_authority(
962    connection_file_path: &Path,
963    store: &mut crate::config_store::ConfigStore,
964) -> Result<BootstrapConfig, String> {
965    let generation = store
966        .select_boot_generation()
967        .map_err(|error| error.to_string())?;
968    let body = store.body().map_err(|error| error.to_string())?;
969    let layer = body
970        .generations
971        .get(&generation)
972        .and_then(|entry| entry.delivery.as_ref())
973        .and_then(|delivery| delivery.daemon.as_ref())
974        .ok_or("approved daemon snapshot is absent")?;
975    let bytes = layer
976        .bytes
977        .as_deref()
978        .ok_or("approved daemon bytes are absent")?;
979    let path = layer
980        .path
981        .as_deref()
982        .ok_or("approved daemon path is absent")?;
983    let doc = std::str::from_utf8(bytes).map_err(|error| error.to_string())?;
984    let (daemon, activity) = daemon_config::parse_with_activity(doc, Path::new(path))
985        .map_err(|error| format!("approved daemon configuration is invalid: {error}"))?;
986    let port = daemon.port;
987    Ok(
988        BootstrapConfig::new(connection_file_path, port.unwrap_or(DEFAULT_SUBC_PORT))
989            .with_parsed_daemon_config(Some(daemon), activity)
990            .with_daemon_config_source(PathBuf::from(path), port),
991    )
992}
993
994#[cfg(target_os = "linux")]
995fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
996    let result = match config {
997        CgroupPlacementConfig::Disabled => return None,
998        CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
999        CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
1000    };
1001
1002    match result {
1003        Ok(Some(placement)) => Some(placement),
1004        Ok(None) => {
1005            warn!(
1006                placement = ?config,
1007                "module cgroup placement is disabled: configured cgroup root is not delegated"
1008            );
1009            None
1010        }
1011        Err(error) => {
1012            warn!(
1013                placement = ?config,
1014                error = %error,
1015                "module cgroup placement is disabled by an unexpected cgroup probe error"
1016            );
1017            None
1018        }
1019    }
1020}
1021
1022/// Target soft limit for open file descriptors, applied to the daemon before any
1023/// module is spawned so children inherit it. Multi-root modules (one process
1024/// aggregating every project root's sqlite stores, index caches, watchers, and
1025/// LSP pipes) trivially exceed the macOS default soft limit of 256; a launchd
1026/// user agent does not pass login-shell ulimits through, so the raise must
1027/// happen in-process.
1028#[cfg(unix)]
1029const NOFILE_TARGET: u64 = 65536;
1030
1031/// Raise RLIMIT_NOFILE to `NOFILE_TARGET` (clamped to the hard limit).
1032/// Best-effort: failure is logged and never fatal, since the daemon can run
1033/// under the inherited limit — modules with few roots just have less headroom.
1034#[cfg(unix)]
1035fn raise_nofile_limit() {
1036    #[cfg(target_os = "macos")]
1037    let ceiling = subc_os::child_spawn::output(
1038        std::process::Command::new("/usr/sbin/sysctl").args(["-n", "kern.maxfilesperproc"]),
1039    )
1040    .ok()
1041    .filter(|output| output.status.success())
1042    .and_then(|output| String::from_utf8(output.stdout).ok())
1043    .and_then(|value| value.trim().parse::<u64>().ok())
1044    .filter(|value| *value > 0);
1045    #[cfg(not(target_os = "macos"))]
1046    let ceiling = None;
1047    raise_nofile_limit_with_ceiling(ceiling);
1048}
1049
1050#[cfg(unix)]
1051fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
1052    match rlimit::Resource::NOFILE.get() {
1053        Ok((soft, hard)) => {
1054            // On XNU an infinite hard limit does not remove the separate
1055            // per-process kernel ceiling. Exceeding it makes setrlimit fail
1056            // without raising the inherited (often 256) soft limit at all.
1057            let target = NOFILE_TARGET
1058                .min(hard)
1059                .min(kernel_ceiling.unwrap_or(u64::MAX));
1060            if soft >= target {
1061                return;
1062            }
1063            match rlimit::Resource::NOFILE.set(target, hard) {
1064                Ok(()) => info!(
1065                    previous_soft = soft,
1066                    new_soft = target,
1067                    hard,
1068                    "raised open-file soft limit for daemon and module children"
1069                ),
1070                Err(err) => warn!(
1071                    soft,
1072                    hard,
1073                    error = %err,
1074                    "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
1075                ),
1076            }
1077        }
1078        Err(err) => warn!(error = %err, "could not read open-file limit"),
1079    }
1080}
1081
1082/// CRT stdio-stream target on Windows (the `_setmaxstdio` maximum). Win32
1083/// HANDLEs — what Rust `File`, tokio sockets, and SQLite's Win32 VFS actually
1084/// consume — have a per-process quota in the millions and need no raise; the
1085/// C-runtime stream table (default 512) is the only low ceiling, and it is
1086/// per-process rather than inherited, so supervised modules linking the CRT
1087/// must raise their own. Raising it here covers the daemon itself.
1088#[cfg(windows)]
1089fn raise_nofile_limit() {
1090    const MAXSTDIO_TARGET: u32 = 8192;
1091    let current = rlimit::getmaxstdio();
1092    if current >= MAXSTDIO_TARGET {
1093        return;
1094    }
1095    match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
1096        Ok(new_max) => info!(
1097            previous = current,
1098            new_max, "raised CRT stdio-stream limit for daemon"
1099        ),
1100        Err(err) => warn!(
1101            current,
1102            error = %err,
1103            "could not raise CRT stdio-stream limit"
1104        ),
1105    }
1106}
1107
1108#[cfg(not(any(unix, windows)))]
1109fn raise_nofile_limit() {}
1110
1111pub async fn run_with_daemon_config_path(
1112    config: BootstrapConfig,
1113    daemon_config_path: impl AsRef<Path>,
1114) -> Result<(), BootstrapError> {
1115    run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
1116}
1117
1118#[allow(clippy::too_many_arguments)]
1119async fn serve_bound_daemon(
1120    bound: BoundDaemon,
1121    forwarding: Arc<ForwardingTable>,
1122    configured_modules: Vec<ConfiguredModule>,
1123    storage_config: Option<daemon_config::StorageConfig>,
1124    admission_facts: AdmissionFactsConfig,
1125    scope_authority_owners: Vec<String>,
1126    federated_scope_owner: String,
1127    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
1128    activity_config: daemon_config::ActivityConfig,
1129    daemon_config_path: Option<PathBuf>,
1130    opened_store: Option<(
1131        crate::config_store::ConfigStore,
1132        crate::config_store::StoreWriterLock,
1133    )>,
1134    configured_port: Option<u16>,
1135    route_bind_relay_default_ms: Option<u64>,
1136    reserved_capabilities: BTreeMap<String, String>,
1137    watchdog_config: DaemonSelfWatchdogConfig,
1138    capture_logs_dir: Option<PathBuf>,
1139    terminal_journal_path: Option<PathBuf>,
1140    live_children_path: Option<PathBuf>,
1141    privacy_trampoline: Option<PathBuf>,
1142    #[cfg(all(test, feature = "test-support"))] test_control: Option<
1143        fn(ControlHandler) -> ControlHandler,
1144    >,
1145    #[cfg(feature = "test-support")] shutdown_intent_window: Duration,
1146    #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
1147) -> Result<(), BootstrapError> {
1148    #[cfg(unix)]
1149    let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
1150        .map_err(BootstrapError::Signal)?;
1151    // Windows has no SIGTERM. Ctrl-C is a different event, not an equivalent
1152    // service-stop contract, so this shutdown handler is intentionally Unix-only.
1153    raise_nofile_limit();
1154    #[cfg(windows)]
1155    if let Err(error) = subc_jobobject::ensure_root_job() {
1156        // Keep the control plane available. The root-job API caches setup
1157        // failures, so later module spawns refuse instead of running unprotected.
1158        warn!(%error, "module launches disabled: Windows root job setup failed");
1159    }
1160
1161    info!(
1162        connection_file = %bound.connection_file_path.display(),
1163        connection_file_source = %bound.connection_file_source,
1164        connection_file_source_reason = bound.connection_file_source.reason(),
1165        endpoints = ?bound.connection_info.endpoints,
1166        configured_modules = configured_modules.len(),
1167        machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
1168        "subc daemon starting"
1169    );
1170
1171    // Before anything can spawn a module (the configured modules below, or a
1172    // client's start request once the listeners are served): a previous daemon
1173    // that died without its shutdown stop may have left children running, and
1174    // a fresh copy beside one would fight it for its port and stores. The
1175    // record is not a running daemon's because this daemon holds the lock on
1176    // the run directory the record lives in, and a running daemon holds that
1177    // lock for its whole life. (Claiming the singleton is not enough: it is
1178    // keyed on the connection file, which can live in a different runtime
1179    // directory from the run directory.) The lock stays held until this
1180    // function returns, which is when the daemon exits.
1181    let run_dir_lock = bound.run_dir_lock;
1182    let publication_lock = bound.publication_lock;
1183    if let Some(owner) = &run_dir_lock {
1184        #[cfg(target_os = "linux")]
1185        crate::live_children::sweep_orphans_with_placement(
1186            owner,
1187            &crate::live_children::AdoptedPids::none(),
1188            crate::live_children::SweepBounds::default(),
1189            cgroup_placement.as_ref(),
1190        )
1191        .await;
1192        #[cfg(not(target_os = "linux"))]
1193        crate::live_children::sweep_orphans(
1194            owner,
1195            &crate::live_children::AdoptedPids::none(),
1196            crate::live_children::SweepBounds::default(),
1197        )
1198        .await;
1199    }
1200
1201    let registry = Arc::new(Registry::default());
1202    let process_liveness = Arc::new(SupervisorProcessLiveness::new());
1203    let supervisor_handle = SupervisorHandle::new();
1204    let _host_health_task = AbortOnDrop::new(supervisor_handle.spawn_host_health_monitor());
1205    #[cfg(feature = "test-support")]
1206    supervisor_handle.set_shutdown_intent_window(shutdown_intent_window);
1207    let connected_clients = ConnectedClients::new();
1208    let daemon_incarnation = format!(
1209        "{:032x}",
1210        u128::from_be_bytes(bound.connection_info.daemon_id)
1211    );
1212    let config_runtime = if let Some((store, writer_lock)) = opened_store {
1213        let path = daemon_config_path.clone().ok_or_else(|| {
1214            BootstrapError::ConfigStore(
1215                "an explicit daemon config path is required with a config store".into(),
1216            )
1217        })?;
1218        let config_home = daemon_config::default_config_home();
1219        let incarnation = daemon_incarnation.clone();
1220        let projects = Arc::new(
1221            crate::config_gate::project_registry::EntorhinalProjectRegistry::new(
1222                registry.clone(),
1223                forwarding.clone(),
1224            ),
1225        );
1226        Some(Arc::new(
1227            tokio::task::spawn_blocking(move || {
1228                crate::config_gate::runtime::RuntimeOwner::new(
1229                    incarnation,
1230                    store,
1231                    crate::config_gate::capture::ConfigCapturer {
1232                        daemon_path: path,
1233                        config_home,
1234                        projects,
1235                    },
1236                )
1237                .retain_writer_lock(writer_lock)
1238                .map_err(io::Error::other)
1239            })
1240            .await
1241            .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?
1242            .map_err(|e| BootstrapError::ConfigStore(e.to_string()))?,
1243        ))
1244    } else {
1245        None
1246    };
1247    let _config_plan_sweep = config_runtime.as_ref().map(|runtime| {
1248        let runtime = Arc::downgrade(runtime);
1249        AbortOnDrop::new(tokio::spawn(async move {
1250            let mut interval =
1251                tokio::time::interval(crate::config_gate::runtime::PLAN_SWEEP_INTERVAL);
1252            loop {
1253                interval.tick().await;
1254                let Some(runtime) = runtime.upgrade() else {
1255                    break;
1256                };
1257                match tokio::task::spawn_blocking(move || runtime.sweep_expired()).await {
1258                    Ok(Ok(())) => {}
1259                    result => tracing::warn!(?result, "configuration plan expiry sweep failed"),
1260                }
1261            }
1262        }))
1263    });
1264    let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
1265        .with_process_liveness(process_liveness.clone())
1266        .with_forwarding(Arc::clone(&forwarding))
1267        .with_handle(supervisor_handle.clone())
1268        .with_connection_file_path(bound.connection_file_path.clone())
1269        .with_daemon_incarnation(daemon_incarnation.clone());
1270    let supervisor = match &config_runtime {
1271        Some(owner) => supervisor.with_config_runtime(owner.clone()),
1272        None => supervisor,
1273    };
1274    let supervisor = match privacy_trampoline {
1275        Some(path) => supervisor.with_privacy_trampoline(path),
1276        None => supervisor,
1277    };
1278    // ABSENT MEANS NO CAPTURE AND NO JOURNAL, NOT "THE REAL RUN DIRECTORY", and
1279    // the difference is a production-corruption hazard rather than a preference.
1280    // Both fields below follow the same rule: `None` for `capture_logs_dir`
1281    // means supervised output is not captured, and `None` for
1282    // `terminal_journal_path` means terminal history lives only in each
1283    // module's in-memory ring (`supervisor.terminals` still answers, with the
1284    // journal counters at zero).
1285    //
1286    // The terminal journal used to fall back to
1287    // `daemon_run_dir().join("terminals.jsonl")`, so an in-process test daemon
1288    // appended its fixture exits to the operator's journal, where they then
1289    // appeared in `ck module terminals`.
1290    //
1291    // The capture line used to be `unwrap_or_else(|| daemon_run_dir().join("logs"))`,
1292    // so ANY caller that did not set the field captured supervised children into
1293    // the operator's live `~/.local/share/cortexkit/run/logs/`. That is twelve
1294    // sibling repos whose integration tests boot an in-process daemon through
1295    // `run_with_config` -- none of which asked for it, and none of which can see
1296    // it from their side.
1297    //
1298    // Harmless while fixture module ids are fixture-shaped: this host carries 20
1299    // zero-byte files from subc's own tests (good-aft, missing-aft,
1300    // preview-consumer...). THE HAZARD IS A COLLISION. A fixture named "broca"
1301    // or "aft" appends to a PRODUCTION capture file that operators read
1302    // forensically and that placement gates count lines in -- with no residue to
1303    // notice, because the file legitimately exists and legitimately grows.
1304    //
1305    // Found by BROCA (2026-09-19) from the other side: their rigs spawn the
1306    // SHIPPED ck-subc and set XDG_CONFIG_HOME + XDG_RUNTIME_DIR but not
1307    // XDG_DATA_HOME, so every local rig run supervised a module named "broca"
1308    // and captured it into production's broca.stderr.log -- the same file I
1309    // count seal lines in before and after placing their binaries.
1310    //
1311    // The supervisor already treats `None` as no-capture and no-journal, so
1312    // this only removes invented defaults. The binary keeps capturing and
1313    // journaling via `BootstrapConfig::from_env_for_daemon_binary`.
1314
1315    // Keep both ledgers in the explicitly selected run directory. An embedded
1316    // daemon without a journal path must not write to the operator's live ledger.
1317    let activity_run_dir = terminal_journal_path
1318        .as_ref()
1319        .and_then(|path| path.parent())
1320        .map(Path::to_path_buf);
1321    let supervisor = match terminal_journal_path {
1322        Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
1323        None => supervisor,
1324    };
1325    let supervisor = match capture_logs_dir {
1326        Some(dir) => supervisor.with_capture_logs_dir(dir),
1327        None => supervisor,
1328    };
1329    let supervisor = match live_children_path {
1330        Some(path) => supervisor.with_live_children_record(path),
1331        None => supervisor,
1332    };
1333    #[cfg(target_os = "linux")]
1334    let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
1335    // Collect per-module route.bind relay overrides BEFORE handing the
1336    // `configured_modules` vector to the supervisor (which only needs each
1337    // module's `drain_timeout_ms`). Each entry was filled in by parse-time
1338    // resolution (per-module > daemon-wide > absent), so modules with no
1339    // override are absent from this map and the daemon-wide default applies.
1340    let route_bind_relay_timeouts = configured_modules
1341        .iter()
1342        .filter_map(|module| {
1343            module
1344                .route_bind_relay_timeout_ms
1345                .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
1346        })
1347        .collect::<std::collections::BTreeMap<_, _>>();
1348    let control_start_clock = crate::clock::StartClock::capture();
1349    let mut control = ControlHandler::with_forwarding(Arc::clone(&registry), forwarding)
1350        .with_daemon_version(bound.connection_info.daemon_ver.clone())
1351        .with_process_liveness(process_liveness)
1352        .with_supervisor(supervisor_handle)
1353        // The same flag `begin_daemon_shutdown` sets below, so requirement
1354        // changes while modules exit during the shutdown log at INFO.
1355        .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
1356        .with_connected_clients(connected_clients.clone())
1357        .with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
1358        .with_storage_config(storage_config)
1359        .with_machine_id(bound.machine_id.clone())
1360        .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
1361        .with_scope_authority_owners(scope_authority_owners)
1362        .with_federated_scope_owner(federated_scope_owner)
1363        .with_privileged_operator_prompt(privileged_operator_prompt)
1364        .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
1365        .with_daemon_provenance(
1366            bound.connection_info.pid,
1367            control_start_clock.started_at_ms(),
1368            std::env::current_exe().ok(),
1369            normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
1370            normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
1371        )
1372        .with_daemon_start_clock(control_start_clock)
1373        .with_capability_config(
1374            configured_modules
1375                .iter()
1376                .map(|module| (module.module_id.clone(), module.enabled)),
1377            reserved_capabilities,
1378        );
1379    if let Some(run_dir) = activity_run_dir {
1380        control =
1381            control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
1382    }
1383    let daemon_ready_runtime = config_runtime.clone();
1384    if let Some(runtime) = config_runtime {
1385        control = control.with_explicit_config_store(runtime);
1386    }
1387    #[cfg(feature = "test-support")]
1388    if config_test_seam::enabled() {
1389        control = control.with_config_os_prompt_policy_for_tests();
1390    }
1391    if let Some(ms) = route_bind_relay_default_ms {
1392        // A daemon-wide config value overrides the built-in default; a
1393        // `None` here leaves the ControlHandler's 12s default in place.
1394        control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
1395    }
1396    if let Some(config_path) = daemon_config_path {
1397        control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
1398        control = control.with_supervisor_rescan_entries(&configured_modules);
1399    }
1400    #[cfg(all(test, feature = "test-support"))]
1401    let control = if let Some(adapt) = test_control {
1402        // A test hook is not a substitute for the executable-name guard.
1403        let executable =
1404            std::env::current_exe().map_err(|e| BootstrapError::ConfigStore(e.to_string()))?;
1405        if !executable
1406            .file_name()
1407            .is_some_and(|name| name.to_string_lossy().starts_with("ckdev-"))
1408        {
1409            return Err(BootstrapError::ConfigStore(
1410                "test control requires a ckdev executable".into(),
1411            ));
1412        }
1413        adapt(control)
1414    } else {
1415        control
1416    };
1417    let control = Arc::new(control);
1418    let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
1419    let auth = ServerAuth::new(
1420        bound.connection_info.key.clone(),
1421        bound.connection_info.daemon_id,
1422        bound.connection_info.daemon_ver.clone(),
1423    )
1424    .with_connected_clients(connected_clients);
1425
1426    // Discovery must expose every accepted configured module, including disabled
1427    // ones. Registration creates status handles but neither starts monitors
1428    // nor launches processes: children need the published file for `--subc`.
1429    let mut registered_modules = Vec::with_capacity(configured_modules.len());
1430    for configured in configured_modules {
1431        let enabled = configured.enabled;
1432        let health = configured.health.clone();
1433        let module_id = configured.module_id.clone();
1434        let registered = match supervisor.register_configured_with_health(
1435            configured.module_spec(),
1436            enabled,
1437            health.clone(),
1438            configured.drain_timeout_ms,
1439            configured.restart,
1440            configured.register_timeout,
1441        ) {
1442            Ok(registered) => registered,
1443            Err(err) => {
1444                error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
1445                continue;
1446            }
1447        };
1448        registered_modules.push(registered);
1449        // A relaxed threshold can silently outlive the temporary allowance
1450        // that required it, so report the extra tolerance on every boot.
1451        let default_threshold = HealthConfig::default().failure_threshold;
1452        if enabled && health.failure_threshold > default_threshold {
1453            warn!(
1454                module_id = %module_id,
1455                failure_threshold = health.failure_threshold,
1456                default_threshold,
1457                tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1458                "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1459            );
1460        }
1461        info!(module_id = %module_id, enabled, "configured module registered");
1462    }
1463    control.refresh_capability_requirements();
1464
1465    // The local endpoint is served beside TCP and answers the same handshake.
1466    // It serves clients only, which the server enforces per connection.
1467    let local_serve_task = bound.local_listener.map(|local_listener| {
1468        AbortOnDrop::new(tokio::spawn(serve_local_listener(
1469            local_listener,
1470            Arc::clone(&router),
1471            auth.clone(),
1472        )))
1473    });
1474    // On Unix the task is dropped below to stop accepting at shutdown. Windows
1475    // leaves it to the process exit; naming it keeps the binding uniform.
1476    #[cfg(windows)]
1477    let _local_serve_task = local_serve_task;
1478    let mut serve_task =
1479        AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1480    verify_serving(&bound.connection_info).await?;
1481    // A published endpoint promises that authentication can complete now.
1482    // Orphan cleanup and control-plane construction can take longer than an
1483    // existing-daemon probe's deadline, so finish both before discovery.
1484    // The write fsyncs, so it runs on tokio's blocking pool (see
1485    // `publish_connection_file`). The singleton lock travels with it and comes
1486    // back only once the file is in place.
1487    let (published, publication_lock) = publish_connection_file(
1488        &bound.connection_file_path,
1489        #[cfg(windows)]
1490        bound.connection_file_copy.as_deref(),
1491        &bound.connection_info,
1492        bound.local_endpoint.as_ref(),
1493        publication_lock,
1494    )
1495    .await;
1496    published?;
1497    // Serving and discoverable: a daemon booted from a restart intent records
1498    // readiness, which makes that generation the daemon's active one. Without
1499    // this record the next start treats the warm-up as failed.
1500    if let Some(owner) = daemon_ready_runtime {
1501        if let Err(error) = owner.on_blocking(|owner| owner.record_daemon_ready()).await {
1502            warn!(%error, "daemon readiness was not recorded in the approved store");
1503        }
1504    }
1505    #[cfg(test)]
1506    startup_publication_hook::run(&bound.connection_file_path).await;
1507    drop(publication_lock);
1508    let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1509    // Off the startup path: it runs `systemctl`, and only ever logs.
1510    #[cfg(target_os = "linux")]
1511    let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1512        crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1513    ));
1514    let _watchdog_task = AbortOnDrop::new(
1515        DaemonSelfWatchdog::new(
1516            bound.connection_info.clone(),
1517            bound.connection_file_path.clone(),
1518        )
1519        .with_config(watchdog_config)
1520        .spawn(),
1521    );
1522
1523    for registered in registered_modules {
1524        match supervisor
1525            .start_registered_configured_async(registered, None, true)
1526            .await
1527        {
1528            Ok(module) => {
1529                info!(module_id = %module.module_id(), "configured module supervised");
1530            }
1531            Err(err) => {
1532                error!(error = %err, "failed to start configured module; continuing daemon startup");
1533            }
1534        }
1535    }
1536
1537    control.refresh_capability_requirements();
1538    Arc::clone(&control).spawn_capability_deadline_loop();
1539    Arc::clone(&control).spawn_scope_expiry_loop();
1540    let _config_delivery_task = control.spawn_config_delivery_worker().map(AbortOnDrop::new);
1541    control.spawn_resource_sampler();
1542
1543    #[cfg(unix)]
1544    {
1545        tokio::select! {
1546            result = serve_task.join() => {
1547                return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1548            }
1549            _ = terminate.recv() => {}
1550        }
1551        // First, before the notice, the drain, or any connection close: from
1552        // here on a module exit is recorded as `daemon_shutdown` and never
1553        // respawned. Also before allowing a second signal to cut the bounded
1554        // wait short, so the journal marker is always written.
1555        supervisor.begin_daemon_shutdown();
1556        // Stop the self-watchdog before closing the listener. Its next tick would
1557        // connect to that listener, fail, and log an ERROR indistinguishable from
1558        // a wedged daemon, once for every interval a planned stop lasts.
1559        drop(_watchdog_task);
1560        // Dropping the listener stops new accepts, not established connections:
1561        // their detached tasks must remain live throughout notice and drain. The
1562        // local endpoint stops accepting the same way.
1563        drop(serve_task);
1564        drop(local_serve_task);
1565        let second_sigterm = async {
1566            terminate.recv().await;
1567            std::time::Instant::now()
1568        };
1569        tokio::pin!(second_sigterm);
1570        let mut escalation_started = tokio::select! {
1571            biased;
1572            at = second_sigterm.as_mut() => {
1573                info!("second SIGTERM: abandoning daemon shutdown wait");
1574                Some(at)
1575            }
1576            result = supervisor.drain_for_daemon_shutdown() => {
1577                if let Err(error) = result {
1578                    warn!(%error, "daemon shutdown drain failed; exiting anyway");
1579                }
1580                None
1581            }
1582        };
1583        // Supervised modules lead their own process groups, so a service
1584        // manager's kill of this process's group does not reach them. (A
1585        // systemd unit with KillMode=control-group kills by cgroup instead and
1586        // does reach them; see `systemd_kill_mode`.) The daemon ends them
1587        // itself: EOF (or SIGTERM for a protocol none child) first, then
1588        // signals at each child's own drain deadline.
1589        let already_escalated = escalation_started.is_some();
1590        supervisor
1591            .end_children_for_daemon_shutdown(already_escalated, async {
1592                if already_escalated {
1593                    std::future::pending::<()>().await;
1594                } else {
1595                    escalation_started = Some(second_sigterm.await);
1596                }
1597            })
1598            .await;
1599        if let Some(at) = escalation_started {
1600            info!(
1601                second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1602                "daemon shutdown wait ended after second SIGTERM"
1603            );
1604        }
1605        info!("daemon exiting: SIGTERM shutdown complete");
1606        Ok(())
1607    }
1608    #[cfg(not(unix))]
1609    serve_task
1610        .join()
1611        .await
1612        .map_err(BootstrapError::ServeJoin)?
1613        .map_err(BootstrapError::Serve)
1614}
1615
1616fn normalized_build_provenance(value: &str) -> Option<String> {
1617    match value.trim() {
1618        "" | "unavailable" => None,
1619        value => Some(value.to_string()),
1620    }
1621}
1622
1623/// Find an existing daemon or atomically bind loopback TCP for this daemon.
1624///
1625/// The algorithm is intentionally connect-first: an endpoint from the connection
1626/// file is treated as live only after the TCP+key server-proof authenticates for
1627/// that file's key and daemon_id. Stale or foreign connection files are reclaimed
1628/// only while holding the per-user start lock; the TCP port is never the
1629/// singleton primitive.
1630pub async fn ensure_singleton(
1631    connection_file_path: impl AsRef<Path>,
1632    port: u16,
1633) -> Result<Outcome, BootstrapError> {
1634    ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1635}
1636
1637pub async fn ensure_singleton_with_config(
1638    config: BootstrapConfig,
1639) -> Result<Outcome, BootstrapError> {
1640    ensure_singleton_inner(config, true).await
1641}
1642
1643async fn ensure_singleton_inner(
1644    config: BootstrapConfig,
1645    publish: bool,
1646) -> Result<Outcome, BootstrapError> {
1647    let Some(lock) = singleton_start_lock(&config.connection_file_path).await? else {
1648        return Ok(Outcome::AlreadyRunning);
1649    };
1650    ensure_singleton_with_start_lock(config, publish, lock).await
1651}
1652
1653async fn singleton_start_lock(path: &Path) -> Result<Option<StartLock>, BootstrapError> {
1654    if stable_peer(path).await? {
1655        return Ok(None);
1656    }
1657
1658    let lock = StartLock::acquire(path).await?;
1659
1660    // Re-probe after acquiring the start lock so a peer that won the race between
1661    // our first failed probe and the lock acquisition is observed instead of
1662    // overwritten.
1663    if stable_peer(path).await? {
1664        return Ok(None);
1665    }
1666    Ok(Some(lock))
1667}
1668
1669async fn ensure_singleton_with_start_lock(
1670    config: BootstrapConfig,
1671    publish: bool,
1672    lock: StartLock,
1673) -> Result<Outcome, BootstrapError> {
1674    #[cfg(windows)]
1675    let copy_path = windows_connection_copy::resolve(&config);
1676    let path = config.connection_file_path;
1677    // The held start lock and its peer check only rule out a daemon using this
1678    // connection file. A daemon started with another runtime directory but
1679    // the same data home shares the run directory while passing both, so the
1680    // run directory needs its own owner: refuse at once if a live daemon holds
1681    // it, before anything below reads or writes run or data-home state.
1682    let run_dir_lock = config
1683        .live_children_path
1684        .as_deref()
1685        .map(crate::run_dir_lock::RunDirLock::acquire)
1686        .transpose()?;
1687
1688    remove_stale_connection_file_if_present(&path)?;
1689    #[cfg(windows)]
1690    let copy_path = windows_connection_copy::reclaim(&path, copy_path).await;
1691
1692    // Established under the start lock, and under the run-directory lock when
1693    // there is one, and before binding: a corrupt file stops boot before
1694    // anything is published, and two daemons racing to start cannot both mint.
1695    //
1696    // Minting a new machine id fsyncs its file, so loading or minting runs on
1697    // tokio's blocking pool. Both locks travel with that work, so neither is
1698    // released while a mint is in flight.
1699    let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1700        None => (None, lock, run_dir_lock),
1701        Some(machine_id_path) => {
1702            let task_path = machine_id_path.clone();
1703            match run_blocking_holding((lock, run_dir_lock), move || {
1704                crate::machine_id::load_or_mint(&task_path)
1705            })
1706            .await
1707            {
1708                Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1709                Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1710                Err(join_error) => {
1711                    return Err(BootstrapError::MachineId(
1712                        crate::machine_id::MachineIdFileError::Write {
1713                            path: machine_id_path,
1714                            source: io::Error::other(describe_blocking_task_failure(
1715                                "loading or minting the machine id",
1716                                &join_error,
1717                            )),
1718                        },
1719                    ))
1720                }
1721            }
1722        }
1723    };
1724
1725    let (listeners, endpoints) = bind_loopback(config.port).await?;
1726    // The local endpoint is an addition to the daemon, not a condition for it:
1727    // a bind failure leaves clients on TCP only, with the connection file naming
1728    // no local endpoint, and the reason reported by `server.describe`. A daemon
1729    // that refused to start would take every module down for a feature nothing
1730    // has to use yet.
1731    let (local_listener, local_endpoint, local_endpoint_unavailable) =
1732        if config.local_listener.configured_dir().is_some() {
1733            match local_listener::bind(&config.local_listener, &path) {
1734                Ok(local_listener) => {
1735                    let endpoint = local_listener.endpoint();
1736                    (Some(local_listener), Some(endpoint), None)
1737                }
1738                Err(error) => {
1739                    let reason = error.to_string();
1740                    warn!(
1741                        error = %reason,
1742                        "the local endpoint is unavailable; serving loopback TCP only"
1743                    );
1744                    (None, None, Some(reason))
1745                }
1746            }
1747        } else {
1748            (None, None, None)
1749        };
1750    let connection_info = ConnectionInfo {
1751        schema: SCHEMA_VERSION,
1752        wire_version: Some(PROTOCOL_VERSION),
1753        endpoints,
1754        key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1755        daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1756        pid: process::id(),
1757        daemon_ver: config.daemon_ver,
1758    };
1759
1760    // The low-level binding API historically publishes for callers that serve
1761    // these listeners themselves. The daemon entry point defers publication
1762    // until its own authentication server is running.
1763    let publication_lock = if publish {
1764        let (published, lock) = publish_connection_file(
1765            &path,
1766            #[cfg(windows)]
1767            copy_path.as_deref(),
1768            &connection_info,
1769            local_endpoint.as_ref(),
1770            lock,
1771        )
1772        .await;
1773        if let Err(error) = published {
1774            if let Some(local_listener) = local_listener.as_ref() {
1775                local_listener::release(local_listener);
1776            }
1777            drop(listeners);
1778            return Err(error);
1779        }
1780        drop(lock);
1781        None
1782    } else {
1783        Some(lock)
1784    };
1785
1786    Ok(Outcome::Bound(BoundDaemon {
1787        listeners,
1788        local_listener,
1789        local_endpoint,
1790        local_endpoint_unavailable,
1791        connection_info,
1792        connection_file_path: path,
1793        connection_file_source: config.connection_file_source,
1794        #[cfg(windows)]
1795        connection_file_copy: copy_path,
1796        machine_id,
1797        run_dir_lock,
1798        publication_lock,
1799    }))
1800}
1801
1802#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1803enum Probe {
1804    Live,
1805    StaleOrAbsent,
1806}
1807
1808async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1809    if !matches!(probe_existing(path).await?, Probe::Live) {
1810        return Ok(false);
1811    }
1812    // A terminating peer can still authenticate while its listeners drain.
1813    // Require another proof after a short settling interval before reporting
1814    // a successful duplicate start. If the peer left, proceed with startup;
1815    // any remaining bind/ownership conflict is an error, not exit 0 (which
1816    // would suppress launchd's restart-on-failure policy).
1817    tokio::time::sleep(Duration::from_millis(1500)).await;
1818    Ok(matches!(probe_existing(path).await?, Probe::Live))
1819}
1820
1821async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1822    let info = match connection_file::read(path) {
1823        Ok(info) => info,
1824        Err(source) if is_absent_or_stale_connection_file(&source) => {
1825            return Ok(Probe::StaleOrAbsent)
1826        }
1827        Err(source) => {
1828            return Err(BootstrapError::ConnectionFileRead {
1829                path: path.to_path_buf(),
1830                source,
1831            })
1832        }
1833    };
1834
1835    for endpoint in &info.endpoints {
1836        if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1837            return Ok(Probe::Live);
1838        }
1839    }
1840
1841    Ok(Probe::StaleOrAbsent)
1842}
1843
1844async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1845    let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1846        return Probe::StaleOrAbsent;
1847    };
1848    if !ip.is_loopback() {
1849        return Probe::StaleOrAbsent;
1850    }
1851    let addr = SocketAddr::new(ip, endpoint.port);
1852
1853    let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1854        Ok(Ok(stream)) => stream,
1855        Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1856    };
1857
1858    match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1859        Ok(()) => Probe::Live,
1860        Err(AuthError::DaemonIdMismatch)
1861        | Err(AuthError::InvalidServerProof)
1862        | Err(AuthError::UnexpectedEof { .. })
1863        | Err(AuthError::Timeout { .. })
1864        | Err(AuthError::JsonEncode { .. })
1865        | Err(AuthError::JsonDecode { .. })
1866        | Err(AuthError::Io { .. })
1867        | Err(AuthError::MessageTooLarge { .. })
1868        | Err(AuthError::KeyTooShort { .. })
1869        | Err(AuthError::Random(_))
1870        | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1871    }
1872}
1873
1874async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1875    // Scheduling a server task is not a readiness barrier. Authenticate once
1876    // against the mandatory IPv4 listener before exposing discovery to peers.
1877    // This uses the normal probe deadline, with no retries or widened budget.
1878    if let Some(endpoint) = info.endpoints.first() {
1879        if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1880            return Ok(());
1881        }
1882    }
1883    Err(BootstrapError::StartupNotServing)
1884}
1885
1886/// Run blocking filesystem work (an fsync, a rename) on tokio's blocking-thread
1887/// pool rather than on the async worker thread that awaits it. On a slow or
1888/// busy disk an fsync can take seconds, and a runtime has few worker threads
1889/// (an in-process test daemon may share two with everything else), so doing it
1890/// inline stalls unrelated tasks for that long.
1891///
1892/// `guards` (lock handles) move into the blocking task and are handed back
1893/// with the work's result. If the awaiting future is dropped first, the work
1894/// still runs to completion on its thread and the guards are released only
1895/// after it ends, so a lock is never given up while the work it protects is
1896/// still writing.
1897///
1898/// `Err` means the task panicked, or was cancelled before it started because
1899/// the runtime is shutting down; the guards have then been released already.
1900async fn run_blocking_holding<G, T>(
1901    guards: G,
1902    work: impl FnOnce() -> T + Send + 'static,
1903) -> Result<(T, G), JoinError>
1904where
1905    G: Send + 'static,
1906    T: Send + 'static,
1907{
1908    tokio::task::spawn_blocking(move || {
1909        let output = work();
1910        (output, guards)
1911    })
1912    .await
1913}
1914
1915/// Explain, for a startup error, why blocking work produced no result.
1916fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1917    if join_error.is_panic() {
1918        format!("the background thread {work} panicked before finishing: {join_error}")
1919    } else {
1920        format!(
1921            "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1922        )
1923    }
1924}
1925
1926/// Publish the connection file: owner-only permissions, written to a temporary
1927/// file, fsynced and renamed over `path` (all done by `write_atomic`). The
1928/// write runs on tokio's blocking pool through [`run_blocking_holding`], so the
1929/// fsync never holds an async worker thread.
1930///
1931/// `guard` is the singleton start lock (or whatever the caller must keep held
1932/// across publication); it is returned with the result. It comes back `None`
1933/// only when the write task panicked or was cancelled, in which case the
1934/// result is an error and the file must be treated as not published.
1935///
1936/// On Windows, `copy_path` asks for the profile copy as well. Its start lock is
1937/// taken before the primary write. Once `path` is written, an error returned
1938/// while preparing or writing the copy only logs a warning, and the result is
1939/// still success. The two writes share one blocking task but aren't one atomic
1940/// step, and a panic or cancellation of that task is still a publication
1941/// error.
1942async fn publish_connection_file<G: Send + 'static>(
1943    path: &Path,
1944    #[cfg(windows)] copy_path: Option<&Path>,
1945    info: &ConnectionInfo,
1946    local_endpoint: Option<&LocalEndpoint>,
1947    guard: G,
1948) -> (Result<(), BootstrapError>, Option<G>) {
1949    let task_path = path.to_path_buf();
1950    let task_info = info.clone();
1951    let task_endpoint = local_endpoint.cloned();
1952    #[cfg(windows)]
1953    let copy = windows_connection_copy::claim_for_publication(path, copy_path, info).await;
1954    let outcome = run_blocking_holding(guard, move || {
1955        write_connection_file(&task_path, &task_info, task_endpoint.as_ref())?;
1956        #[cfg(windows)]
1957        if let Some((copy_path, _lock)) = copy {
1958            if let Err(error) =
1959                write_connection_file(&copy_path, &task_info, task_endpoint.as_ref())
1960            {
1961                windows_connection_copy::warn_unavailable(&task_path, Some(&copy_path), &error);
1962            }
1963        }
1964        Ok(())
1965    })
1966    .await;
1967    match outcome {
1968        Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1969        Ok((Err(source), guard)) => (
1970            Err(BootstrapError::ConnectionFileWrite {
1971                path: path.to_path_buf(),
1972                source,
1973            }),
1974            Some(guard),
1975        ),
1976        Err(join_error) => (
1977            Err(BootstrapError::ConnectionFileWrite {
1978                path: path.to_path_buf(),
1979                source: ConnectionFileError::Io {
1980                    op: "write",
1981                    path: path.to_path_buf(),
1982                    source: io::Error::other(describe_blocking_task_failure(
1983                        "writing the connection file",
1984                        &join_error,
1985                    )),
1986                },
1987            }),
1988            None,
1989        ),
1990    }
1991}
1992
1993/// The synchronous connection-file write, including its fsync. Call it only
1994/// through [`publish_connection_file`], which keeps it off the async workers.
1995///
1996/// `local_endpoint` is written through subc-transport's endpoint-aware writer,
1997/// so the file carries `local_endpoint` when there is a local listener and is
1998/// byte-identical to before when there is none.
1999fn write_connection_file(
2000    path: &Path,
2001    info: &ConnectionInfo,
2002    local_endpoint: Option<&LocalEndpoint>,
2003) -> Result<(), ConnectionFileError> {
2004    #[cfg(test)]
2005    connection_file_write_hook::run(path);
2006    write_atomic_with_local_endpoint(path, info, local_endpoint)
2007}
2008
2009/// Lets a test run code inside the connection-file write, on whichever thread
2010/// performs it, for example to block there and check what else can still run.
2011#[cfg(test)]
2012mod connection_file_write_hook {
2013    use std::{
2014        path::{Path, PathBuf},
2015        sync::{Mutex, PoisonError},
2016    };
2017
2018    type Hook = Box<dyn FnOnce() + Send>;
2019
2020    /// Keyed by connection-file path, so a hook fires only for the test that
2021    /// installed it while other tests publish their own files in parallel.
2022    static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
2023
2024    /// Run `hook` once, at the start of the next write of `path`.
2025    pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
2026        HOOKS
2027            .lock()
2028            .unwrap_or_else(PoisonError::into_inner)
2029            .push((path.to_path_buf(), Box::new(hook)));
2030    }
2031
2032    pub(super) fn run(path: &Path) {
2033        let hook = {
2034            let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
2035            hooks
2036                .iter()
2037                .position(|(hooked, _)| hooked == path)
2038                .map(|index| hooks.swap_remove(index).1)
2039        };
2040        if let Some(hook) = hook {
2041            hook();
2042        }
2043    }
2044}
2045
2046/// A path-scoped barrier after publication and before any configured activation.
2047/// The async wait leaves the listeners free to answer discovery-time requests.
2048#[cfg(test)]
2049mod startup_publication_hook {
2050    use std::{
2051        path::{Path, PathBuf},
2052        sync::{Mutex, PoisonError},
2053    };
2054    use tokio::sync::oneshot;
2055
2056    type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
2057    static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
2058
2059    pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
2060        let (entered_tx, entered_rx) = oneshot::channel();
2061        let (release_tx, release_rx) = oneshot::channel();
2062        BARRIERS
2063            .lock()
2064            .unwrap_or_else(PoisonError::into_inner)
2065            .push((path.to_path_buf(), (entered_tx, release_rx)));
2066        (entered_rx, release_tx)
2067    }
2068
2069    pub(super) async fn run(path: &Path) {
2070        let barrier = {
2071            let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
2072            barriers
2073                .iter()
2074                .position(|(hooked, _)| hooked == path)
2075                .map(|index| barriers.swap_remove(index).1)
2076        };
2077        if let Some((entered, release)) = barrier {
2078            let _ = entered.send(());
2079            let _ = release.await;
2080        }
2081    }
2082}
2083
2084fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
2085    match err {
2086        ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
2087        ConnectionFileError::JsonRead { .. }
2088        | ConnectionFileError::UnsupportedSchema { .. }
2089        | ConnectionFileError::Invalid { .. }
2090        | ConnectionFileError::KeyTooShort { .. }
2091        // A live daemon always publishes the file owner-only (0600), so a file
2092        // with insecure permissions is never a daemon we should defer to: treat it
2093        // as stale and take over (which republishes a correct 0600 file).
2094        | ConnectionFileError::InsecurePermissions { .. } => true,
2095        ConnectionFileError::MissingParent { .. }
2096        | ConnectionFileError::MissingFileName { .. }
2097        // A writable ancestor is an operator misconfiguration, never evidence
2098        // about whether a daemon is live. Reclaiming the file would republish key
2099        // material into the same directory the refusal is about.
2100        | ConnectionFileError::InsecureParentDirectory { .. }
2101        | ConnectionFileError::Io { .. }
2102        | ConnectionFileError::JsonWrite { .. }
2103        | ConnectionFileError::Random(_)
2104        // A wire mismatch may identify a newer live daemon, so never reclaim its
2105        // connection file merely because this binary cannot speak its envelope.
2106        | ConnectionFileError::WireVersionMismatch { .. } => false,
2107    }
2108}
2109
2110fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
2111    match fs::remove_file(path) {
2112        Ok(()) => Ok(()),
2113        Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
2114        Err(source) => Err(BootstrapError::RemoveStale {
2115            path: path.to_path_buf(),
2116            source,
2117        }),
2118    }
2119}
2120
2121async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
2122    bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
2123}
2124
2125async fn bind_loopback_with<F, Fut>(
2126    port: u16,
2127    mut bind_v6: F,
2128) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
2129where
2130    F: FnMut(u16) -> Fut,
2131    Fut: std::future::Future<Output = io::Result<TcpListener>>,
2132{
2133    let v4_host = Ipv4Addr::LOCALHOST;
2134    let v4 = TcpListener::bind((v4_host, port))
2135        .await
2136        .map_err(|source| BootstrapError::Bind {
2137            host: v4_host.to_string(),
2138            port,
2139            source,
2140        })?;
2141    let actual_port = v4
2142        .local_addr()
2143        .map_err(|source| BootstrapError::LocalAddr {
2144            host: v4_host.to_string(),
2145            source,
2146        })?
2147        .port();
2148
2149    let mut listeners = vec![v4];
2150    let mut endpoints = vec![Endpoint {
2151        host: v4_host.to_string(),
2152        port: actual_port,
2153    }];
2154
2155    let v6_host = Ipv6Addr::LOCALHOST;
2156    match bind_v6(actual_port).await {
2157        Ok(v6) => {
2158            listeners.push(v6);
2159            endpoints.push(Endpoint {
2160                host: v6_host.to_string(),
2161                port: actual_port,
2162            });
2163        }
2164        Err(err)
2165            if ipv6_loopback_unavailable(&err)
2166                || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
2167        {
2168            warn!(
2169                port = actual_port,
2170                error = %err,
2171                "IPv6 loopback unavailable; serving only IPv4 loopback"
2172            );
2173        }
2174        Err(source) => {
2175            drop(listeners);
2176            return Err(BootstrapError::Bind {
2177                host: v6_host.to_string(),
2178                port: actual_port,
2179                source,
2180            });
2181        }
2182    }
2183
2184    Ok((listeners, endpoints))
2185}
2186
2187fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
2188    matches!(
2189        err.kind(),
2190        io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
2191    ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
2192}
2193
2194struct AbortOnDrop<T> {
2195    handle: JoinHandle<T>,
2196}
2197
2198impl<T> AbortOnDrop<T> {
2199    fn new(handle: JoinHandle<T>) -> Self {
2200        Self { handle }
2201    }
2202
2203    async fn join(&mut self) -> Result<T, JoinError> {
2204        (&mut self.handle).await
2205    }
2206}
2207
2208impl<T> Drop for AbortOnDrop<T> {
2209    fn drop(&mut self) {
2210        if !self.handle.is_finished() {
2211            self.handle.abort();
2212        }
2213    }
2214}
2215
2216#[derive(Debug)]
2217struct StartLock {
2218    // Keep the locked file handle alive for the duration of bootstrap.
2219    _file: fs::File,
2220}
2221
2222impl Drop for StartLock {
2223    fn drop(&mut self) {
2224        // Closing alone leaves the lock held if a concurrent fork inherited
2225        // the open file description and has not exec'd yet. Unlock explicitly
2226        // so a restart can reclaim the singleton as soon as bootstrap ends.
2227        if let Err(error) = FileExt::unlock(&self._file) {
2228            tracing::warn!(%error, "failed to release the singleton start lock");
2229        }
2230    }
2231}
2232
2233impl StartLock {
2234    async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
2235        let path = start_lock_path(connection_file_path);
2236        for _ in 0..START_LOCK_RETRIES {
2237            let file = match open_owner_only_lock(&path) {
2238                Ok(file) => file,
2239                Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
2240            };
2241            match FileExt::try_lock(&file) {
2242                Ok(()) => return Ok(Self { _file: file }),
2243                Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
2244                Err(TryLockError::Error(source)) => {
2245                    return Err(BootstrapError::StartLockCreate { path, source });
2246                }
2247            }
2248        }
2249
2250        Err(BootstrapError::StartLockBusy {
2251            path,
2252            attempts: START_LOCK_RETRIES,
2253        })
2254    }
2255}
2256
2257pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
2258    let mut options = fs::OpenOptions::new();
2259    options.read(true).write(true).create(true);
2260    #[cfg(unix)]
2261    {
2262        use std::os::unix::fs::OpenOptionsExt;
2263        options.mode(0o600);
2264    }
2265    options.open(path)
2266}
2267
2268fn start_lock_path(connection_file_path: &Path) -> PathBuf {
2269    let file_name = connection_file_path
2270        .file_name()
2271        .map(|name| name.to_string_lossy())
2272        .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
2273    let lock_name = format!("{file_name}.start-lock");
2274    connection_file_path
2275        .parent()
2276        .filter(|parent| !parent.as_os_str().is_empty())
2277        .unwrap_or_else(|| Path::new("."))
2278        .join(lock_name)
2279}
2280
2281fn non_empty_os_var(key: &str) -> Option<OsString> {
2282    let value = env::var_os(key)?;
2283    if value.is_empty() {
2284        None
2285    } else {
2286        Some(value)
2287    }
2288}
2289
2290/// Bootstrap-layer errors are deliberately typed so startup never panics for
2291/// ordinary daemon-discovery races or stale filesystem state.
2292#[derive(Debug)]
2293#[non_exhaustive]
2294pub enum BootstrapError {
2295    ConfigStore(String),
2296    /// The bound server could not authenticate its own readiness probe.
2297    StartupNotServing,
2298    #[cfg(unix)]
2299    Signal(io::Error),
2300    InvalidPort {
2301        raw: String,
2302        source: std::num::ParseIntError,
2303    },
2304    ConnectionFileRead {
2305        path: PathBuf,
2306        source: ConnectionFileError,
2307    },
2308    ConnectionFileWrite {
2309        path: PathBuf,
2310        source: ConnectionFileError,
2311    },
2312    GenerateConnectionFile(ConnectionFileError),
2313    StartLockCreate {
2314        path: PathBuf,
2315        source: io::Error,
2316    },
2317    StartLockBusy {
2318        path: PathBuf,
2319        attempts: usize,
2320    },
2321    /// The run-directory lock file could not be created, opened or locked.
2322    RunDirLockCreate {
2323        path: PathBuf,
2324        source: io::Error,
2325    },
2326    /// Another live process, almost certainly a daemon started with a
2327    /// different runtime directory over the same data home, owns the run
2328    /// directory. The daemon does not start. `holder_pid` is read from the
2329    /// lock file when possible and is informational only.
2330    RunDirBusy {
2331        path: PathBuf,
2332        holder_pid: Option<u32>,
2333    },
2334    RemoveStale {
2335        path: PathBuf,
2336        source: io::Error,
2337    },
2338    Bind {
2339        host: String,
2340        port: u16,
2341        source: io::Error,
2342    },
2343    LocalAddr {
2344        host: String,
2345        source: io::Error,
2346    },
2347    DaemonConfig(DaemonConfigError),
2348    /// The machine id could not be established: its file is corrupt, unreadable
2349    /// or unwritable, or the data home is relative. The daemon does not start.
2350    MachineId(crate::machine_id::MachineIdFileError),
2351    /// The daemon run directory could not be resolved because the data home is
2352    /// relative. The daemon does not start.
2353    RunDir(daemon_config::DaemonRunDirError),
2354    Serve(ServerError),
2355    ServeJoin(tokio::task::JoinError),
2356}
2357
2358impl From<ServerError> for BootstrapError {
2359    fn from(error: ServerError) -> Self {
2360        Self::Serve(error)
2361    }
2362}
2363
2364impl fmt::Display for BootstrapError {
2365    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2366        match self {
2367            Self::ConfigStore(error) => write!(f, "configuration store startup refused: {error}"),
2368            Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
2369            #[cfg(unix)]
2370            Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
2371            Self::InvalidPort { raw, source } => {
2372                write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
2373            }
2374            Self::ConnectionFileRead { path, source } => write!(
2375                f,
2376                "failed to read connection file {}: {source}",
2377                path.display()
2378            ),
2379            Self::ConnectionFileWrite { path, source } => write!(
2380                f,
2381                "failed to publish connection file {}: {source}",
2382                path.display()
2383            ),
2384            Self::GenerateConnectionFile(err) => {
2385                write!(f, "failed to generate connection-file auth material: {err}")
2386            }
2387            Self::StartLockCreate { path, source } => {
2388                write!(
2389                    f,
2390                    "failed to create start lock {}: {source}",
2391                    path.display()
2392                )
2393            }
2394            Self::StartLockBusy { path, attempts } => write!(
2395                f,
2396                "start lock {} remained busy after {attempts} attempts",
2397                path.display()
2398            ),
2399            Self::RunDirLockCreate { path, source } => write!(
2400                f,
2401                "refusing to start: failed to lock run directory via {}: {source}",
2402                path.display()
2403            ),
2404            Self::RunDirBusy { path, holder_pid } => {
2405                write!(
2406                    f,
2407                    "refusing to start: run directory lock {} is held by another daemon",
2408                    path.display()
2409                )?;
2410                if let Some(pid) = holder_pid {
2411                    write!(f, " (pid {pid})")?;
2412                }
2413                write!(
2414                    f,
2415                    "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
2416                )
2417            }
2418            Self::RemoveStale { path, source } => write!(
2419                f,
2420                "failed to remove stale connection file {}: {source}",
2421                path.display()
2422            ),
2423            Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
2424                write!(
2425                    f,
2426                    "port {port} in use on loopback {host}: {source}; set the port in config"
2427                )
2428            }
2429            Self::Bind { host, port, source } => {
2430                write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
2431            }
2432            Self::LocalAddr { host, source } => {
2433                write!(f, "failed to read local address for {host}: {source}")
2434            }
2435            Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
2436            Self::MachineId(err) => write!(f, "refusing to start: {err}"),
2437            Self::RunDir(err) => write!(f, "refusing to start: {err}"),
2438            Self::Serve(err) => write!(f, "daemon server failed: {err}"),
2439            Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
2440        }
2441    }
2442}
2443
2444impl Error for BootstrapError {
2445    fn source(&self) -> Option<&(dyn Error + 'static)> {
2446        match self {
2447            Self::ConfigStore(_) => None,
2448            Self::StartupNotServing => None,
2449            #[cfg(unix)]
2450            Self::Signal(source) => Some(source),
2451            Self::InvalidPort { source, .. } => Some(source),
2452            Self::ConnectionFileRead { source, .. }
2453            | Self::ConnectionFileWrite { source, .. }
2454            | Self::GenerateConnectionFile(source) => Some(source),
2455            Self::StartLockCreate { source, .. }
2456            | Self::RunDirLockCreate { source, .. }
2457            | Self::RemoveStale { source, .. }
2458            | Self::Bind { source, .. }
2459            | Self::LocalAddr { source, .. } => Some(source),
2460            Self::DaemonConfig(err) => Some(err),
2461            Self::MachineId(err) => Some(err),
2462            Self::RunDir(err) => Some(err),
2463            Self::Serve(err) => Some(err),
2464            Self::ServeJoin(err) => Some(err),
2465            Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
2466        }
2467    }
2468}
2469
2470#[cfg(all(test, feature = "test-support", unix))]
2471#[path = "bootstrap_recovery_tests.rs"]
2472mod recovery_tests;
2473
2474#[cfg(test)]
2475mod tests {
2476    mod activity {
2477        include!("activity_bootstrap_tests.rs");
2478    }
2479    mod startup_order {
2480        include!("startup_order_tests.rs");
2481    }
2482    use super::*;
2483    use crate::server::ServerAuth;
2484    use cortexkit_test_support::ScratchDir;
2485    #[cfg(target_os = "linux")]
2486    use std::collections::BTreeSet;
2487    use std::sync::Mutex;
2488    #[cfg(target_os = "linux")]
2489    use subc_control::ModuleProtocol;
2490    use subc_transport::{write_atomic, MIN_KEY_LEN};
2491    use tokio::io::AsyncReadExt;
2492    use tokio::task::JoinHandle;
2493
2494    #[cfg(unix)]
2495    use std::os::unix::fs::PermissionsExt;
2496
2497    static ENV_LOCK: Mutex<()> = Mutex::new(());
2498
2499    #[test]
2500    fn normalized_build_provenance_preserves_real_values() {
2501        assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
2502    }
2503
2504    #[test]
2505    fn config_recovery_bootstrap_preserves_all_loaded_fields_and_only_argument_overrides() {
2506        let _env_lock = ENV_LOCK.lock().unwrap();
2507        let scratch = unique_temp_dir("bootstrap-loaded-fields");
2508        let file = scratch.join("subc.jsonc");
2509        std::fs::write(
2510            &file,
2511            br#"{"version":1,"modules":{},"scope_authority_owners":["file-owner"]}"#,
2512        )
2513        .unwrap();
2514        let mut config = BootstrapConfig::new(scratch.join("connection.json"), 0)
2515            .with_daemon_config_source(file.clone(), None);
2516        config.defer_file_load = true;
2517        config.operator_timeout = Duration::from_secs(7);
2518        let selected = load_bootstrap_authority(config, None).unwrap();
2519        assert_eq!(
2520            selected.file_derived_test_probe,
2521            Some(vec!["file-owner".into()])
2522        );
2523        assert_eq!(selected.operator_timeout, Duration::from_secs(7));
2524        assert_eq!(selected.daemon_config_path, Some(file));
2525        assert_eq!(
2526            selected.connection_file_path,
2527            scratch.join("connection.json")
2528        );
2529    }
2530
2531    #[test]
2532    fn config_recovery_bootstrap_ignores_invalid_live_file_and_poisoned_settings() {
2533        let scratch = unique_temp_dir("recovery-bootstrap");
2534        let root = scratch.join("store");
2535        std::fs::create_dir_all(&root).unwrap();
2536        std::fs::write(
2537            root.join(crate::config_store::STORE_FILE),
2538            br#"{"format_version":999}"#,
2539        )
2540        .unwrap();
2541        let mut store = crate::config_store::ConfigStore::open_digest_only_for_tests(
2542            crate::config_store::StoreRoot::from_service_definition(root).unwrap(),
2543        );
2544        assert_eq!(
2545            store.status().state,
2546            crate::config_store::StoreState::StoreUnverified
2547        );
2548        let file = scratch.join("subc.jsonc");
2549        std::fs::write(&file, b"invalid live config").unwrap();
2550        let mut config = BootstrapConfig::new(scratch.join("connection.json"), 1)
2551            .with_daemon_config_source(file, Some(1))
2552            .with_scope_authority_owners(vec!["untrusted".into()])
2553            .with_reserved_capabilities(BTreeMap::from([("untrusted".into(), "untrusted".into())]));
2554        config.defer_file_load = true;
2555        config.file_derived_test_probe = Some(vec!["untrusted".into()]);
2556        let config = load_bootstrap_authority(config, Some(&mut store)).unwrap();
2557        assert_eq!(config.file_derived_test_probe, None);
2558        assert_eq!(config.port, DEFAULT_SUBC_PORT);
2559        assert!(config.configured_modules.is_empty());
2560        assert_eq!(
2561            config.scope_authority_owners,
2562            daemon_config::default_scope_authority_owners()
2563        );
2564        assert!(config.reserved_capabilities.is_empty());
2565        assert_eq!(
2566            config.privileged_operator_prompt,
2567            daemon_config::PrivilegedOperatorPrompt::default()
2568        );
2569    }
2570
2571    /// An approved generation whose daemon bytes listen on `port`. Changing
2572    /// the port is a setting only bootstrap reads, so it waits for a restart.
2573    fn approved_port_entry(
2574        kind: crate::config_store::CommitKind,
2575        port: u16,
2576    ) -> crate::config_store::HistoryEntry {
2577        use subc_protocol::config::{ConfigLayer, ConfigLayerSnapshot};
2578        crate::config_store::HistoryEntry {
2579            generation: subc_config::Generation::new(
2580                0,
2581                subc_config::ApprovalChannel::OsPrompt,
2582                subc_config::raw_digest(b"approved-plan"),
2583                subc_config::InputVector::default(),
2584                subc_config::Changeset::new(),
2585            ),
2586            kind,
2587            approval_id: "approval".into(),
2588            outcome: "approved".into(),
2589            tuples: Default::default(),
2590            configuration: Default::default(),
2591            delivery: Some(crate::config_store::DeliverySnapshot {
2592                daemon: Some(ConfigLayerSnapshot::file(
2593                    "subc",
2594                    ConfigLayer::User,
2595                    "/approved/subc.jsonc",
2596                    Some(format!(r#"{{"version":1,"port":{port},"modules":{{}}}}"#).into_bytes()),
2597                )),
2598                modules: Default::default(),
2599                declarations: Default::default(),
2600            }),
2601        }
2602    }
2603
2604    fn approve_port(
2605        store: &mut crate::config_store::ConfigStore,
2606        kind: crate::config_store::CommitKind,
2607        port: u16,
2608    ) {
2609        store
2610            .commit_generation(
2611                approved_port_entry(kind, port),
2612                Default::default(),
2613                Default::default(),
2614                1,
2615            )
2616            .unwrap();
2617    }
2618
2619    fn stamped_boot(
2620        scratch: &Path,
2621        store: &mut crate::config_store::ConfigStore,
2622    ) -> BootstrapConfig {
2623        let mut config = BootstrapConfig::new(scratch.join("connection.json"), 1)
2624            .with_daemon_config_source(scratch.join("subc.jsonc"), Some(1));
2625        config.defer_file_load = true;
2626        load_bootstrap_authority(config, Some(store)).unwrap()
2627    }
2628
2629    #[test]
2630    fn config_stamped_boot_uses_last_active_generation_until_restart_intent() {
2631        use crate::config_store::{CommitKind, ConfigStore, StoreRoot};
2632        let scratch = unique_temp_dir("stamped-boot-selection");
2633        let root = scratch.join("store");
2634        std::fs::create_dir_all(&root).unwrap();
2635        let open = || {
2636            ConfigStore::open_digest_only_for_tests(
2637                StoreRoot::from_service_definition(root.clone()).unwrap(),
2638            )
2639        };
2640        let mut store = open();
2641        approve_port(&mut store, CommitKind::Stamp, 41001);
2642        approve_port(&mut store, CommitKind::Approved, 41002);
2643        drop(store);
2644        let mut store = open();
2645        assert_eq!(
2646            stamped_boot(&scratch, &mut store).port,
2647            41001,
2648            "an ordinary start must not boot an approval still waiting for a daemon restart"
2649        );
2650        assert_eq!(store.assign_daemon_restart().unwrap(), Some(2));
2651        drop(store);
2652        let mut store = open();
2653        assert_eq!(stamped_boot(&scratch, &mut store).port, 41002);
2654    }
2655
2656    #[test]
2657    fn config_stamped_boot_enters_recovery_when_anchor_retry_fails_at_start() {
2658        use crate::config_store::{CommitKind, ConfigStore, StoreRoot, StoreState, TestStoreKey};
2659        let scratch = unique_temp_dir("stamped-boot-anchor-pending");
2660        let root = scratch.join("store");
2661        std::fs::create_dir_all(&root).unwrap();
2662        let key = TestStoreKey::default();
2663        let open = || {
2664            ConfigStore::open_sealed_for_tests(
2665                StoreRoot::from_service_definition(root.clone()).unwrap(),
2666                key.clone(),
2667            )
2668        };
2669        let mut store = open();
2670        approve_port(&mut store, CommitKind::Stamp, 41001);
2671        // Crash between publication and anchor: the bytes are renamed into
2672        // place but the anchor never catches up, and keeps failing at start.
2673        key.fail_anchor(true);
2674        assert!(store
2675            .commit_generation(
2676                approved_port_entry(CommitKind::Approved, 41001),
2677                Default::default(),
2678                Default::default(),
2679                2,
2680            )
2681            .is_err());
2682        drop(store);
2683        let mut store = open();
2684        let status = store.status();
2685        assert_eq!(status.state, StoreState::Stamped);
2686        assert!(status.anchor_pending);
2687        let config = stamped_boot(&scratch, &mut store);
2688        assert_eq!(config.port, DEFAULT_SUBC_PORT);
2689        assert!(config.configured_modules.is_empty());
2690        assert_eq!(store.status().state, StoreState::StoreUnverified);
2691    }
2692
2693    #[test]
2694    fn normalized_build_provenance_omits_unavailable_and_empty_values() {
2695        assert_eq!(normalized_build_provenance("unavailable"), None);
2696        assert_eq!(normalized_build_provenance(""), None);
2697    }
2698
2699    #[cfg(target_os = "linux")]
2700    fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
2701        let cgroups = fs::read_to_string("/proc/self/cgroup")?;
2702        let relative = cgroups
2703            .lines()
2704            .find_map(|line| line.strip_prefix("0::"))
2705            .ok_or_else(|| {
2706                io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
2707            })?;
2708        Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
2709    }
2710
2711    #[cfg(target_os = "linux")]
2712    fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
2713        let modules = current_cgroup_path_for_test()?.join("subc-modules");
2714        let entries = match fs::read_dir(modules) {
2715            Ok(entries) => entries,
2716            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
2717            Err(error) => return Err(error),
2718        };
2719        let mut directories = BTreeSet::new();
2720        for entry in entries {
2721            let entry = entry?;
2722            if entry.file_type()?.is_dir() {
2723                directories.insert(entry.file_name());
2724            }
2725        }
2726        Ok(directories)
2727    }
2728
2729    #[cfg(target_os = "linux")]
2730    async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
2731        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2732        while !path.exists() && tokio::time::Instant::now() < deadline {
2733            assert!(
2734                !task.is_finished(),
2735                "daemon exited before creating {}",
2736                path.display()
2737            );
2738            sleep(Duration::from_millis(10)).await;
2739        }
2740        assert!(path.exists(), "daemon did not create {}", path.display());
2741    }
2742
2743    /// An in-process daemon with the default (disabled) cgroup placement must
2744    /// leave the host's live module cgroup tree exactly as it found it.
2745    /// Red-checking this test (making it fail) creates a
2746    /// `cgroup-isolation-probe-*` directory in the live cgroup tree, which must be removed by hand.
2747    #[cfg(target_os = "linux")]
2748    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2749    async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
2750        let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
2751        let module_id = format!("cgroup-isolation-probe-{}", process::id());
2752        let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
2753        assert!(
2754            !before.contains(&OsString::from(&module_id)),
2755            "isolation probe cgroup already exists before this daemon starts"
2756        );
2757        let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
2758        let module = ConfiguredModule {
2759            script: None,
2760            carry_containment: false,
2761            module_id,
2762            program: PathBuf::from("sh"),
2763            args: vec!["-c".to_string(), "sleep 30".to_string()],
2764            env: Vec::new(),
2765            log: None,
2766            enabled: true,
2767            registration_channel: false,
2768            reserved: false,
2769            reserved_prefixes: Vec::new(),
2770            protocol: ModuleProtocol::None,
2771            overlap: Default::default(),
2772            health: HealthConfig::default(),
2773            drain_timeout_ms: None,
2774            register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2775            route_bind_relay_timeout_ms: None,
2776            restart: RestartPolicy::default(),
2777        };
2778        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2779            .with_configured_modules([module])
2780            .with_capture_logs_dir(temp.join("logs"))
2781            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2782        let task = tokio::spawn(run_with_config(config));
2783
2784        wait_for_path(&capture, &task).await;
2785        let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2786
2787        task.abort();
2788        assert!(task
2789            .await
2790            .expect_err("aborted daemon task must cancel")
2791            .is_cancelled());
2792        assert_eq!(
2793            before, after,
2794            "an in-process daemon must not create or reconcile ambient module cgroups"
2795        );
2796    }
2797
2798    #[cfg(target_os = "linux")]
2799    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2800    async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2801        let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2802        let cgroup_root = temp.join("cgroup");
2803        fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2804        fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2805        let modules = cgroup_root.join("subc-modules");
2806        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2807            .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2808            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2809        let task = tokio::spawn(run_with_config(config));
2810
2811        wait_for_path(&modules, &task).await;
2812
2813        task.abort();
2814        assert!(task
2815            .await
2816            .expect_err("aborted daemon task must cancel")
2817            .is_cancelled());
2818        assert!(
2819            fs::read_dir(&modules)
2820                .expect("read prepared modules directory")
2821                .next()
2822                .is_none(),
2823            "the delegation probe must clean up after itself"
2824        );
2825    }
2826
2827    struct EnvGuard {
2828        key: &'static str,
2829        previous: Option<OsString>,
2830    }
2831
2832    impl EnvGuard {
2833        fn set(key: &'static str, value: &Path) -> Self {
2834            let previous = env::var_os(key);
2835            env::set_var(key, value);
2836            Self { key, previous }
2837        }
2838
2839        fn set_str(key: &'static str, value: &str) -> Self {
2840            let previous = env::var_os(key);
2841            env::set_var(key, value);
2842            Self { key, previous }
2843        }
2844
2845        fn unset(key: &'static str) -> Self {
2846            let previous = env::var_os(key);
2847            env::remove_var(key);
2848            Self { key, previous }
2849        }
2850    }
2851
2852    impl Drop for EnvGuard {
2853        fn drop(&mut self) {
2854            match &self.previous {
2855                Some(value) => env::set_var(self.key, value),
2856                None => env::remove_var(self.key),
2857            }
2858        }
2859    }
2860
2861    fn unique_temp_dir(name: &str) -> ScratchDir {
2862        ScratchDir::new(name)
2863    }
2864
2865    fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2866        let dir = unique_temp_dir(name);
2867        let path = dir.join("conn.json");
2868        (dir, path)
2869    }
2870
2871    fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2872        ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2873    }
2874
2875    fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2876        let auth = auth_for(&bound.connection_info);
2877        tokio::spawn(serve_listeners(
2878            bound.listeners,
2879            Arc::new(Router::with_default_self_handler()),
2880            auth,
2881        ))
2882    }
2883
2884    fn expect_bound(outcome: Outcome) -> BoundDaemon {
2885        match outcome {
2886            Outcome::Bound(bound) => bound,
2887            Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2888        }
2889    }
2890
2891    async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2892        let endpoint = conn
2893            .endpoints
2894            .first()
2895            .expect("test connection file should have an endpoint");
2896        let ip: IpAddr = endpoint.host.parse().unwrap();
2897        TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2898    }
2899
2900    fn make_connection_info(port: u16) -> ConnectionInfo {
2901        ConnectionInfo {
2902            schema: SCHEMA_VERSION,
2903            wire_version: Some(PROTOCOL_VERSION),
2904            endpoints: vec![Endpoint {
2905                host: "127.0.0.1".to_owned(),
2906                port,
2907            }],
2908            key: generate_key().unwrap(),
2909            daemon_id: generate_daemon_id().unwrap(),
2910            pid: process::id(),
2911            daemon_ver: "test-subc".to_owned(),
2912        }
2913    }
2914
2915    fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2916        fs::write(path, contents).unwrap();
2917        #[cfg(unix)]
2918        fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2919    }
2920
2921    fn assert_owner_only_connection_file(path: &Path) {
2922        // `path` is only inspected on Unix (mode bits); on Windows the owner-only
2923        // guarantee comes from the inherited %TEMP% ACL, nothing to assert here.
2924        #[cfg(unix)]
2925        {
2926            let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2927            assert_eq!(mode, 0o600);
2928        }
2929        #[cfg(not(unix))]
2930        let _ = path;
2931    }
2932
2933    #[test]
2934    fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2935        let _env_lock = ENV_LOCK.lock().unwrap();
2936        let runtime_dir = unique_temp_dir("xdg-runtime");
2937        let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2938
2939        assert_eq!(
2940            connection_file_path(),
2941            runtime_dir.join(CONNECTION_FILE_NAME)
2942        );
2943    }
2944
2945    #[test]
2946    fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2947        let runtime_dir = OsString::from("/run/user/1000");
2948
2949        let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2950
2951        assert_eq!(
2952            path,
2953            PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2954        );
2955        assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2956    }
2957
2958    #[test]
2959    fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2960        let _env_lock = ENV_LOCK.lock().unwrap();
2961        let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2962
2963        assert_eq!(
2964            connection_file_path(),
2965            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2966        );
2967    }
2968
2969    #[test]
2970    fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2971        let (path, source) = connection_file_path_with_source(None);
2972
2973        assert_eq!(
2974            path,
2975            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2976        );
2977        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2978    }
2979
2980    /// Concurrent callers must derive one token. The former temp-file uid probe
2981    /// could fail transiently (same-tick name collision, fd exhaustion) and send
2982    /// the loser down the env-derived fallback with a different identity for the
2983    /// same user; this fence keeps identity independent of filesystem luck.
2984    #[test]
2985    fn user_connection_token_is_stable_under_concurrent_callers() {
2986        let expected = user_connection_token();
2987        let workers: Vec<_> = (0..32)
2988            .map(|_| {
2989                std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2990            })
2991            .collect();
2992        for worker in workers {
2993            for token in worker.join().expect("probe thread") {
2994                assert_eq!(token, expected, "token diverged under concurrent probes");
2995            }
2996        }
2997    }
2998
2999    #[test]
3000    fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
3001        let (path, source) = connection_file_path_with_source(Some(OsString::new()));
3002
3003        assert_eq!(
3004            path,
3005            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
3006        );
3007        assert_eq!(source, ConnectionFileSource::TempDirFallback);
3008    }
3009
3010    /// The shipped binary is the only caller that journals terminal exits into
3011    /// the real run directory, so its constructor must supply that path itself;
3012    /// the in-process default leaves it unset.
3013    #[test]
3014    fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
3015        let _env_lock = ENV_LOCK.lock().unwrap();
3016        let root = unique_temp_dir("daemon-binary-config");
3017        let data_home = root.join("data");
3018        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3019        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3020        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3021
3022        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3023
3024        let run_dir = data_home.join("cortexkit").join("run");
3025        assert_eq!(
3026            config.terminal_journal_path,
3027            Some(run_dir.join("terminals.jsonl"))
3028        );
3029        assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
3030        assert_eq!(
3031            BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
3032            None,
3033            "an in-process config must not journal anywhere unless asked to"
3034        );
3035    }
3036
3037    #[test]
3038    fn daemon_binary_config_refuses_a_relative_data_home() {
3039        let _env_lock = ENV_LOCK.lock().unwrap();
3040        let root = unique_temp_dir("daemon-binary-relative-data");
3041        let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
3042        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3043        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3044
3045        let error = BootstrapConfig::from_env_for_daemon_binary()
3046            .expect_err("a relative data home must refuse the daemon binary's config");
3047        assert!(
3048            matches!(error, BootstrapError::RunDir(_)),
3049            "expected a run-directory refusal, got {error}"
3050        );
3051        assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
3052    }
3053
3054    #[test]
3055    fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
3056        let _env_lock = ENV_LOCK.lock().unwrap();
3057        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3058        let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
3059        let config_path = conn_path.with_file_name("subc.jsonc");
3060        for (doc, expected) in [
3061            (None, daemon_config::PrivilegedOperatorPrompt::Log),
3062            (
3063                Some(r#"{"version":1}"#),
3064                daemon_config::PrivilegedOperatorPrompt::Log,
3065            ),
3066            (
3067                Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
3068                daemon_config::PrivilegedOperatorPrompt::Log,
3069            ),
3070            (
3071                Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
3072                daemon_config::PrivilegedOperatorPrompt::Enforce,
3073            ),
3074        ] {
3075            if let Some(doc) = doc {
3076                fs::write(&config_path, doc).unwrap();
3077            }
3078            assert_eq!(
3079                BootstrapConfig::from_env_with_daemon_config_path(&config_path)
3080                    .unwrap()
3081                    .privileged_operator_prompt,
3082                expected
3083            );
3084            assert_eq!(
3085                BootstrapConfig::new(&conn_path, 0)
3086                    .with_daemon_config_path(&config_path)
3087                    .unwrap()
3088                    .privileged_operator_prompt,
3089                expected
3090            );
3091        }
3092    }
3093
3094    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3095    async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
3096        let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
3097        let connection_path = dir.join("connection.json");
3098        let config_path = dir.join("subc.jsonc");
3099        fs::write(
3100            &config_path,
3101            r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
3102        )
3103        .unwrap();
3104        let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
3105            .with_daemon_config_path(&config_path)
3106            .unwrap();
3107        let connection_path = config.connection_file_path.clone();
3108        let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
3109        tokio::time::timeout(Duration::from_secs(5), async {
3110            while !connection_path.exists() {
3111                assert!(
3112                    !daemon.handle.is_finished(),
3113                    "daemon exited before publishing"
3114                );
3115                tokio::task::yield_now().await;
3116            }
3117        })
3118        .await
3119        .unwrap();
3120        let info = connection_file::read(&connection_path).unwrap();
3121        let mut client = connect_from_info(&info).await.unwrap();
3122        authenticate_client(&mut client, &info, Duration::from_secs(5))
3123            .await
3124            .unwrap();
3125        for (correlation, mode, expected) in [
3126            (1, "enforce", Vec::new()),
3127            (2, "log", vec!["privileged_operator_prompt"]),
3128            (3, "log", vec!["privileged_operator_prompt"]),
3129        ] {
3130            fs::write(
3131                &config_path,
3132                serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
3133            )
3134            .unwrap();
3135            let frame = crate::Frame::build(
3136                subc_protocol::FrameType::Request,
3137                subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
3138                0,
3139                0,
3140                correlation,
3141                serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
3142                    preview: true,
3143                })
3144                .unwrap(),
3145            )
3146            .unwrap();
3147            crate::write_frame(&mut client, &frame).await.unwrap();
3148            let reply =
3149                tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
3150                    .await
3151                    .unwrap()
3152                    .unwrap()
3153                    .unwrap();
3154            assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
3155            assert_eq!(reply.header.corr, correlation);
3156            let subc_control::ClientControlResponse::SupervisorRescan { result } =
3157                serde_json::from_slice(&reply.body).unwrap()
3158            else {
3159                panic!("expected a rescan response");
3160            };
3161            assert_eq!(result.restart_required, expected);
3162        }
3163    }
3164
3165    #[test]
3166    fn configured_port_uses_default_config_and_env_override() {
3167        let _env_lock = ENV_LOCK.lock().unwrap();
3168        let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
3169        let config_path = conn_path.with_file_name("subc.jsonc");
3170
3171        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3172        assert_eq!(
3173            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
3174                .unwrap()
3175                .port,
3176            DEFAULT_SUBC_PORT
3177        );
3178
3179        fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
3180        assert_eq!(
3181            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
3182                .unwrap()
3183                .port,
3184            8123
3185        );
3186
3187        let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
3188        assert_eq!(
3189            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
3190                .unwrap()
3191                .port,
3192            9012
3193        );
3194    }
3195
3196    #[tokio::test]
3197    async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
3198        let (_dir, path) = temp_connection_file_path("departing-peer");
3199        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3200        let info = make_connection_info(listener.local_addr().unwrap().port());
3201        write_connection_file(&path, &info, None).unwrap();
3202        let peer = tokio::spawn(async move {
3203            let (mut stream, _) = listener.accept().await.unwrap();
3204            subc_transport::authenticate_server(
3205                &mut stream,
3206                &info.key,
3207                &info.daemon_id,
3208                &info.daemon_ver,
3209                PROBE_AUTH_DEADLINE,
3210            )
3211            .await
3212            .unwrap();
3213        });
3214        let outcome = ensure_singleton(&path, 0).await.unwrap();
3215        peer.await.unwrap();
3216        assert!(
3217            matches!(outcome, Outcome::Bound(_)),
3218            "a departing peer is not AlreadyRunning"
3219        );
3220    }
3221
3222    #[tokio::test]
3223    async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
3224        let (_dir, path) = temp_connection_file_path("already-running");
3225
3226        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3227        let server = start_server(bound);
3228
3229        let second = ensure_singleton(&path, 0).await.unwrap();
3230        assert!(matches!(second, Outcome::AlreadyRunning));
3231
3232        server.abort();
3233        let _ = server.await;
3234    }
3235
3236    #[tokio::test]
3237    async fn daemon_connection_file_publishes_protocol_wire_version() {
3238        let (_dir, path) = temp_connection_file_path("wire-version");
3239        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3240        assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
3241        assert_eq!(
3242            connection_file::read(&path).unwrap().wire_version,
3243            Some(PROTOCOL_VERSION)
3244        );
3245
3246        drop(bound.listeners);
3247    }
3248
3249    #[tokio::test]
3250    async fn stale_unbound_connection_file_is_reclaimed() {
3251        let (_dir, path) = temp_connection_file_path("stale-reclaim");
3252        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
3253        let stale_port = stale.local_addr().unwrap().port();
3254        drop(stale);
3255        let stale_info = make_connection_info(stale_port);
3256        write_atomic(&path, &stale_info).unwrap();
3257
3258        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3259        assert_ne!(bound.connection_info.key, stale_info.key);
3260        drop(bound.listeners);
3261    }
3262
3263    #[cfg(unix)]
3264    #[tokio::test]
3265    async fn ensure_singleton_reclaims_insecure_connection_file() {
3266        let (_dir, path) = temp_connection_file_path("insecure-reclaim");
3267        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
3268        let stale_port = stale.local_addr().unwrap().port();
3269        drop(stale);
3270        let stale_info = make_connection_info(stale_port);
3271        write_atomic(&path, &stale_info).unwrap();
3272        fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
3273
3274        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3275        assert_ne!(bound.connection_info.key, stale_info.key);
3276        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
3277        assert_owner_only_connection_file(&path);
3278
3279        drop(bound.listeners);
3280    }
3281
3282    #[tokio::test]
3283    async fn ensure_singleton_reclaims_non_loopback_connection_file() {
3284        let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
3285        let mut stale_info = make_connection_info(8757);
3286        stale_info.endpoints = vec![Endpoint {
3287            host: "192.0.2.10".to_owned(),
3288            port: 8757,
3289        }];
3290        write_atomic(&path, &stale_info).unwrap();
3291
3292        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
3293        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3294        assert_ne!(bound.connection_info.key, stale_info.key);
3295        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
3296        assert!(bound
3297            .connection_info
3298            .endpoints
3299            .iter()
3300            .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
3301        assert_owner_only_connection_file(&path);
3302
3303        drop(bound.listeners);
3304    }
3305
3306    #[tokio::test]
3307    async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
3308        let mut unsupported_schema = make_connection_info(8757);
3309        unsupported_schema.schema = SCHEMA_VERSION + 1;
3310
3311        let mut empty_endpoints = make_connection_info(8757);
3312        empty_endpoints.endpoints.clear();
3313
3314        let mut short_key = make_connection_info(8757);
3315        short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
3316
3317        let cases = vec![
3318            (
3319                "unsupported-schema",
3320                serde_json::to_vec(&unsupported_schema).unwrap(),
3321                Some(unsupported_schema),
3322            ),
3323            (
3324                "empty-endpoints",
3325                serde_json::to_vec(&empty_endpoints).unwrap(),
3326                Some(empty_endpoints),
3327            ),
3328            (
3329                "short-key",
3330                serde_json::to_vec(&short_key).unwrap(),
3331                Some(short_key),
3332            ),
3333            ("invalid-json", b"{not valid connection json".to_vec(), None),
3334        ];
3335
3336        for (label, contents, old_info) in cases {
3337            let (_dir, path) = temp_connection_file_path(label);
3338            write_raw_owner_only_connection_file(&path, &contents);
3339
3340            let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3341            if let Some(old_info) = old_info {
3342                assert_ne!(bound.connection_info.key, old_info.key, "{label}");
3343                assert_ne!(
3344                    bound.connection_info.daemon_id, old_info.daemon_id,
3345                    "{label}"
3346                );
3347            }
3348            assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
3349            assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
3350            assert_owner_only_connection_file(&path);
3351
3352            drop(bound.listeners);
3353        }
3354    }
3355
3356    #[tokio::test]
3357    async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
3358        let (_dir, path) = temp_connection_file_path("foreign-reclaim");
3359        let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
3360        let foreign_port = foreign.local_addr().unwrap().port();
3361        write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
3362        let foreign_task = tokio::spawn(async move {
3363            if let Ok((mut stream, _)) = foreign.accept().await {
3364                let mut buf = [0u8; 64];
3365                let _ = stream.read(&mut buf).await;
3366            }
3367        });
3368
3369        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3370        assert!(bound
3371            .connection_info
3372            .endpoints
3373            .iter()
3374            .all(|endpoint| endpoint.port != foreign_port));
3375
3376        drop(bound.listeners);
3377        let _ = foreign_task.await;
3378    }
3379
3380    #[tokio::test]
3381    async fn stale_start_lock_file_is_reclaimable() {
3382        let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
3383        let lock_path = start_lock_path(&path);
3384        drop(open_owner_only_lock(&lock_path).unwrap());
3385        assert!(lock_path.is_file());
3386
3387        let lock = StartLock::acquire(&path).await.unwrap();
3388        assert!(lock_path.is_file());
3389
3390        drop(lock);
3391        assert!(lock_path.is_file());
3392    }
3393
3394    #[tokio::test]
3395    async fn held_start_lock_blocks_second_acquire_until_release() {
3396        let (_dir, path) = temp_connection_file_path("start-lock-held");
3397        let lock_path = start_lock_path(&path);
3398        let first = StartLock::acquire(&path).await.unwrap();
3399
3400        let err = match StartLock::acquire(&path).await {
3401            Ok(_) => panic!("second acquire while held must stay busy"),
3402            Err(err) => err,
3403        };
3404        assert!(matches!(
3405            err,
3406            BootstrapError::StartLockBusy {
3407                ref path,
3408                attempts: START_LOCK_RETRIES,
3409            } if path == &lock_path
3410        ));
3411
3412        drop(first);
3413
3414        let second = StartLock::acquire(&path)
3415            .await
3416            .expect("released advisory lock should be reclaimable");
3417        drop(second);
3418    }
3419
3420    #[tokio::test]
3421    async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
3422        let (_dir, path) = temp_connection_file_path("bind-conflict");
3423        let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
3424        let occupied_port = occupied.local_addr().unwrap().port();
3425
3426        let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
3427        assert!(matches!(
3428            err,
3429            BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
3430        ));
3431        assert!(err.to_string().contains("set the port in config"));
3432
3433        drop(occupied);
3434    }
3435
3436    #[tokio::test]
3437    async fn ephemeral_ipv6_collision_still_serves_ipv4() {
3438        let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
3439            let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
3440            let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
3441            drop(occupied);
3442            result
3443        })
3444        .await
3445        .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
3446        assert_eq!(endpoints.len(), 1);
3447        assert_eq!(endpoints[0].host, "127.0.0.1");
3448        TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
3449            .await
3450            .unwrap();
3451        drop(listeners);
3452    }
3453
3454    #[tokio::test]
3455    async fn unpublished_boot_holds_singleton_lock_until_publication() {
3456        let dir = ScratchDir::new("ul");
3457        let path = dir.join("c");
3458        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
3459        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
3460        assert!(!path.exists());
3461        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
3462        assert!(
3463            matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
3464            "an unpublished boot must keep its singleton claim"
3465        );
3466        drop(bound);
3467        FileExt::try_lock(&contender).unwrap();
3468    }
3469
3470    #[tokio::test]
3471    async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
3472        let dir = ScratchDir::new("sl");
3473        let path = dir.join("c");
3474        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
3475        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
3476        // try_clone shares the open file description just as fork does,
3477        // without relying on the timing of another test's child spawn.
3478        let inherited = bound
3479            .publication_lock
3480            .as_ref()
3481            .unwrap()
3482            ._file
3483            .try_clone()
3484            .unwrap();
3485        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
3486        assert!(matches!(
3487            FileExt::try_lock(&contender),
3488            Err(TryLockError::WouldBlock)
3489        ));
3490        drop(bound);
3491        FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
3492        drop(inherited);
3493    }
3494
3495    #[tokio::test]
3496    async fn readiness_probe_requires_an_authenticating_server() {
3497        let dir = ScratchDir::new("unserved-readiness");
3498        let bound = expect_bound(
3499            ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
3500                .await
3501                .unwrap(),
3502        );
3503        assert!(matches!(
3504            verify_serving(&bound.connection_info).await,
3505            Err(BootstrapError::StartupNotServing)
3506        ));
3507    }
3508
3509    #[cfg(unix)]
3510    #[tokio::test]
3511    async fn orphan_sweep_finishes_before_connection_file_publication() {
3512        use crate::live_children::{ExecutableIdentity, LiveChild};
3513        let dir = ScratchDir::new("pas");
3514        // A one-letter connection file name, and no long socket path: the sweep
3515        // this test covers runs before the local endpoint is bound, so a longer
3516        // name would only add a way for the test to fail on something else.
3517        let path = dir.join("c");
3518        let record = dir.join("live-children.json");
3519        let ready = dir.join("ready");
3520        let mut child = std::process::Command::new("/bin/sh")
3521            .args([
3522                "-c",
3523                "trap '' TERM; : > \"$0\"; read -r _",
3524                ready.to_str().unwrap(),
3525            ])
3526            .stdin(std::process::Stdio::piped())
3527            .env("XDG_DATA_HOME", dir.path())
3528            .env("XDG_RUNTIME_DIR", dir.path())
3529            .env("XDG_CONFIG_HOME", dir.path())
3530            .spawn()
3531            .unwrap();
3532        timeout(Duration::from_secs(5), async {
3533            while !ready.exists() {
3534                sleep(Duration::from_millis(5)).await;
3535            }
3536        })
3537        .await
3538        .unwrap();
3539        let observed = subc_os::Process::open(child.id())
3540            .unwrap()
3541            .unwrap()
3542            .observe()
3543            .unwrap();
3544        crate::live_children::write_record(
3545            &record,
3546            &[LiveChild {
3547                container: None,
3548                module_id: "old-sleep".into(),
3549                pid: child.id(),
3550                protocol: subc_control::ModuleProtocol::None,
3551                start_time: Some(observed.start_time),
3552                executable: observed.executable.map(ExecutableIdentity::from),
3553                cgroup_name: None,
3554            }],
3555        )
3556        .unwrap();
3557        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
3558            .with_live_children_record(&record);
3559        let daemon = tokio::spawn(run_with_config(config));
3560        timeout(Duration::from_secs(5), async {
3561            while !path.exists() {
3562                sleep(Duration::from_millis(5)).await;
3563            }
3564        })
3565        .await
3566        .unwrap();
3567        assert!(path.exists());
3568        let ended = child.try_wait().unwrap().is_some();
3569        if !ended {
3570            child.kill().unwrap();
3571        }
3572        child.wait().unwrap();
3573        if ended {
3574            assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
3575        }
3576        daemon.abort();
3577        let _ = daemon.await;
3578        assert!(
3579            ended,
3580            "the published daemon must not still owe its orphan sweep"
3581        );
3582    }
3583
3584    /// Publishing the connection file fsyncs, which can take seconds on a busy
3585    /// disk, so the daemon must do it off the async worker thread. The hook
3586    /// blocks inside the write until another task on the same single-threaded
3587    /// runtime has run. Writing on the worker thread would starve that task: the
3588    /// hook then gives up after its deadline and the test fails, as it does if
3589    /// startup publishes without going through the hooked write at all.
3590    #[tokio::test(flavor = "current_thread")]
3591    async fn connection_file_publication_leaves_the_async_runtime_free() {
3592        use std::sync::atomic::{AtomicBool, Ordering};
3593
3594        let (_dir, path) = temp_connection_file_path("publish-off-runtime");
3595        let write_started = Arc::new(AtomicBool::new(false));
3596        let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
3597        let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
3598        let hook_started = Arc::clone(&write_started);
3599        connection_file_write_hook::install(&path, move || {
3600            hook_started.store(true, Ordering::SeqCst);
3601            let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
3602            let _ = verdict_tx.send(progressed);
3603        });
3604        // Signals only after the write has started, so a message can only come
3605        // from this task running while the write is still blocked.
3606        let ticker = tokio::spawn(async move {
3607            loop {
3608                if write_started.load(Ordering::SeqCst) {
3609                    let _ = progress_tx.send(());
3610                    return;
3611                }
3612                sleep(Duration::from_millis(5)).await;
3613            }
3614        });
3615
3616        let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
3617        let published = timeout(Duration::from_secs(15), async {
3618            while !path.exists() && !daemon.is_finished() {
3619                sleep(Duration::from_millis(5)).await;
3620            }
3621        })
3622        .await;
3623        let daemon_exited = daemon.is_finished();
3624        daemon.abort();
3625        let daemon_result = daemon.await;
3626        ticker.abort();
3627
3628        assert!(
3629            published.is_ok() && !daemon_exited,
3630            "the daemon must start and publish its connection file: {daemon_result:?}"
3631        );
3632        match verdict_rx.try_recv() {
3633            Ok(true) => {}
3634            Ok(false) => panic!(
3635                "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
3636            ),
3637            Err(_) => panic!("startup published the connection file without the hooked write"),
3638        }
3639    }
3640
3641    /// A write that never reports back must stop startup with an error naming
3642    /// the connection file, never be taken as published.
3643    #[tokio::test]
3644    async fn a_panicking_connection_file_write_fails_startup() {
3645        let (_dir, path) = temp_connection_file_path("publish-panics");
3646        connection_file_write_hook::install(&path, || {
3647            panic!("deliberate panic inside the connection-file write")
3648        });
3649
3650        let error = timeout(
3651            Duration::from_secs(15),
3652            run_with_config(BootstrapConfig::new(&path, 0)),
3653        )
3654        .await
3655        .expect("startup must end once the write task fails")
3656        .expect_err("startup must fail when the connection-file write panics");
3657
3658        assert!(
3659            matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
3660            "expected a connection-file write error, got {error:?}"
3661        );
3662        let message = error.to_string();
3663        assert!(message.contains(&path.display().to_string()), "{message}");
3664        assert!(message.contains("panicked"), "{message}");
3665        assert!(!path.exists(), "no connection file may be published");
3666    }
3667
3668    #[cfg(target_os = "macos")]
3669    #[test]
3670    fn nofile_raise_worker() {
3671        let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
3672            return;
3673        };
3674        let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
3675        rlimit::Resource::NOFILE.set(256, hard).unwrap();
3676        raise_nofile_limit_with_ceiling(Some(10240));
3677        std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
3678    }
3679
3680    #[cfg(target_os = "macos")]
3681    #[test]
3682    fn macos_nofile_raise_clamps_to_kernel_ceiling() {
3683        let dir = ScratchDir::new("nofile-ceiling");
3684        let result = dir.join("soft");
3685        let status = std::process::Command::new(std::env::current_exe().unwrap())
3686            .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
3687            .env("SUBC_NOFILE_TEST_FILE", &result)
3688            .env("XDG_DATA_HOME", dir.path())
3689            .env("XDG_RUNTIME_DIR", dir.path())
3690            .env("XDG_CONFIG_HOME", dir.path())
3691            .status()
3692            .unwrap();
3693        assert!(status.success());
3694        let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
3695        assert!(
3696            soft > 256 && soft <= 10240,
3697            "the inherited limit must be raised within the kernel ceiling; got {soft}"
3698        );
3699    }
3700
3701    #[tokio::test]
3702    async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
3703        let (_dir, path) = temp_connection_file_path("key-rotation");
3704        let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3705        let old_info = first.connection_info.clone();
3706        let fixed_port = old_info.endpoints[0].port;
3707        drop(first.listeners);
3708
3709        let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
3710        let new_info = second.connection_info.clone();
3711        assert_ne!(old_info.key, new_info.key);
3712        assert_ne!(old_info.daemon_id, new_info.daemon_id);
3713        let server = start_server(second);
3714
3715        let mut old_stream = connect_from_info(&old_info).await.unwrap();
3716        let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
3717        assert!(
3718            old_auth.is_err(),
3719            "old key must not authenticate after restart"
3720        );
3721
3722        let reread = connection_file::read(&path).unwrap();
3723        let mut new_stream = connect_from_info(&reread).await.unwrap();
3724        authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
3725            .await
3726            .unwrap();
3727
3728        server.abort();
3729        let _ = server.await;
3730    }
3731
3732    #[cfg(unix)]
3733    #[tokio::test]
3734    async fn published_connection_file_permissions_are_owner_only() {
3735        let (_dir, path) = temp_connection_file_path("permissions");
3736        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3737
3738        let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
3739        assert_eq!(mode, 0o600);
3740
3741        drop(bound.listeners);
3742    }
3743
3744    /// A config with the socket in `dir`'s own short subdirectory.
3745    ///
3746    /// A Unix socket path is limited to just over a hundred bytes, and this
3747    /// harness's temporary directories use nearly all of that, so a test that
3748    /// boots a real daemon passes a short directory and a one-letter connection
3749    /// file name; anything longer makes the bind refuse. On Windows there is no
3750    /// socket path to keep short, so the config is returned unchanged.
3751    fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
3752        let dir = config
3753            .connection_file_path
3754            .parent()
3755            .map(Path::to_path_buf)
3756            .unwrap_or_else(|| PathBuf::from("."));
3757        #[cfg(unix)]
3758        let config = BootstrapConfig {
3759            connection_file_path: dir.join("c"),
3760            local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
3761            ..config
3762        };
3763        #[cfg(not(unix))]
3764        let _ = dir;
3765        config
3766    }
3767
3768    /// The shipped binary puts the socket in its own run directory. The file is
3769    /// named from the connection file's path, so a restart finds it again.
3770    #[cfg(unix)]
3771    #[test]
3772    fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
3773        let _env_lock = ENV_LOCK.lock().unwrap();
3774        let root = ScratchDir::new("binary-local-socket");
3775        let data_home = root.join("data");
3776        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3777        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3778        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3779
3780        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3781
3782        let run_dir = data_home.join("cortexkit").join("run");
3783        assert_eq!(
3784            config.local_listener.configured_dir(),
3785            Some(run_dir.as_path()),
3786            "the binary serves from its run directory"
3787        );
3788        let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
3789            .expect("the binary config names a directory");
3790        assert_eq!(socket.parent().unwrap(), run_dir);
3791        assert_eq!(
3792            socket.file_name().unwrap().to_string_lossy(),
3793            "subc.sock",
3794            "a fixed name: the run directory belongs to one data home"
3795        );
3796    }
3797
3798    /// An in-process daemon that names no directory serves TCP only: it publishes
3799    /// no local endpoint, creates nothing under the data home, and says nothing is
3800    /// unavailable, because nothing was asked for.
3801    /// The environment lock covers only the setup: the daemon under test reads
3802    /// its data home from the process environment while it boots, and holding
3803    /// the lock across that await would block every other test that sets one.
3804    #[cfg(unix)]
3805    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3806    async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
3807        let root = ScratchDir::new("in-process-no-local-socket");
3808        let data_home = root.join("data");
3809        fs::create_dir_all(&data_home).unwrap();
3810        let connection_path = root.join("c");
3811        let _data;
3812        let _config;
3813        let _port;
3814        {
3815            let _env_lock = ENV_LOCK.lock().unwrap();
3816            _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3817            _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3818            _port = EnvGuard::unset(SUBC_PORT_ENV);
3819        }
3820
3821        let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
3822
3823        assert!(bound.local_endpoint.is_none());
3824        assert!(bound.local_endpoint_unavailable.is_none());
3825        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3826        assert!(published.local_endpoint.is_none());
3827        assert!(
3828            local_listener::socket_path_for(None, &connection_path).is_none(),
3829            "no configured directory, no socket"
3830        );
3831        let mut leftovers = Vec::new();
3832        let mut pending = vec![data_home.clone()];
3833        while let Some(dir) = pending.pop() {
3834            for entry in fs::read_dir(&dir).unwrap() {
3835                let entry = entry.unwrap();
3836                let path = entry.path();
3837                if entry.file_type().unwrap().is_dir() {
3838                    pending.push(path.clone());
3839                }
3840                if entry.file_name().to_string_lossy().ends_with(".sock") {
3841                    leftovers.push(path);
3842                }
3843            }
3844        }
3845        assert!(
3846            leftovers.is_empty(),
3847            "an in-process daemon must not create a socket in the data home: {leftovers:?}"
3848        );
3849    }
3850
3851    /// A local endpoint that cannot be bound does not stop the daemon: it keeps
3852    /// serving TCP, publishes no local endpoint, and reports the reason where an
3853    /// operator reads daemon facts. Every module stays up for a feature nothing
3854    /// has to use.
3855    #[cfg(unix)]
3856    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3857    async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
3858        let dir = ScratchDir::new("tcp-only");
3859        let canonical = std::fs::canonicalize(&dir).unwrap();
3860        let connection_path = canonical.join("c");
3861        // The socket directory is deep enough that the socket's path is over the
3862        // Unix socket limit, so binding it must fail.
3863        let deep = canonical.join("x".repeat(80));
3864        fs::create_dir(&deep).unwrap();
3865        fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
3866        let config =
3867            BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
3868        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3869        assert!(
3870            bound.local_endpoint.is_none(),
3871            "a socket that cannot be bound is not published"
3872        );
3873        assert!(
3874            bound
3875                .local_endpoint_unavailable
3876                .as_deref()
3877                .is_some_and(|reason| reason.contains("too long")),
3878            "{:?}",
3879            bound.local_endpoint_unavailable
3880        );
3881        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3882        assert!(published.local_endpoint.is_none());
3883        assert!(
3884            !published.info.endpoints.is_empty(),
3885            "TCP is still published"
3886        );
3887
3888        // The daemon serves TCP as usual.
3889        let server = start_server(bound);
3890        let info = connection_file::read(&connection_path).unwrap();
3891        let mut client = connect_from_info(&info).await.unwrap();
3892        authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
3893            .await
3894            .expect("a TCP client still completes the handshake");
3895        server.abort();
3896        let _ = server.await;
3897    }
3898
3899    /// The published file names the local endpoint the daemon actually serves,
3900    /// and a client that trusts the file completes the handshake over it.
3901    #[cfg(unix)]
3902    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3903    async fn published_connection_file_names_the_served_local_endpoint() {
3904        let (_dir, path) = temp_connection_file_path("local-endpoint");
3905        // The socket goes in the system temporary directory rather than inside
3906        // the harness's own directory, whose long lockdown-sandbox path leaves
3907        // no room inside the Unix socket path limit.
3908        let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
3909        let config =
3910            BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
3911        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3912        let endpoint = connection_file::read_with_local_endpoint(&path)
3913            .unwrap()
3914            .local_endpoint
3915            .expect("the daemon serves a local endpoint, so the file names it");
3916        assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
3917        let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
3918            panic!("a Unix daemon publishes a socket path");
3919        };
3920        assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
3921        assert_eq!(
3922            fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
3923            0o600
3924        );
3925
3926        let router = Arc::new(Router::with_default_self_handler());
3927        let auth = auth_for(&bound.connection_info);
3928        let server = tokio::spawn(serve_local_listener(
3929            bound.local_listener.expect("the local listener is bound"),
3930            router,
3931            auth,
3932        ));
3933        let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
3934        authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
3935            .await
3936            .expect("a socket client completes the same handshake as a TCP client");
3937
3938        server.abort();
3939        drop(bound.listeners);
3940    }
3941
3942    /// A configured directory that is not a directory refuses at bind rather
3943    /// than publishing an endpoint nothing serves.
3944    #[cfg(unix)]
3945    #[test]
3946    fn a_socket_directory_that_is_a_file_refuses() {
3947        let dir = ScratchDir::new("local-listener-file-as-dir");
3948        let file = dir.join("not-a-directory");
3949        fs::write(&file, b"x").unwrap();
3950        let error = local_listener::bind(
3951            &LocalListenerConfig::in_socket_dir(file),
3952            &dir.join("connection.json"),
3953        )
3954        .expect_err("a path that is not a directory cannot hold the socket");
3955        assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
3956    }
3957
3958    /// The socket sits directly in the configured directory under one fixed
3959    /// name, and a directory that would put the path over the platform limit is
3960    /// refused rather than shortened around.
3961    #[cfg(unix)]
3962    #[test]
3963    fn the_socket_path_stays_within_the_unix_limit() {
3964        let run_dir = PathBuf::from("/tmp/short-run");
3965        assert!(
3966            local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
3967            "no configured directory means no socket"
3968        );
3969        let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
3970            .expect("a configured directory gets a socket");
3971        assert_eq!(socket, run_dir.join("subc.sock"));
3972        assert_eq!(
3973            socket,
3974            local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
3975            "the name does not come from the connection file"
3976        );
3977
3978        let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
3979        let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
3980            .expect("a configured directory gets a socket");
3981        assert!(
3982            long.as_os_str().len() < 104,
3983            "{} is {} bytes",
3984            long.display(),
3985            long.as_os_str().len()
3986        );
3987    }
3988}