Skip to main content

subc_daemon/
bootstrap.rs

1use std::{
2    collections::BTreeMap,
3    env,
4    error::Error,
5    ffi::OsString,
6    fmt, fs, io,
7    net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8    path::{Path, PathBuf},
9    process,
10    time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17    authenticate_client, connection_file, generate_daemon_id, generate_key,
18    write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
19    LocalEndpoint, SCHEMA_VERSION,
20};
21use tokio::{
22    net::{TcpListener, TcpStream},
23    task::{JoinError, JoinHandle},
24    time::{sleep, timeout},
25};
26use tracing::{error, info, warn};
27
28use crate::{
29    daemon_config::{self, ConfiguredModule, DaemonConfigError},
30    local_listener::{self, BoundLocalListener, LocalListenerConfig},
31    server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
32    supervise::HealthConfig,
33    ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
34    ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
35    SupervisorProcessLiveness,
36};
37use std::sync::Arc;
38
39#[cfg(windows)]
40mod windows_connection_copy;
41
42pub const DEFAULT_SUBC_PORT: u16 = 8757;
43pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
44use subc_transport::CONNECTION_FILE_NAME;
45const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
46const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
47const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
48const START_LOCK_RETRIES: usize = 40;
49const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum ConnectionFileSource {
53    XdgRuntimeDir,
54    TempDirFallback,
55    Explicit,
56}
57
58impl ConnectionFileSource {
59    fn reason(self) -> &'static str {
60        match self {
61            Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
62            Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
63            Self::Explicit => "configured path",
64        }
65    }
66}
67
68impl fmt::Display for ConnectionFileSource {
69    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
70        formatter.write_str(match self {
71            Self::XdgRuntimeDir => "xdg_runtime_dir",
72            Self::TempDirFallback => "temp_dir_fallback",
73            Self::Explicit => "explicit",
74        })
75    }
76}
77
78/// Runtime bootstrap configuration. Production uses the default fixed port and
79/// optional daemon-config override; tests pass port 0 to let the OS assign a free
80/// loopback port and discover it from the connection file.
81#[derive(Debug, Clone, Default)]
82struct AdmissionFactsConfig {
83    carrier_module_id: Option<String>,
84    targets: Option<Vec<String>>,
85}
86
87/// Controls where module cgroups are prepared.
88///
89/// In-process daemons default to [`Self::Disabled`] so they never derive a
90/// production location from the host process. The shipped daemon explicitly uses
91/// [`Self::Current`]. The `ck-subc` binary accepts
92/// `SUBC_CGROUP_PLACEMENT=disabled` for isolated test processes; an unset
93/// variable retains `Current`, and every other value is rejected at startup.
94/// Tests that exercise placement can own a [`Self::Root`].
95#[derive(Debug, Clone, Default, PartialEq, Eq)]
96pub enum CgroupPlacementConfig {
97    #[default]
98    Disabled,
99    Current,
100    Root(PathBuf),
101}
102
103#[derive(Debug, Clone)]
104pub struct BootstrapConfig {
105    #[cfg(feature = "test-support")]
106    shutdown_intent_window: Duration,
107    pub operator_queue_wait: Duration,
108    pub operator_timeout: Duration,
109    pub operator_stuck_grace: Duration,
110    #[cfg(feature = "test-support")]
111    pub operator_script: Option<PathBuf>,
112    #[cfg(feature = "test-support")]
113    pub operator_events: Option<PathBuf>,
114    pub connection_file_path: PathBuf,
115    pub port: u16,
116    pub daemon_ver: String,
117    configured_modules: Vec<ConfiguredModule>,
118    storage_config: Option<daemon_config::StorageConfig>,
119    admission_facts: AdmissionFactsConfig,
120    /// Module ids whose scopes may carry the attributes that grant authority.
121    scope_authority_owners: Vec<String>,
122    federated_scope_owner: String,
123    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
124    activity_config: daemon_config::ActivityConfig,
125    daemon_config_path: Option<PathBuf>,
126    configured_port: Option<u16>,
127    /// Daemon-wide route.bind relay budget in milliseconds (the fallback for
128    /// any module without a per-module override). `None` = built-in default
129    /// (12s — see `control::DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`).
130    route_bind_relay_default_ms: Option<u64>,
131    reserved_capabilities: BTreeMap<String, String>,
132    watchdog_config: DaemonSelfWatchdogConfig,
133    connection_file_source: ConnectionFileSource,
134    #[cfg(all(test, windows))]
135    profile_override: Option<PathBuf>,
136    /// Where module cgroups are prepared. Disabled unless a caller explicitly
137    /// opts in, because `Current` derives a host location from `/proc/self/cgroup`.
138    cgroup_placement: CgroupPlacementConfig,
139    /// Directory the supervisor writes per-module stdout/stderr capture files
140    /// into. `None` disables capture; the shipped binary supplies its real run
141    /// directory explicitly.
142    capture_logs_dir: Option<PathBuf>,
143    /// File the supervisor appends every module's terminal exits to, so exit
144    /// history survives a daemon restart. `None` keeps terminal history in
145    /// memory only (each module's ring). Absent by default for the same reason
146    /// as `capture_logs_dir`: an in-process daemon booted by a test must not
147    /// append its exits to the operator's real `terminals.jsonl`, where they
148    /// would show up in `ck module terminals`. The shipped binary supplies
149    /// `<run dir>/terminals.jsonl` explicitly.
150    terminal_journal_path: Option<PathBuf>,
151    /// Where the machine id is read from, or minted into when absent. `None`
152    /// serves no machine id. Absent by default for the same reason as
153    /// `capture_logs_dir`: an in-process daemon booted by a test must never
154    /// derive the operator's real data home and mint into it. The shipped binary
155    /// supplies `<data home>/cortexkit/machine-id` explicitly.
156    machine_id_path: Option<PathBuf>,
157
158    /// Where this daemon's same-user local listener lives (a Unix-domain socket
159    /// on macOS and Linux; nothing on Windows, where the pipe name is minted per
160    /// start). The default serves no local endpoint, so an in-process daemon
161    /// never touches the operator's run directory. The shipped binary names its
162    /// run directory; [`BootstrapConfig::with_local_listener_socket_dir`] names
163    /// any other, and an in-process daemon that wants the endpoint names one.
164    local_listener: LocalListenerConfig,
165    /// The live-children record: every supervised process this daemon has
166    /// running, kept so the next daemon can end the ones a crash left behind.
167    /// At startup, before any module is spawned, the previous daemon's record
168    /// here is swept. `None` keeps no record and sweeps nothing, for the same
169    /// reason as `capture_logs_dir`: an in-process daemon booted by a test
170    /// must never signal processes listed in the operator's real run
171    /// directory. The shipped binary supplies `<run dir>/live-children.json`.
172    live_children_path: Option<PathBuf>,
173    privacy_trampoline: Option<PathBuf>,
174}
175
176impl BootstrapConfig {
177    /// Shorten restart intent validity in daemon integration fixtures.
178    #[cfg(feature = "test-support")]
179    pub fn with_shutdown_intent_window(mut self, window: Duration) -> Self {
180        self.shutdown_intent_window = window;
181        self
182    }
183
184    pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
185        Self {
186            #[cfg(feature = "test-support")]
187            shutdown_intent_window: Duration::from_secs(120),
188            connection_file_path: connection_file_path.into(),
189            operator_queue_wait: Duration::from_secs(150),
190            operator_timeout: Duration::from_secs(120),
191            operator_stuck_grace: Duration::from_secs(10),
192            #[cfg(feature = "test-support")]
193            operator_script: None,
194            #[cfg(feature = "test-support")]
195            operator_events: None,
196            port,
197            daemon_ver: DAEMON_VERSION.to_owned(),
198            configured_modules: Vec::new(),
199            storage_config: None,
200            admission_facts: AdmissionFactsConfig::default(),
201            scope_authority_owners: daemon_config::default_scope_authority_owners(),
202            federated_scope_owner: daemon_config::default_federated_scope_owner(),
203            privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
204            activity_config: daemon_config::ActivityConfig::default(),
205            daemon_config_path: None,
206            configured_port: None,
207            route_bind_relay_default_ms: None,
208            reserved_capabilities: BTreeMap::new(),
209            watchdog_config: DaemonSelfWatchdogConfig::default(),
210            connection_file_source: ConnectionFileSource::Explicit,
211            #[cfg(all(test, windows))]
212            profile_override: None,
213            cgroup_placement: CgroupPlacementConfig::default(),
214            capture_logs_dir: None,
215            terminal_journal_path: None,
216            machine_id_path: None,
217            local_listener: LocalListenerConfig { socket_dir: None },
218            live_children_path: None,
219            privacy_trampoline: None,
220        }
221    }
222
223    /// Keep the local listener's socket in `dir`. Embedding daemons and tests
224    /// pass a path inside their own fixture tree.
225    pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
226        self.local_listener = LocalListenerConfig::in_socket_dir(dir);
227        self
228    }
229
230    /// Set where the local listener lives, as [`LocalListenerConfig`] describes.
231    pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
232        self.local_listener = config;
233        self
234    }
235
236    /// Keep the live-children record at `path`, and at startup end the
237    /// processes a previous daemon recorded there that are still running.
238    /// Embedding daemons and tests pass a path inside their own fixture tree.
239    pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
240        self.live_children_path = Some(path.into());
241        self
242    }
243
244    /// macOS module launches require an explicit executable implementing the
245    /// subc-os trampoline entry point. Other systems ignore this setting.
246    pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
247        self.privacy_trampoline = Some(path.into());
248        self
249    }
250
251    /// Serve the machine id stored at `path`, minting it there at startup when
252    /// the file is absent. Embedding daemons and tests pass a path inside their
253    /// own fixture tree.
254    pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
255        self.machine_id_path = Some(path.into());
256        self
257    }
258
259    /// Selects module cgroup placement. The default is disabled.
260    pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
261        self.cgroup_placement = placement;
262        self
263    }
264
265    /// Redirects per-module stdout/stderr capture files out of the real run
266    /// directory. Tests that start an in-process daemon must call this with a
267    /// path inside their fixture tree.
268    pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
269        self.capture_logs_dir = Some(dir.into());
270        self
271    }
272
273    /// Redirects the daemon-private journal, allowing embedded daemons and tests
274    /// to keep their observations out of the operator's live run directory.
275    pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
276        self.terminal_journal_path = Some(path.into());
277        self
278    }
279
280    pub fn from_env() -> Result<Self, BootstrapError> {
281        Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
282    }
283
284    /// The SHIPPED BINARY's config, which is the only caller that should capture
285    /// child output into the operator's real run directory.
286    ///
287    /// Kept separate from `from_env` deliberately: see `capture_logs_dir` and
288    /// `terminal_journal_path` on this struct for why an absent value must mean
289    /// NO CAPTURE and NO JOURNAL rather than the operator's real run directory.
290    /// A run directory that cannot be resolved (a relative data home) refuses
291    /// startup instead of landing under the working directory.
292    pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
293        let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
294        let machine_id_path =
295            crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
296        let config = Self::from_env()?
297            .with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
298            .with_capture_logs_dir(run_dir.join("logs"))
299            .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
300            .with_live_children_record(crate::live_children::record_path(&run_dir))
301            .with_machine_id_path(machine_id_path);
302        #[cfg(feature = "test-support")]
303        let config = {
304            let mut config = config;
305            config.operator_script =
306                std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
307            config.operator_events =
308                std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
309            for (name, value) in [
310                (
311                    "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
312                    &mut config.operator_queue_wait,
313                ),
314                (
315                    "SUBC_TEST_OPERATOR_TIMEOUT_MS",
316                    &mut config.operator_timeout,
317                ),
318                (
319                    "SUBC_TEST_OPERATOR_STUCK_MS",
320                    &mut config.operator_stuck_grace,
321                ),
322            ] {
323                if let Ok(ms) = std::env::var(name) {
324                    if let Ok(ms) = ms.parse::<u64>() {
325                        *value = Duration::from_millis(ms);
326                    }
327                }
328            }
329            config
330        };
331        Ok(config)
332    }
333
334    pub fn from_env_with_daemon_config_path(
335        daemon_config_path: impl AsRef<Path>,
336    ) -> Result<Self, BootstrapError> {
337        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
338        let (daemon_config, activity_config) =
339            daemon_config::load_with_activity(&daemon_config_path)
340                .map_err(BootstrapError::DaemonConfig)?
341                .map(|(config, activity)| (Some(config), activity))
342                .unwrap_or_default();
343        let config_port = daemon_config.as_ref().and_then(|config| config.port);
344        let storage_config = daemon_config
345            .as_ref()
346            .and_then(|config| config.storage.clone());
347        let admission_facts_carrier_module_id = daemon_config
348            .as_ref()
349            .and_then(|config| config.admission_facts_carrier_module_id.clone());
350        let admission_facts_targets = daemon_config
351            .as_ref()
352            .and_then(|config| config.admission_facts_targets.clone());
353        let scope_authority_owners = daemon_config
354            .as_ref()
355            .map(|config| config.scope_authority_owners.clone())
356            .unwrap_or_else(daemon_config::default_scope_authority_owners);
357        let federated_scope_owner = daemon_config
358            .as_ref()
359            .map(|config| config.federated_scope_owner.clone())
360            .unwrap_or_else(daemon_config::default_federated_scope_owner);
361        let privileged_operator_prompt = daemon_config
362            .as_ref()
363            .map(|config| config.privileged_operator_prompt)
364            .unwrap_or_default();
365        let route_bind_relay_default_ms = daemon_config
366            .as_ref()
367            .and_then(|config| config.route_bind_relay_timeout_ms);
368        let reserved_capabilities = daemon_config
369            .as_ref()
370            .map(|config| config.reserved_capabilities.clone())
371            .unwrap_or_default();
372        let configured_modules = daemon_config
373            .map(|config| config.modules)
374            .unwrap_or_default();
375
376        let port = match env::var(SUBC_PORT_ENV) {
377            Ok(raw) if !raw.trim().is_empty() => {
378                let port = raw
379                    .parse::<u16>()
380                    .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
381                if let Some(config_port) = config_port {
382                    info!(
383                        env = SUBC_PORT_ENV,
384                        env_port = port,
385                        config_port,
386                        "SUBC_PORT overrides daemon config port"
387                    );
388                }
389                port
390            }
391            Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
392        };
393
394        let (connection_file_path, connection_file_source) =
395            connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
396        Ok(Self::new(connection_file_path, port)
397            .with_configured_modules(configured_modules)
398            .with_storage_config(storage_config)
399            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
400            .with_scope_authority_owners(scope_authority_owners)
401            .with_federated_scope_owner(federated_scope_owner)
402            .with_privileged_operator_prompt(privileged_operator_prompt)
403            .with_activity_config(activity_config)
404            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
405            .with_reserved_capabilities(reserved_capabilities)
406            .with_daemon_config_source(daemon_config_path, config_port)
407            .with_connection_file_source(connection_file_source))
408    }
409
410    pub fn with_daemon_config_path(
411        self,
412        daemon_config_path: impl AsRef<Path>,
413    ) -> Result<Self, BootstrapError> {
414        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
415        let (daemon_config, activity_config) =
416            daemon_config::load_with_activity(&daemon_config_path)
417                .map_err(BootstrapError::DaemonConfig)?
418                .map(|(config, activity)| (Some(config), activity))
419                .unwrap_or_default();
420        let configured_port = daemon_config.as_ref().and_then(|config| config.port);
421        let storage_config = daemon_config
422            .as_ref()
423            .and_then(|config| config.storage.clone());
424        let admission_facts_carrier_module_id = daemon_config
425            .as_ref()
426            .and_then(|config| config.admission_facts_carrier_module_id.clone());
427        let admission_facts_targets = daemon_config
428            .as_ref()
429            .and_then(|config| config.admission_facts_targets.clone());
430        let scope_authority_owners = daemon_config
431            .as_ref()
432            .map(|config| config.scope_authority_owners.clone())
433            .unwrap_or_else(daemon_config::default_scope_authority_owners);
434        let federated_scope_owner = daemon_config
435            .as_ref()
436            .map(|config| config.federated_scope_owner.clone())
437            .unwrap_or_else(daemon_config::default_federated_scope_owner);
438        let privileged_operator_prompt = daemon_config
439            .as_ref()
440            .map(|config| config.privileged_operator_prompt)
441            .unwrap_or_default();
442        let route_bind_relay_default_ms = daemon_config
443            .as_ref()
444            .and_then(|config| config.route_bind_relay_timeout_ms);
445        let reserved_capabilities = daemon_config
446            .as_ref()
447            .map(|config| config.reserved_capabilities.clone())
448            .unwrap_or_default();
449        let configured_modules = daemon_config
450            .map(|config| config.modules)
451            .unwrap_or_default();
452        Ok(self
453            .with_configured_modules(configured_modules)
454            .with_storage_config(storage_config)
455            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
456            .with_scope_authority_owners(scope_authority_owners)
457            .with_federated_scope_owner(federated_scope_owner)
458            .with_privileged_operator_prompt(privileged_operator_prompt)
459            .with_activity_config(activity_config)
460            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
461            .with_reserved_capabilities(reserved_capabilities)
462            .with_daemon_config_source(daemon_config_path, configured_port))
463    }
464
465    pub fn with_configured_modules(
466        mut self,
467        modules: impl IntoIterator<Item = ConfiguredModule>,
468    ) -> Self {
469        self.configured_modules = modules.into_iter().collect();
470        self.configured_modules
471            .sort_by(|left, right| left.module_id.cmp(&right.module_id));
472        self
473    }
474
475    pub fn with_storage_config(
476        mut self,
477        storage_config: Option<daemon_config::StorageConfig>,
478    ) -> Self {
479        self.storage_config = storage_config;
480        self
481    }
482
483    pub fn with_admission_facts_config(
484        mut self,
485        carrier_module_id: Option<String>,
486        targets: Option<Vec<String>>,
487    ) -> Self {
488        self.admission_facts = AdmissionFactsConfig {
489            carrier_module_id,
490            targets,
491        };
492        self
493    }
494
495    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
496    /// Absent from a config, this is `daemon_config::default_scope_authority_owners`.
497    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
498        self.scope_authority_owners = owners;
499        self
500    }
501
502    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
503        self.federated_scope_owner = owner;
504        self
505    }
506
507    /// Choose the privileged-operation policy for the daemon's entire lifetime.
508    pub fn with_privileged_operator_prompt(
509        mut self,
510        mode: daemon_config::PrivilegedOperatorPrompt,
511    ) -> Self {
512        self.privileged_operator_prompt = mode;
513        self
514    }
515
516    pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
517        self.activity_config = config;
518        self
519    }
520
521    /// Set the daemon-wide route.bind relay default (the fallback for any
522    /// module without a per-module override). `None` preserves the built-in
523    /// default (12s). `serve_bound_daemon` reads this at startup and threads
524    /// it into the control handler's daemon-wide field.
525    pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
526        self.route_bind_relay_default_ms = ms;
527        self
528    }
529
530    pub fn with_reserved_capabilities(
531        mut self,
532        reserved_capabilities: BTreeMap<String, String>,
533    ) -> Self {
534        self.reserved_capabilities = reserved_capabilities;
535        self
536    }
537
538    fn with_daemon_config_source(
539        mut self,
540        daemon_config_path: PathBuf,
541        configured_port: Option<u16>,
542    ) -> Self {
543        self.daemon_config_path = Some(daemon_config_path);
544        self.configured_port = configured_port;
545        self
546    }
547
548    fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
549        self.connection_file_source = source;
550        self
551    }
552
553    pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
554        self.watchdog_config = watchdog_config;
555        self
556    }
557}
558
559/// Result of singleton discovery.
560// Built once per daemon start and immediately matched, so the size gap between
561// the variants costs nothing worth a box on the public shape.
562#[allow(clippy::large_enum_variant)]
563#[derive(Debug)]
564pub enum Outcome {
565    /// A live daemon authenticated from the connection file; this invocation should exit 0.
566    AlreadyRunning,
567    /// This process won the singleton race, owns bound loopback listener(s), and
568    /// has published a fresh connection file.
569    Bound(BoundDaemon),
570}
571
572#[derive(Debug)]
573pub struct BoundDaemon {
574    pub listeners: Vec<TcpListener>,
575    /// The same-user local listener, bound and not yet served. `None` when
576    /// binding it failed, in which case `local_endpoint` is `None` and
577    /// `local_endpoint_unavailable` says why.
578    pub local_listener: Option<BoundLocalListener>,
579    /// Why the daemon serves no local endpoint, when binding it failed. Reported
580    /// by `server.describe` so an operator can see it.
581    pub local_endpoint_unavailable: Option<String>,
582    /// What the connection file publishes under `local_endpoint`, so a client can
583    /// find the local listener without knowing how it was bound.
584    pub local_endpoint: Option<LocalEndpoint>,
585    pub connection_info: ConnectionInfo,
586    pub connection_file_path: PathBuf,
587    pub connection_file_source: ConnectionFileSource,
588    #[cfg(windows)]
589    connection_file_copy: Option<PathBuf>,
590    /// The machine id this daemon serves, established before any connection is
591    /// accepted. `None` when the config named no machine id path.
592    pub machine_id: Option<crate::machine_id::MachineId>,
593    /// Ownership of the run directory, held until the daemon exits so no other
594    /// daemon sweeps or writes this one's run state. `None` when the config
595    /// keeps no live-children record, and so has no run directory to own.
596    run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
597    /// The singleton start lock, kept when binding deferred publishing the
598    /// connection file. It stays held until the daemon has verified it is serving
599    /// and written the connection file, so a second daemon starting meanwhile
600    /// cannot claim the singleton too. `None` when the connection
601    /// file was already written at bind time.
602    publication_lock: Option<StartLock>,
603}
604
605/// Resolve subc's per-user TCP connection-file path.
606///
607/// `$XDG_RUNTIME_DIR/subc-connection.json` is preferred because the runtime
608/// directory is already per-user on Unix desktops. Without it, subc falls back
609/// to the system temp dir with a per-user token in the filename so different OS
610/// users do not collide on shared temp directories.
611pub fn connection_file_path() -> PathBuf {
612    connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
613}
614
615fn connection_file_path_with_source(
616    runtime_dir: Option<OsString>,
617) -> (PathBuf, ConnectionFileSource) {
618    if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
619        return (
620            PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
621            ConnectionFileSource::XdgRuntimeDir,
622        );
623    }
624
625    (
626        env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
627        ConnectionFileSource::TempDirFallback,
628    )
629}
630
631/// Resolve, claim, and serve the per-user daemon singleton.
632///
633/// A second invocation is successful: if a live daemon authenticates from the
634/// existing connection file, this returns `Ok(())` after logging and the caller
635/// exits with status 0.
636pub async fn run() -> Result<(), BootstrapError> {
637    // `run` is a binary entry point, so it opts into the production cgroup and
638    // child-log locations. In-process callers keep both features disabled by default.
639    run_with_config(
640        BootstrapConfig::from_env_for_daemon_binary()?
641            .with_cgroup_placement(CgroupPlacementConfig::Current),
642    )
643    .await
644}
645
646/// Serve a daemon from an explicit config. This is the entry point the twelve
647/// sibling repos use to boot an in-process daemon in their integration tests.
648///
649/// # THIS INSTALLS NO TRACING SUBSCRIBER, SO THE DAEMON IS SILENT BY DEFAULT
650///
651/// The daemon's own diagnostics go through `tracing`, and `tracing` DISCARDS
652/// every event when no subscriber is installed. The shipped binary installs one
653/// in `main` (`init_tracing`); this function deliberately does not, because a
654/// library that installs a global subscriber fights with whatever the host
655/// process already set up.
656///
657/// The consequence for a test harness is not "less verbose": it is that the
658/// daemon has NOTHING TO SAY about any failure, and a missing instrument reads
659/// exactly like a clean one. PLEX found this on 2026-09-18 while trying to
660/// capture daemon logs beside an intermittent bind failure, and discovered the
661/// daemon had been silent in every conformance run that repo had ever done --
662/// so the one client-side error string was all the evidence that could exist,
663/// and they had spent a real investigation on a failure whose second source was
664/// never being recorded.
665///
666/// Install one in the harness before calling this, and assert it did something
667/// (they measured 236 daemon lines with the subscriber installed, 0 with the
668/// call commented out) -- otherwise the fix is itself unverified.
669pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
670    #[cfg(feature = "test-support")]
671    let shutdown_intent_window = config.shutdown_intent_window;
672    let forwarding = Arc::new(ForwardingTable::default());
673    forwarding.operator_confirms().configure_bootstrap(&config);
674    let configured_modules = config.configured_modules.clone();
675    let storage_config = config.storage_config.clone();
676    let admission_facts = config.admission_facts.clone();
677    let scope_authority_owners = config.scope_authority_owners.clone();
678    let federated_scope_owner = config.federated_scope_owner.clone();
679    let privileged_operator_prompt = config.privileged_operator_prompt;
680    let activity_config = config.activity_config;
681    let daemon_config_path = config.daemon_config_path.clone();
682    let configured_port = config.configured_port;
683    let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
684    let reserved_capabilities = config.reserved_capabilities.clone();
685    let watchdog_config = config.watchdog_config.clone();
686    let cgroup_placement_config = config.cgroup_placement.clone();
687    let capture_logs_dir = config.capture_logs_dir.clone();
688    let terminal_journal_path = config.terminal_journal_path.clone();
689    let live_children_path = config.live_children_path.clone();
690    let privacy_trampoline = config.privacy_trampoline.clone();
691    match ensure_singleton_inner(config, false).await? {
692        Outcome::AlreadyRunning => {
693            info!("subc daemon already running");
694            Ok(())
695        }
696        Outcome::Bound(bound) => {
697            #[cfg(target_os = "linux")]
698            let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
699            #[cfg(not(target_os = "linux"))]
700            let _ = cgroup_placement_config;
701            serve_bound_daemon(
702                bound,
703                forwarding,
704                configured_modules,
705                storage_config,
706                admission_facts,
707                scope_authority_owners,
708                federated_scope_owner,
709                privileged_operator_prompt,
710                activity_config,
711                daemon_config_path,
712                configured_port,
713                route_bind_relay_default_ms,
714                reserved_capabilities,
715                watchdog_config,
716                capture_logs_dir,
717                terminal_journal_path,
718                live_children_path,
719                privacy_trampoline,
720                #[cfg(feature = "test-support")]
721                shutdown_intent_window,
722                #[cfg(target_os = "linux")]
723                cgroup_placement,
724            )
725            .await
726        }
727    }
728}
729
730#[cfg(target_os = "linux")]
731fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
732    let result = match config {
733        CgroupPlacementConfig::Disabled => return None,
734        CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
735        CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
736    };
737
738    match result {
739        Ok(Some(placement)) => Some(placement),
740        Ok(None) => {
741            warn!(
742                placement = ?config,
743                "module cgroup placement is disabled: configured cgroup root is not delegated"
744            );
745            None
746        }
747        Err(error) => {
748            warn!(
749                placement = ?config,
750                error = %error,
751                "module cgroup placement is disabled by an unexpected cgroup probe error"
752            );
753            None
754        }
755    }
756}
757
758/// Target soft limit for open file descriptors, applied to the daemon before any
759/// module is spawned so children inherit it. Multi-root modules (one process
760/// aggregating every project root's sqlite stores, index caches, watchers, and
761/// LSP pipes) trivially exceed the macOS default soft limit of 256; a launchd
762/// user agent does not pass login-shell ulimits through, so the raise must
763/// happen in-process.
764#[cfg(unix)]
765const NOFILE_TARGET: u64 = 65536;
766
767/// Raise RLIMIT_NOFILE to `NOFILE_TARGET` (clamped to the hard limit).
768/// Best-effort: failure is logged and never fatal, since the daemon can run
769/// under the inherited limit — modules with few roots just have less headroom.
770#[cfg(unix)]
771fn raise_nofile_limit() {
772    #[cfg(target_os = "macos")]
773    let ceiling = std::process::Command::new("/usr/sbin/sysctl")
774        .args(["-n", "kern.maxfilesperproc"])
775        .output()
776        .ok()
777        .filter(|output| output.status.success())
778        .and_then(|output| String::from_utf8(output.stdout).ok())
779        .and_then(|value| value.trim().parse::<u64>().ok())
780        .filter(|value| *value > 0);
781    #[cfg(not(target_os = "macos"))]
782    let ceiling = None;
783    raise_nofile_limit_with_ceiling(ceiling);
784}
785
786#[cfg(unix)]
787fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
788    match rlimit::Resource::NOFILE.get() {
789        Ok((soft, hard)) => {
790            // On XNU an infinite hard limit does not remove the separate
791            // per-process kernel ceiling. Exceeding it makes setrlimit fail
792            // without raising the inherited (often 256) soft limit at all.
793            let target = NOFILE_TARGET
794                .min(hard)
795                .min(kernel_ceiling.unwrap_or(u64::MAX));
796            if soft >= target {
797                return;
798            }
799            match rlimit::Resource::NOFILE.set(target, hard) {
800                Ok(()) => info!(
801                    previous_soft = soft,
802                    new_soft = target,
803                    hard,
804                    "raised open-file soft limit for daemon and module children"
805                ),
806                Err(err) => warn!(
807                    soft,
808                    hard,
809                    error = %err,
810                    "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
811                ),
812            }
813        }
814        Err(err) => warn!(error = %err, "could not read open-file limit"),
815    }
816}
817
818/// CRT stdio-stream target on Windows (the `_setmaxstdio` maximum). Win32
819/// HANDLEs — what Rust `File`, tokio sockets, and SQLite's Win32 VFS actually
820/// consume — have a per-process quota in the millions and need no raise; the
821/// C-runtime stream table (default 512) is the only low ceiling, and it is
822/// per-process rather than inherited, so supervised modules linking the CRT
823/// must raise their own. Raising it here covers the daemon itself.
824#[cfg(windows)]
825fn raise_nofile_limit() {
826    const MAXSTDIO_TARGET: u32 = 8192;
827    let current = rlimit::getmaxstdio();
828    if current >= MAXSTDIO_TARGET {
829        return;
830    }
831    match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
832        Ok(new_max) => info!(
833            previous = current,
834            new_max, "raised CRT stdio-stream limit for daemon"
835        ),
836        Err(err) => warn!(
837            current,
838            error = %err,
839            "could not raise CRT stdio-stream limit"
840        ),
841    }
842}
843
844#[cfg(not(any(unix, windows)))]
845fn raise_nofile_limit() {}
846
847pub async fn run_with_daemon_config_path(
848    config: BootstrapConfig,
849    daemon_config_path: impl AsRef<Path>,
850) -> Result<(), BootstrapError> {
851    run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
852}
853
854#[allow(clippy::too_many_arguments)]
855async fn serve_bound_daemon(
856    bound: BoundDaemon,
857    forwarding: Arc<ForwardingTable>,
858    configured_modules: Vec<ConfiguredModule>,
859    storage_config: Option<daemon_config::StorageConfig>,
860    admission_facts: AdmissionFactsConfig,
861    scope_authority_owners: Vec<String>,
862    federated_scope_owner: String,
863    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
864    activity_config: daemon_config::ActivityConfig,
865    daemon_config_path: Option<PathBuf>,
866    configured_port: Option<u16>,
867    route_bind_relay_default_ms: Option<u64>,
868    reserved_capabilities: BTreeMap<String, String>,
869    watchdog_config: DaemonSelfWatchdogConfig,
870    capture_logs_dir: Option<PathBuf>,
871    terminal_journal_path: Option<PathBuf>,
872    live_children_path: Option<PathBuf>,
873    privacy_trampoline: Option<PathBuf>,
874    #[cfg(feature = "test-support")] shutdown_intent_window: Duration,
875    #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
876) -> Result<(), BootstrapError> {
877    #[cfg(unix)]
878    let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
879        .map_err(BootstrapError::Signal)?;
880    // Windows has no SIGTERM. Ctrl-C is a different event, not an equivalent
881    // service-stop contract, so this shutdown handler is intentionally Unix-only.
882    raise_nofile_limit();
883    #[cfg(windows)]
884    if let Err(error) = subc_jobobject::ensure_root_job() {
885        // Keep the control plane available. The root-job API caches setup
886        // failures, so later module spawns refuse instead of running unprotected.
887        warn!(%error, "module launches disabled: Windows root job setup failed");
888    }
889
890    info!(
891        connection_file = %bound.connection_file_path.display(),
892        connection_file_source = %bound.connection_file_source,
893        connection_file_source_reason = bound.connection_file_source.reason(),
894        endpoints = ?bound.connection_info.endpoints,
895        configured_modules = configured_modules.len(),
896        machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
897        "subc daemon starting"
898    );
899
900    // Before anything can spawn a module (the configured modules below, or a
901    // client's start request once the listeners are served): a previous daemon
902    // that died without its shutdown stop may have left children running, and
903    // a fresh copy beside one would fight it for its port and stores. The
904    // record is not a running daemon's because this daemon holds the lock on
905    // the run directory the record lives in, and a running daemon holds that
906    // lock for its whole life. (Claiming the singleton is not enough: it is
907    // keyed on the connection file, which can live in a different runtime
908    // directory from the run directory.) The lock stays held until this
909    // function returns, which is when the daemon exits.
910    let run_dir_lock = bound.run_dir_lock;
911    let publication_lock = bound.publication_lock;
912    if let Some(owner) = &run_dir_lock {
913        #[cfg(target_os = "linux")]
914        crate::live_children::sweep_orphans_with_placement(
915            owner,
916            &crate::live_children::AdoptedPids::none(),
917            crate::live_children::SweepBounds::default(),
918            cgroup_placement.as_ref(),
919        )
920        .await;
921        #[cfg(not(target_os = "linux"))]
922        crate::live_children::sweep_orphans(
923            owner,
924            &crate::live_children::AdoptedPids::none(),
925            crate::live_children::SweepBounds::default(),
926        )
927        .await;
928    }
929
930    let registry = Arc::new(Registry::default());
931    let process_liveness = Arc::new(SupervisorProcessLiveness::new());
932    let supervisor_handle = SupervisorHandle::new();
933    let _host_health_task = AbortOnDrop::new(supervisor_handle.spawn_host_health_monitor());
934    #[cfg(feature = "test-support")]
935    supervisor_handle.set_shutdown_intent_window(shutdown_intent_window);
936    let connected_clients = ConnectedClients::new();
937    let daemon_incarnation = format!(
938        "{:032x}",
939        u128::from_be_bytes(bound.connection_info.daemon_id)
940    );
941    let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
942        .with_process_liveness(process_liveness.clone())
943        .with_forwarding(Arc::clone(&forwarding))
944        .with_handle(supervisor_handle.clone())
945        .with_connection_file_path(bound.connection_file_path.clone())
946        .with_daemon_incarnation(daemon_incarnation.clone());
947    let supervisor = match privacy_trampoline {
948        Some(path) => supervisor.with_privacy_trampoline(path),
949        None => supervisor,
950    };
951    // ABSENT MEANS NO CAPTURE AND NO JOURNAL, NOT "THE REAL RUN DIRECTORY", and
952    // the difference is a production-corruption hazard rather than a preference.
953    // Both fields below follow the same rule: `None` for `capture_logs_dir`
954    // means supervised output is not captured, and `None` for
955    // `terminal_journal_path` means terminal history lives only in each
956    // module's in-memory ring (`supervisor.terminals` still answers, with the
957    // journal counters at zero).
958    //
959    // The terminal journal used to fall back to
960    // `daemon_run_dir().join("terminals.jsonl")`, so an in-process test daemon
961    // appended its fixture exits to the operator's journal, where they then
962    // appeared in `ck module terminals`.
963    //
964    // The capture line used to be `unwrap_or_else(|| daemon_run_dir().join("logs"))`,
965    // so ANY caller that did not set the field captured supervised children into
966    // the operator's live `~/.local/share/cortexkit/run/logs/`. That is twelve
967    // sibling repos whose integration tests boot an in-process daemon through
968    // `run_with_config` -- none of which asked for it, and none of which can see
969    // it from their side.
970    //
971    // Harmless while fixture module ids are fixture-shaped: this host carries 20
972    // zero-byte files from subc's own tests (good-aft, missing-aft,
973    // preview-consumer...). THE HAZARD IS A COLLISION. A fixture named "broca"
974    // or "aft" appends to a PRODUCTION capture file that operators read
975    // forensically and that placement gates count lines in -- with no residue to
976    // notice, because the file legitimately exists and legitimately grows.
977    //
978    // Found by BROCA (2026-09-19) from the other side: their rigs spawn the
979    // SHIPPED ck-subc and set XDG_CONFIG_HOME + XDG_RUNTIME_DIR but not
980    // XDG_DATA_HOME, so every local rig run supervised a module named "broca"
981    // and captured it into production's broca.stderr.log -- the same file I
982    // count seal lines in before and after placing their binaries.
983    //
984    // The supervisor already treats `None` as no-capture and no-journal, so
985    // this only removes invented defaults. The binary keeps capturing and
986    // journaling via `BootstrapConfig::from_env_for_daemon_binary`.
987
988    // Keep both ledgers in the explicitly selected run directory. An embedded
989    // daemon without a journal path must not write to the operator's live ledger.
990    let activity_run_dir = terminal_journal_path
991        .as_ref()
992        .and_then(|path| path.parent())
993        .map(Path::to_path_buf);
994    let supervisor = match terminal_journal_path {
995        Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
996        None => supervisor,
997    };
998    let supervisor = match capture_logs_dir {
999        Some(dir) => supervisor.with_capture_logs_dir(dir),
1000        None => supervisor,
1001    };
1002    let supervisor = match live_children_path {
1003        Some(path) => supervisor.with_live_children_record(path),
1004        None => supervisor,
1005    };
1006    #[cfg(target_os = "linux")]
1007    let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
1008    // Collect per-module route.bind relay overrides BEFORE handing the
1009    // `configured_modules` vector to the supervisor (which only needs each
1010    // module's `drain_timeout_ms`). Each entry was filled in by parse-time
1011    // resolution (per-module > daemon-wide > absent), so modules with no
1012    // override are absent from this map and the daemon-wide default applies.
1013    let route_bind_relay_timeouts = configured_modules
1014        .iter()
1015        .filter_map(|module| {
1016            module
1017                .route_bind_relay_timeout_ms
1018                .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
1019        })
1020        .collect::<std::collections::BTreeMap<_, _>>();
1021    let control_start_clock = crate::clock::StartClock::capture();
1022    let mut control = ControlHandler::with_forwarding(Arc::clone(&registry), forwarding)
1023        .with_daemon_version(bound.connection_info.daemon_ver.clone())
1024        .with_process_liveness(process_liveness)
1025        .with_supervisor(supervisor_handle)
1026        // The same flag `begin_daemon_shutdown` sets below, so requirement
1027        // changes while modules exit during the shutdown log at INFO.
1028        .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
1029        .with_connected_clients(connected_clients.clone())
1030        .with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
1031        .with_storage_config(storage_config)
1032        .with_machine_id(bound.machine_id.clone())
1033        .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
1034        .with_scope_authority_owners(scope_authority_owners)
1035        .with_federated_scope_owner(federated_scope_owner)
1036        .with_privileged_operator_prompt(privileged_operator_prompt)
1037        .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
1038        .with_daemon_provenance(
1039            bound.connection_info.pid,
1040            control_start_clock.started_at_ms(),
1041            std::env::current_exe().ok(),
1042            normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
1043            normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
1044        )
1045        .with_daemon_start_clock(control_start_clock)
1046        .with_capability_config(
1047            configured_modules
1048                .iter()
1049                .map(|module| (module.module_id.clone(), module.enabled)),
1050            reserved_capabilities,
1051        );
1052    if let Some(run_dir) = activity_run_dir {
1053        control =
1054            control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
1055    }
1056    if let Some(ms) = route_bind_relay_default_ms {
1057        // A daemon-wide config value overrides the built-in default; a
1058        // `None` here leaves the ControlHandler's 12s default in place.
1059        control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
1060    }
1061    if let Some(config_path) = daemon_config_path {
1062        control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
1063    }
1064    let control = Arc::new(control);
1065    let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
1066    let auth = ServerAuth::new(
1067        bound.connection_info.key.clone(),
1068        bound.connection_info.daemon_id,
1069        bound.connection_info.daemon_ver.clone(),
1070    )
1071    .with_connected_clients(connected_clients);
1072
1073    // Discovery must expose every accepted configured module, including disabled
1074    // ones. Registration creates status handles but neither starts monitors
1075    // nor launches processes: children need the published file for `--subc`.
1076    let mut registered_modules = Vec::with_capacity(configured_modules.len());
1077    for configured in configured_modules {
1078        let enabled = configured.enabled;
1079        let health = configured.health.clone();
1080        let module_id = configured.module_id.clone();
1081        let registered = match supervisor.register_configured_with_health(
1082            configured.module_spec(),
1083            enabled,
1084            health.clone(),
1085            configured.drain_timeout_ms,
1086            configured.restart,
1087            configured.register_timeout,
1088        ) {
1089            Ok(registered) => registered,
1090            Err(err) => {
1091                error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
1092                continue;
1093            }
1094        };
1095        registered_modules.push(registered);
1096        // A relaxed threshold can silently outlive the temporary allowance
1097        // that required it, so report the extra tolerance on every boot.
1098        let default_threshold = HealthConfig::default().failure_threshold;
1099        if enabled && health.failure_threshold > default_threshold {
1100            warn!(
1101                module_id = %module_id,
1102                failure_threshold = health.failure_threshold,
1103                default_threshold,
1104                tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1105                "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1106            );
1107        }
1108        info!(module_id = %module_id, enabled, "configured module registered");
1109    }
1110    control.refresh_capability_requirements();
1111
1112    // The local endpoint is served beside TCP and answers the same handshake.
1113    // It serves clients only, which the server enforces per connection.
1114    let local_serve_task = bound.local_listener.map(|local_listener| {
1115        AbortOnDrop::new(tokio::spawn(serve_local_listener(
1116            local_listener,
1117            Arc::clone(&router),
1118            auth.clone(),
1119        )))
1120    });
1121    // On Unix the task is dropped below to stop accepting at shutdown. Windows
1122    // leaves it to the process exit; naming it keeps the binding uniform.
1123    #[cfg(windows)]
1124    let _local_serve_task = local_serve_task;
1125    let mut serve_task =
1126        AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1127    verify_serving(&bound.connection_info).await?;
1128    // A published endpoint promises that authentication can complete now.
1129    // Orphan cleanup and control-plane construction can take longer than an
1130    // existing-daemon probe's deadline, so finish both before discovery.
1131    // The write fsyncs, so it runs on tokio's blocking pool (see
1132    // `publish_connection_file`). The singleton lock travels with it and comes
1133    // back only once the file is in place.
1134    let (published, publication_lock) = publish_connection_file(
1135        &bound.connection_file_path,
1136        #[cfg(windows)]
1137        bound.connection_file_copy.as_deref(),
1138        &bound.connection_info,
1139        bound.local_endpoint.as_ref(),
1140        publication_lock,
1141    )
1142    .await;
1143    published?;
1144    #[cfg(test)]
1145    startup_publication_hook::run(&bound.connection_file_path).await;
1146    drop(publication_lock);
1147    let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1148    // Off the startup path: it runs `systemctl`, and only ever logs.
1149    #[cfg(target_os = "linux")]
1150    let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1151        crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1152    ));
1153    let _watchdog_task = AbortOnDrop::new(
1154        DaemonSelfWatchdog::new(
1155            bound.connection_info.clone(),
1156            bound.connection_file_path.clone(),
1157        )
1158        .with_config(watchdog_config)
1159        .spawn(),
1160    );
1161
1162    for registered in registered_modules {
1163        match supervisor.start_registered_configured(registered, None, true) {
1164            Ok(module) => {
1165                info!(module_id = %module.module_id(), "configured module supervised");
1166            }
1167            Err(err) => {
1168                error!(error = %err, "failed to start configured module; continuing daemon startup");
1169            }
1170        }
1171    }
1172
1173    control.refresh_capability_requirements();
1174    Arc::clone(&control).spawn_capability_deadline_loop();
1175    Arc::clone(&control).spawn_scope_expiry_loop();
1176    control.spawn_resource_sampler();
1177
1178    #[cfg(unix)]
1179    {
1180        tokio::select! {
1181            result = serve_task.join() => {
1182                return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1183            }
1184            _ = terminate.recv() => {}
1185        }
1186        // First, before the notice, the drain, or any connection close: from
1187        // here on a module exit is recorded as `daemon_shutdown` and never
1188        // respawned. Also before allowing a second signal to cut the bounded
1189        // wait short, so the journal marker is always written.
1190        supervisor.begin_daemon_shutdown();
1191        // Stop the self-watchdog before closing the listener. Its next tick would
1192        // connect to that listener, fail, and log an ERROR indistinguishable from
1193        // a wedged daemon, once for every interval a planned stop lasts.
1194        drop(_watchdog_task);
1195        // Dropping the listener stops new accepts, not established connections:
1196        // their detached tasks must remain live throughout notice and drain. The
1197        // local endpoint stops accepting the same way.
1198        drop(serve_task);
1199        drop(local_serve_task);
1200        let second_sigterm = async {
1201            terminate.recv().await;
1202            std::time::Instant::now()
1203        };
1204        tokio::pin!(second_sigterm);
1205        let mut escalation_started = tokio::select! {
1206            biased;
1207            at = second_sigterm.as_mut() => {
1208                info!("second SIGTERM: abandoning daemon shutdown wait");
1209                Some(at)
1210            }
1211            result = supervisor.drain_for_daemon_shutdown() => {
1212                if let Err(error) = result {
1213                    warn!(%error, "daemon shutdown drain failed; exiting anyway");
1214                }
1215                None
1216            }
1217        };
1218        // Supervised modules lead their own process groups, so a service
1219        // manager's kill of this process's group does not reach them. (A
1220        // systemd unit with KillMode=control-group kills by cgroup instead and
1221        // does reach them; see `systemd_kill_mode`.) The daemon ends them
1222        // itself: EOF (or SIGTERM for a protocol none child) first, then
1223        // signals at each child's own drain deadline.
1224        let already_escalated = escalation_started.is_some();
1225        supervisor
1226            .end_children_for_daemon_shutdown(already_escalated, async {
1227                if already_escalated {
1228                    std::future::pending::<()>().await;
1229                } else {
1230                    escalation_started = Some(second_sigterm.await);
1231                }
1232            })
1233            .await;
1234        if let Some(at) = escalation_started {
1235            info!(
1236                second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1237                "daemon shutdown wait ended after second SIGTERM"
1238            );
1239        }
1240        info!("daemon exiting: SIGTERM shutdown complete");
1241        Ok(())
1242    }
1243    #[cfg(not(unix))]
1244    serve_task
1245        .join()
1246        .await
1247        .map_err(BootstrapError::ServeJoin)?
1248        .map_err(BootstrapError::Serve)
1249}
1250
1251fn normalized_build_provenance(value: &str) -> Option<String> {
1252    match value.trim() {
1253        "" | "unavailable" => None,
1254        value => Some(value.to_string()),
1255    }
1256}
1257
1258/// Find an existing daemon or atomically bind loopback TCP for this daemon.
1259///
1260/// The algorithm is intentionally connect-first: an endpoint from the connection
1261/// file is treated as live only after the TCP+key server-proof authenticates for
1262/// that file's key and daemon_id. Stale or foreign connection files are reclaimed
1263/// only while holding the per-user start lock; the TCP port is never the
1264/// singleton primitive.
1265pub async fn ensure_singleton(
1266    connection_file_path: impl AsRef<Path>,
1267    port: u16,
1268) -> Result<Outcome, BootstrapError> {
1269    ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1270}
1271
1272pub async fn ensure_singleton_with_config(
1273    config: BootstrapConfig,
1274) -> Result<Outcome, BootstrapError> {
1275    ensure_singleton_inner(config, true).await
1276}
1277
1278async fn ensure_singleton_inner(
1279    config: BootstrapConfig,
1280    publish: bool,
1281) -> Result<Outcome, BootstrapError> {
1282    #[cfg(windows)]
1283    let copy_path = windows_connection_copy::resolve(&config);
1284    let path = config.connection_file_path;
1285
1286    if stable_peer(&path).await? {
1287        return Ok(Outcome::AlreadyRunning);
1288    }
1289
1290    let lock = StartLock::acquire(&path).await?;
1291
1292    // Re-probe after acquiring the start lock so a peer that won the race between
1293    // our first failed probe and the lock acquisition is observed instead of
1294    // overwritten.
1295    if stable_peer(&path).await? {
1296        return Ok(Outcome::AlreadyRunning);
1297    }
1298
1299    // The start lock and the probe above only rule out a daemon using this
1300    // connection file. A daemon started with another runtime directory but
1301    // the same data home shares the run directory while passing both, so the
1302    // run directory needs its own owner: refuse at once if a live daemon holds
1303    // it, before anything below reads or writes run or data-home state.
1304    let run_dir_lock = config
1305        .live_children_path
1306        .as_deref()
1307        .map(crate::run_dir_lock::RunDirLock::acquire)
1308        .transpose()?;
1309
1310    remove_stale_connection_file_if_present(&path)?;
1311    #[cfg(windows)]
1312    let copy_path = windows_connection_copy::reclaim(&path, copy_path).await;
1313
1314    // Established under the start lock, and under the run-directory lock when
1315    // there is one, and before binding: a corrupt file stops boot before
1316    // anything is published, and two daemons racing to start cannot both mint.
1317    //
1318    // Minting a new machine id fsyncs its file, so loading or minting runs on
1319    // tokio's blocking pool. Both locks travel with that work, so neither is
1320    // released while a mint is in flight.
1321    let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1322        None => (None, lock, run_dir_lock),
1323        Some(machine_id_path) => {
1324            let task_path = machine_id_path.clone();
1325            match run_blocking_holding((lock, run_dir_lock), move || {
1326                crate::machine_id::load_or_mint(&task_path)
1327            })
1328            .await
1329            {
1330                Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1331                Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1332                Err(join_error) => {
1333                    return Err(BootstrapError::MachineId(
1334                        crate::machine_id::MachineIdFileError::Write {
1335                            path: machine_id_path,
1336                            source: io::Error::other(describe_blocking_task_failure(
1337                                "loading or minting the machine id",
1338                                &join_error,
1339                            )),
1340                        },
1341                    ))
1342                }
1343            }
1344        }
1345    };
1346
1347    let (listeners, endpoints) = bind_loopback(config.port).await?;
1348    // The local endpoint is an addition to the daemon, not a condition for it:
1349    // a bind failure leaves clients on TCP only, with the connection file naming
1350    // no local endpoint, and the reason reported by `server.describe`. A daemon
1351    // that refused to start would take every module down for a feature nothing
1352    // has to use yet.
1353    let (local_listener, local_endpoint, local_endpoint_unavailable) =
1354        if config.local_listener.configured_dir().is_some() {
1355            match local_listener::bind(&config.local_listener, &path) {
1356                Ok(local_listener) => {
1357                    let endpoint = local_listener.endpoint();
1358                    (Some(local_listener), Some(endpoint), None)
1359                }
1360                Err(error) => {
1361                    let reason = error.to_string();
1362                    warn!(
1363                        error = %reason,
1364                        "the local endpoint is unavailable; serving loopback TCP only"
1365                    );
1366                    (None, None, Some(reason))
1367                }
1368            }
1369        } else {
1370            (None, None, None)
1371        };
1372    let connection_info = ConnectionInfo {
1373        schema: SCHEMA_VERSION,
1374        wire_version: Some(PROTOCOL_VERSION),
1375        endpoints,
1376        key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1377        daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1378        pid: process::id(),
1379        daemon_ver: config.daemon_ver,
1380    };
1381
1382    // The low-level binding API historically publishes for callers that serve
1383    // these listeners themselves. The daemon entry point defers publication
1384    // until its own authentication server is running.
1385    let publication_lock = if publish {
1386        let (published, lock) = publish_connection_file(
1387            &path,
1388            #[cfg(windows)]
1389            copy_path.as_deref(),
1390            &connection_info,
1391            local_endpoint.as_ref(),
1392            lock,
1393        )
1394        .await;
1395        if let Err(error) = published {
1396            if let Some(local_listener) = local_listener.as_ref() {
1397                local_listener::release(local_listener);
1398            }
1399            drop(listeners);
1400            return Err(error);
1401        }
1402        drop(lock);
1403        None
1404    } else {
1405        Some(lock)
1406    };
1407
1408    Ok(Outcome::Bound(BoundDaemon {
1409        listeners,
1410        local_listener,
1411        local_endpoint,
1412        local_endpoint_unavailable,
1413        connection_info,
1414        connection_file_path: path,
1415        connection_file_source: config.connection_file_source,
1416        #[cfg(windows)]
1417        connection_file_copy: copy_path,
1418        machine_id,
1419        run_dir_lock,
1420        publication_lock,
1421    }))
1422}
1423
1424#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1425enum Probe {
1426    Live,
1427    StaleOrAbsent,
1428}
1429
1430async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1431    if !matches!(probe_existing(path).await?, Probe::Live) {
1432        return Ok(false);
1433    }
1434    // A terminating peer can still authenticate while its listeners drain.
1435    // Require another proof after a short settling interval before reporting
1436    // a successful duplicate start. If the peer left, proceed with startup;
1437    // any remaining bind/ownership conflict is an error, not exit 0 (which
1438    // would suppress launchd's restart-on-failure policy).
1439    tokio::time::sleep(Duration::from_millis(1500)).await;
1440    Ok(matches!(probe_existing(path).await?, Probe::Live))
1441}
1442
1443async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1444    let info = match connection_file::read(path) {
1445        Ok(info) => info,
1446        Err(source) if is_absent_or_stale_connection_file(&source) => {
1447            return Ok(Probe::StaleOrAbsent)
1448        }
1449        Err(source) => {
1450            return Err(BootstrapError::ConnectionFileRead {
1451                path: path.to_path_buf(),
1452                source,
1453            })
1454        }
1455    };
1456
1457    for endpoint in &info.endpoints {
1458        if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1459            return Ok(Probe::Live);
1460        }
1461    }
1462
1463    Ok(Probe::StaleOrAbsent)
1464}
1465
1466async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1467    let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1468        return Probe::StaleOrAbsent;
1469    };
1470    if !ip.is_loopback() {
1471        return Probe::StaleOrAbsent;
1472    }
1473    let addr = SocketAddr::new(ip, endpoint.port);
1474
1475    let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1476        Ok(Ok(stream)) => stream,
1477        Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1478    };
1479
1480    match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1481        Ok(()) => Probe::Live,
1482        Err(AuthError::DaemonIdMismatch)
1483        | Err(AuthError::InvalidServerProof)
1484        | Err(AuthError::UnexpectedEof { .. })
1485        | Err(AuthError::Timeout { .. })
1486        | Err(AuthError::JsonEncode { .. })
1487        | Err(AuthError::JsonDecode { .. })
1488        | Err(AuthError::Io { .. })
1489        | Err(AuthError::MessageTooLarge { .. })
1490        | Err(AuthError::KeyTooShort { .. })
1491        | Err(AuthError::Random(_))
1492        | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1493    }
1494}
1495
1496async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1497    // Scheduling a server task is not a readiness barrier. Authenticate once
1498    // against the mandatory IPv4 listener before exposing discovery to peers.
1499    // This uses the normal probe deadline, with no retries or widened budget.
1500    if let Some(endpoint) = info.endpoints.first() {
1501        if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1502            return Ok(());
1503        }
1504    }
1505    Err(BootstrapError::StartupNotServing)
1506}
1507
1508/// Run blocking filesystem work (an fsync, a rename) on tokio's blocking-thread
1509/// pool rather than on the async worker thread that awaits it. On a slow or
1510/// busy disk an fsync can take seconds, and a runtime has few worker threads
1511/// (an in-process test daemon may share two with everything else), so doing it
1512/// inline stalls unrelated tasks for that long.
1513///
1514/// `guards` (lock handles) move into the blocking task and are handed back
1515/// with the work's result. If the awaiting future is dropped first, the work
1516/// still runs to completion on its thread and the guards are released only
1517/// after it ends, so a lock is never given up while the work it protects is
1518/// still writing.
1519///
1520/// `Err` means the task panicked, or was cancelled before it started because
1521/// the runtime is shutting down; the guards have then been released already.
1522async fn run_blocking_holding<G, T>(
1523    guards: G,
1524    work: impl FnOnce() -> T + Send + 'static,
1525) -> Result<(T, G), JoinError>
1526where
1527    G: Send + 'static,
1528    T: Send + 'static,
1529{
1530    tokio::task::spawn_blocking(move || {
1531        let output = work();
1532        (output, guards)
1533    })
1534    .await
1535}
1536
1537/// Explain, for a startup error, why blocking work produced no result.
1538fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1539    if join_error.is_panic() {
1540        format!("the background thread {work} panicked before finishing: {join_error}")
1541    } else {
1542        format!(
1543            "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1544        )
1545    }
1546}
1547
1548/// Publish the connection file: owner-only permissions, written to a temporary
1549/// file, fsynced and renamed over `path` (all done by `write_atomic`). The
1550/// write runs on tokio's blocking pool through [`run_blocking_holding`], so the
1551/// fsync never holds an async worker thread.
1552///
1553/// `guard` is the singleton start lock (or whatever the caller must keep held
1554/// across publication); it is returned with the result. It comes back `None`
1555/// only when the write task panicked or was cancelled, in which case the
1556/// result is an error and the file must be treated as not published.
1557///
1558/// On Windows, `copy_path` asks for the profile copy as well. Its start lock is
1559/// taken before the primary write. Once `path` is written, an error returned
1560/// while preparing or writing the copy only logs a warning, and the result is
1561/// still success. The two writes share one blocking task but aren't one atomic
1562/// step, and a panic or cancellation of that task is still a publication
1563/// error.
1564async fn publish_connection_file<G: Send + 'static>(
1565    path: &Path,
1566    #[cfg(windows)] copy_path: Option<&Path>,
1567    info: &ConnectionInfo,
1568    local_endpoint: Option<&LocalEndpoint>,
1569    guard: G,
1570) -> (Result<(), BootstrapError>, Option<G>) {
1571    let task_path = path.to_path_buf();
1572    let task_info = info.clone();
1573    let task_endpoint = local_endpoint.cloned();
1574    #[cfg(windows)]
1575    let copy = windows_connection_copy::claim_for_publication(path, copy_path, info).await;
1576    let outcome = run_blocking_holding(guard, move || {
1577        write_connection_file(&task_path, &task_info, task_endpoint.as_ref())?;
1578        #[cfg(windows)]
1579        if let Some((copy_path, _lock)) = copy {
1580            if let Err(error) =
1581                write_connection_file(&copy_path, &task_info, task_endpoint.as_ref())
1582            {
1583                windows_connection_copy::warn_unavailable(&task_path, Some(&copy_path), &error);
1584            }
1585        }
1586        Ok(())
1587    })
1588    .await;
1589    match outcome {
1590        Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1591        Ok((Err(source), guard)) => (
1592            Err(BootstrapError::ConnectionFileWrite {
1593                path: path.to_path_buf(),
1594                source,
1595            }),
1596            Some(guard),
1597        ),
1598        Err(join_error) => (
1599            Err(BootstrapError::ConnectionFileWrite {
1600                path: path.to_path_buf(),
1601                source: ConnectionFileError::Io {
1602                    op: "write",
1603                    path: path.to_path_buf(),
1604                    source: io::Error::other(describe_blocking_task_failure(
1605                        "writing the connection file",
1606                        &join_error,
1607                    )),
1608                },
1609            }),
1610            None,
1611        ),
1612    }
1613}
1614
1615/// The synchronous connection-file write, including its fsync. Call it only
1616/// through [`publish_connection_file`], which keeps it off the async workers.
1617///
1618/// `local_endpoint` is written through subc-transport's endpoint-aware writer,
1619/// so the file carries `local_endpoint` when there is a local listener and is
1620/// byte-identical to before when there is none.
1621fn write_connection_file(
1622    path: &Path,
1623    info: &ConnectionInfo,
1624    local_endpoint: Option<&LocalEndpoint>,
1625) -> Result<(), ConnectionFileError> {
1626    #[cfg(test)]
1627    connection_file_write_hook::run(path);
1628    write_atomic_with_local_endpoint(path, info, local_endpoint)
1629}
1630
1631/// Lets a test run code inside the connection-file write, on whichever thread
1632/// performs it, for example to block there and check what else can still run.
1633#[cfg(test)]
1634mod connection_file_write_hook {
1635    use std::{
1636        path::{Path, PathBuf},
1637        sync::{Mutex, PoisonError},
1638    };
1639
1640    type Hook = Box<dyn FnOnce() + Send>;
1641
1642    /// Keyed by connection-file path, so a hook fires only for the test that
1643    /// installed it while other tests publish their own files in parallel.
1644    static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
1645
1646    /// Run `hook` once, at the start of the next write of `path`.
1647    pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
1648        HOOKS
1649            .lock()
1650            .unwrap_or_else(PoisonError::into_inner)
1651            .push((path.to_path_buf(), Box::new(hook)));
1652    }
1653
1654    pub(super) fn run(path: &Path) {
1655        let hook = {
1656            let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
1657            hooks
1658                .iter()
1659                .position(|(hooked, _)| hooked == path)
1660                .map(|index| hooks.swap_remove(index).1)
1661        };
1662        if let Some(hook) = hook {
1663            hook();
1664        }
1665    }
1666}
1667
1668/// A path-scoped barrier after publication and before any configured activation.
1669/// The async wait leaves the listeners free to answer discovery-time requests.
1670#[cfg(test)]
1671mod startup_publication_hook {
1672    use std::{
1673        path::{Path, PathBuf},
1674        sync::{Mutex, PoisonError},
1675    };
1676    use tokio::sync::oneshot;
1677
1678    type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
1679    static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
1680
1681    pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
1682        let (entered_tx, entered_rx) = oneshot::channel();
1683        let (release_tx, release_rx) = oneshot::channel();
1684        BARRIERS
1685            .lock()
1686            .unwrap_or_else(PoisonError::into_inner)
1687            .push((path.to_path_buf(), (entered_tx, release_rx)));
1688        (entered_rx, release_tx)
1689    }
1690
1691    pub(super) async fn run(path: &Path) {
1692        let barrier = {
1693            let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
1694            barriers
1695                .iter()
1696                .position(|(hooked, _)| hooked == path)
1697                .map(|index| barriers.swap_remove(index).1)
1698        };
1699        if let Some((entered, release)) = barrier {
1700            let _ = entered.send(());
1701            let _ = release.await;
1702        }
1703    }
1704}
1705
1706fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
1707    match err {
1708        ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
1709        ConnectionFileError::JsonRead { .. }
1710        | ConnectionFileError::UnsupportedSchema { .. }
1711        | ConnectionFileError::Invalid { .. }
1712        | ConnectionFileError::KeyTooShort { .. }
1713        // A live daemon always publishes the file owner-only (0600), so a file
1714        // with insecure permissions is never a daemon we should defer to: treat it
1715        // as stale and take over (which republishes a correct 0600 file).
1716        | ConnectionFileError::InsecurePermissions { .. } => true,
1717        ConnectionFileError::MissingParent { .. }
1718        | ConnectionFileError::MissingFileName { .. }
1719        // A writable ancestor is an operator misconfiguration, never evidence
1720        // about whether a daemon is live. Reclaiming the file would republish key
1721        // material into the same directory the refusal is about.
1722        | ConnectionFileError::InsecureParentDirectory { .. }
1723        | ConnectionFileError::Io { .. }
1724        | ConnectionFileError::JsonWrite { .. }
1725        | ConnectionFileError::Random(_)
1726        // A wire mismatch may identify a newer live daemon, so never reclaim its
1727        // connection file merely because this binary cannot speak its envelope.
1728        | ConnectionFileError::WireVersionMismatch { .. } => false,
1729    }
1730}
1731
1732fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
1733    match fs::remove_file(path) {
1734        Ok(()) => Ok(()),
1735        Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
1736        Err(source) => Err(BootstrapError::RemoveStale {
1737            path: path.to_path_buf(),
1738            source,
1739        }),
1740    }
1741}
1742
1743async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
1744    bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
1745}
1746
1747async fn bind_loopback_with<F, Fut>(
1748    port: u16,
1749    mut bind_v6: F,
1750) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
1751where
1752    F: FnMut(u16) -> Fut,
1753    Fut: std::future::Future<Output = io::Result<TcpListener>>,
1754{
1755    let v4_host = Ipv4Addr::LOCALHOST;
1756    let v4 = TcpListener::bind((v4_host, port))
1757        .await
1758        .map_err(|source| BootstrapError::Bind {
1759            host: v4_host.to_string(),
1760            port,
1761            source,
1762        })?;
1763    let actual_port = v4
1764        .local_addr()
1765        .map_err(|source| BootstrapError::LocalAddr {
1766            host: v4_host.to_string(),
1767            source,
1768        })?
1769        .port();
1770
1771    let mut listeners = vec![v4];
1772    let mut endpoints = vec![Endpoint {
1773        host: v4_host.to_string(),
1774        port: actual_port,
1775    }];
1776
1777    let v6_host = Ipv6Addr::LOCALHOST;
1778    match bind_v6(actual_port).await {
1779        Ok(v6) => {
1780            listeners.push(v6);
1781            endpoints.push(Endpoint {
1782                host: v6_host.to_string(),
1783                port: actual_port,
1784            });
1785        }
1786        Err(err)
1787            if ipv6_loopback_unavailable(&err)
1788                || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
1789        {
1790            warn!(
1791                port = actual_port,
1792                error = %err,
1793                "IPv6 loopback unavailable; serving only IPv4 loopback"
1794            );
1795        }
1796        Err(source) => {
1797            drop(listeners);
1798            return Err(BootstrapError::Bind {
1799                host: v6_host.to_string(),
1800                port: actual_port,
1801                source,
1802            });
1803        }
1804    }
1805
1806    Ok((listeners, endpoints))
1807}
1808
1809fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
1810    matches!(
1811        err.kind(),
1812        io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
1813    ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
1814}
1815
1816struct AbortOnDrop<T> {
1817    handle: JoinHandle<T>,
1818}
1819
1820impl<T> AbortOnDrop<T> {
1821    fn new(handle: JoinHandle<T>) -> Self {
1822        Self { handle }
1823    }
1824
1825    async fn join(&mut self) -> Result<T, JoinError> {
1826        (&mut self.handle).await
1827    }
1828}
1829
1830impl<T> Drop for AbortOnDrop<T> {
1831    fn drop(&mut self) {
1832        if !self.handle.is_finished() {
1833            self.handle.abort();
1834        }
1835    }
1836}
1837
1838#[derive(Debug)]
1839struct StartLock {
1840    // Keep the locked file handle alive for the duration of bootstrap.
1841    _file: fs::File,
1842}
1843
1844impl Drop for StartLock {
1845    fn drop(&mut self) {
1846        // Closing alone leaves the lock held if a concurrent fork inherited
1847        // the open file description and has not exec'd yet. Unlock explicitly
1848        // so a restart can reclaim the singleton as soon as bootstrap ends.
1849        if let Err(error) = FileExt::unlock(&self._file) {
1850            tracing::warn!(%error, "failed to release the singleton start lock");
1851        }
1852    }
1853}
1854
1855impl StartLock {
1856    async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
1857        let path = start_lock_path(connection_file_path);
1858        for _ in 0..START_LOCK_RETRIES {
1859            let file = match open_owner_only_lock(&path) {
1860                Ok(file) => file,
1861                Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
1862            };
1863            match FileExt::try_lock(&file) {
1864                Ok(()) => return Ok(Self { _file: file }),
1865                Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
1866                Err(TryLockError::Error(source)) => {
1867                    return Err(BootstrapError::StartLockCreate { path, source });
1868                }
1869            }
1870        }
1871
1872        Err(BootstrapError::StartLockBusy {
1873            path,
1874            attempts: START_LOCK_RETRIES,
1875        })
1876    }
1877}
1878
1879pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
1880    let mut options = fs::OpenOptions::new();
1881    options.read(true).write(true).create(true);
1882    #[cfg(unix)]
1883    {
1884        use std::os::unix::fs::OpenOptionsExt;
1885        options.mode(0o600);
1886    }
1887    options.open(path)
1888}
1889
1890fn start_lock_path(connection_file_path: &Path) -> PathBuf {
1891    let file_name = connection_file_path
1892        .file_name()
1893        .map(|name| name.to_string_lossy())
1894        .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
1895    let lock_name = format!("{file_name}.start-lock");
1896    connection_file_path
1897        .parent()
1898        .filter(|parent| !parent.as_os_str().is_empty())
1899        .unwrap_or_else(|| Path::new("."))
1900        .join(lock_name)
1901}
1902
1903fn non_empty_os_var(key: &str) -> Option<OsString> {
1904    let value = env::var_os(key)?;
1905    if value.is_empty() {
1906        None
1907    } else {
1908        Some(value)
1909    }
1910}
1911
1912/// Bootstrap-layer errors are deliberately typed so startup never panics for
1913/// ordinary daemon-discovery races or stale filesystem state.
1914#[derive(Debug)]
1915pub enum BootstrapError {
1916    /// The bound server could not authenticate its own readiness probe.
1917    StartupNotServing,
1918    #[cfg(unix)]
1919    Signal(io::Error),
1920    InvalidPort {
1921        raw: String,
1922        source: std::num::ParseIntError,
1923    },
1924    ConnectionFileRead {
1925        path: PathBuf,
1926        source: ConnectionFileError,
1927    },
1928    ConnectionFileWrite {
1929        path: PathBuf,
1930        source: ConnectionFileError,
1931    },
1932    GenerateConnectionFile(ConnectionFileError),
1933    StartLockCreate {
1934        path: PathBuf,
1935        source: io::Error,
1936    },
1937    StartLockBusy {
1938        path: PathBuf,
1939        attempts: usize,
1940    },
1941    /// The run-directory lock file could not be created, opened or locked.
1942    RunDirLockCreate {
1943        path: PathBuf,
1944        source: io::Error,
1945    },
1946    /// Another live process, almost certainly a daemon started with a
1947    /// different runtime directory over the same data home, owns the run
1948    /// directory. The daemon does not start. `holder_pid` is read from the
1949    /// lock file when possible and is informational only.
1950    RunDirBusy {
1951        path: PathBuf,
1952        holder_pid: Option<u32>,
1953    },
1954    RemoveStale {
1955        path: PathBuf,
1956        source: io::Error,
1957    },
1958    Bind {
1959        host: String,
1960        port: u16,
1961        source: io::Error,
1962    },
1963    LocalAddr {
1964        host: String,
1965        source: io::Error,
1966    },
1967    DaemonConfig(DaemonConfigError),
1968    /// The machine id could not be established: its file is corrupt, unreadable
1969    /// or unwritable, or the data home is relative. The daemon does not start.
1970    MachineId(crate::machine_id::MachineIdFileError),
1971    /// The daemon run directory could not be resolved because the data home is
1972    /// relative. The daemon does not start.
1973    RunDir(daemon_config::DaemonRunDirError),
1974    Serve(ServerError),
1975    ServeJoin(tokio::task::JoinError),
1976}
1977
1978impl From<ServerError> for BootstrapError {
1979    fn from(error: ServerError) -> Self {
1980        Self::Serve(error)
1981    }
1982}
1983
1984impl fmt::Display for BootstrapError {
1985    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1986        match self {
1987            Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
1988            #[cfg(unix)]
1989            Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
1990            Self::InvalidPort { raw, source } => {
1991                write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
1992            }
1993            Self::ConnectionFileRead { path, source } => write!(
1994                f,
1995                "failed to read connection file {}: {source}",
1996                path.display()
1997            ),
1998            Self::ConnectionFileWrite { path, source } => write!(
1999                f,
2000                "failed to publish connection file {}: {source}",
2001                path.display()
2002            ),
2003            Self::GenerateConnectionFile(err) => {
2004                write!(f, "failed to generate connection-file auth material: {err}")
2005            }
2006            Self::StartLockCreate { path, source } => {
2007                write!(
2008                    f,
2009                    "failed to create start lock {}: {source}",
2010                    path.display()
2011                )
2012            }
2013            Self::StartLockBusy { path, attempts } => write!(
2014                f,
2015                "start lock {} remained busy after {attempts} attempts",
2016                path.display()
2017            ),
2018            Self::RunDirLockCreate { path, source } => write!(
2019                f,
2020                "refusing to start: failed to lock run directory via {}: {source}",
2021                path.display()
2022            ),
2023            Self::RunDirBusy { path, holder_pid } => {
2024                write!(
2025                    f,
2026                    "refusing to start: run directory lock {} is held by another daemon",
2027                    path.display()
2028                )?;
2029                if let Some(pid) = holder_pid {
2030                    write!(f, " (pid {pid})")?;
2031                }
2032                write!(
2033                    f,
2034                    "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
2035                )
2036            }
2037            Self::RemoveStale { path, source } => write!(
2038                f,
2039                "failed to remove stale connection file {}: {source}",
2040                path.display()
2041            ),
2042            Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
2043                write!(
2044                    f,
2045                    "port {port} in use on loopback {host}: {source}; set the port in config"
2046                )
2047            }
2048            Self::Bind { host, port, source } => {
2049                write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
2050            }
2051            Self::LocalAddr { host, source } => {
2052                write!(f, "failed to read local address for {host}: {source}")
2053            }
2054            Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
2055            Self::MachineId(err) => write!(f, "refusing to start: {err}"),
2056            Self::RunDir(err) => write!(f, "refusing to start: {err}"),
2057            Self::Serve(err) => write!(f, "daemon server failed: {err}"),
2058            Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
2059        }
2060    }
2061}
2062
2063impl Error for BootstrapError {
2064    fn source(&self) -> Option<&(dyn Error + 'static)> {
2065        match self {
2066            Self::StartupNotServing => None,
2067            #[cfg(unix)]
2068            Self::Signal(source) => Some(source),
2069            Self::InvalidPort { source, .. } => Some(source),
2070            Self::ConnectionFileRead { source, .. }
2071            | Self::ConnectionFileWrite { source, .. }
2072            | Self::GenerateConnectionFile(source) => Some(source),
2073            Self::StartLockCreate { source, .. }
2074            | Self::RunDirLockCreate { source, .. }
2075            | Self::RemoveStale { source, .. }
2076            | Self::Bind { source, .. }
2077            | Self::LocalAddr { source, .. } => Some(source),
2078            Self::DaemonConfig(err) => Some(err),
2079            Self::MachineId(err) => Some(err),
2080            Self::RunDir(err) => Some(err),
2081            Self::Serve(err) => Some(err),
2082            Self::ServeJoin(err) => Some(err),
2083            Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
2084        }
2085    }
2086}
2087
2088#[cfg(test)]
2089mod tests {
2090    mod activity {
2091        include!("activity_bootstrap_tests.rs");
2092    }
2093    mod startup_order {
2094        include!("startup_order_tests.rs");
2095    }
2096    use super::*;
2097    use crate::server::ServerAuth;
2098    use cortexkit_test_support::ScratchDir;
2099    #[cfg(target_os = "linux")]
2100    use std::collections::BTreeSet;
2101    use std::sync::Mutex;
2102    #[cfg(target_os = "linux")]
2103    use subc_control::ModuleProtocol;
2104    use subc_transport::{write_atomic, MIN_KEY_LEN};
2105    use tokio::io::AsyncReadExt;
2106    use tokio::task::JoinHandle;
2107
2108    #[cfg(unix)]
2109    use std::os::unix::fs::PermissionsExt;
2110
2111    static ENV_LOCK: Mutex<()> = Mutex::new(());
2112
2113    #[test]
2114    fn normalized_build_provenance_preserves_real_values() {
2115        assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
2116    }
2117
2118    #[test]
2119    fn normalized_build_provenance_omits_unavailable_and_empty_values() {
2120        assert_eq!(normalized_build_provenance("unavailable"), None);
2121        assert_eq!(normalized_build_provenance(""), None);
2122    }
2123
2124    #[cfg(target_os = "linux")]
2125    fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
2126        let cgroups = fs::read_to_string("/proc/self/cgroup")?;
2127        let relative = cgroups
2128            .lines()
2129            .find_map(|line| line.strip_prefix("0::"))
2130            .ok_or_else(|| {
2131                io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
2132            })?;
2133        Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
2134    }
2135
2136    #[cfg(target_os = "linux")]
2137    fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
2138        let modules = current_cgroup_path_for_test()?.join("subc-modules");
2139        let entries = match fs::read_dir(modules) {
2140            Ok(entries) => entries,
2141            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
2142            Err(error) => return Err(error),
2143        };
2144        let mut directories = BTreeSet::new();
2145        for entry in entries {
2146            let entry = entry?;
2147            if entry.file_type()?.is_dir() {
2148                directories.insert(entry.file_name());
2149            }
2150        }
2151        Ok(directories)
2152    }
2153
2154    #[cfg(target_os = "linux")]
2155    async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
2156        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2157        while !path.exists() && tokio::time::Instant::now() < deadline {
2158            assert!(
2159                !task.is_finished(),
2160                "daemon exited before creating {}",
2161                path.display()
2162            );
2163            sleep(Duration::from_millis(10)).await;
2164        }
2165        assert!(path.exists(), "daemon did not create {}", path.display());
2166    }
2167
2168    /// An in-process daemon with the default (disabled) cgroup placement must
2169    /// leave the host's live module cgroup tree exactly as it found it.
2170    /// Red-checking this test (making it fail) creates a
2171    /// `cgroup-isolation-probe-*` directory in the live cgroup tree, which must be removed by hand.
2172    #[cfg(target_os = "linux")]
2173    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2174    async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
2175        let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
2176        let module_id = format!("cgroup-isolation-probe-{}", process::id());
2177        let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
2178        assert!(
2179            !before.contains(&OsString::from(&module_id)),
2180            "isolation probe cgroup already exists before this daemon starts"
2181        );
2182        let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
2183        let module = ConfiguredModule {
2184            carry_containment: false,
2185            module_id,
2186            program: PathBuf::from("sh"),
2187            args: vec!["-c".to_string(), "sleep 30".to_string()],
2188            env: Vec::new(),
2189            log: None,
2190            enabled: true,
2191            reserved: false,
2192            reserved_prefixes: Vec::new(),
2193            protocol: ModuleProtocol::None,
2194            overlap: Default::default(),
2195            health: HealthConfig::default(),
2196            drain_timeout_ms: None,
2197            register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2198            route_bind_relay_timeout_ms: None,
2199            restart: RestartPolicy::default(),
2200        };
2201        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2202            .with_configured_modules([module])
2203            .with_capture_logs_dir(temp.join("logs"))
2204            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2205        let task = tokio::spawn(run_with_config(config));
2206
2207        wait_for_path(&capture, &task).await;
2208        let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2209
2210        task.abort();
2211        assert!(task
2212            .await
2213            .expect_err("aborted daemon task must cancel")
2214            .is_cancelled());
2215        assert_eq!(
2216            before, after,
2217            "an in-process daemon must not create or reconcile ambient module cgroups"
2218        );
2219    }
2220
2221    #[cfg(target_os = "linux")]
2222    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2223    async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2224        let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2225        let cgroup_root = temp.join("cgroup");
2226        fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2227        fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2228        let modules = cgroup_root.join("subc-modules");
2229        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2230            .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2231            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2232        let task = tokio::spawn(run_with_config(config));
2233
2234        wait_for_path(&modules, &task).await;
2235
2236        task.abort();
2237        assert!(task
2238            .await
2239            .expect_err("aborted daemon task must cancel")
2240            .is_cancelled());
2241        assert!(
2242            fs::read_dir(&modules)
2243                .expect("read prepared modules directory")
2244                .next()
2245                .is_none(),
2246            "the delegation probe must clean up after itself"
2247        );
2248    }
2249
2250    struct EnvGuard {
2251        key: &'static str,
2252        previous: Option<OsString>,
2253    }
2254
2255    impl EnvGuard {
2256        fn set(key: &'static str, value: &Path) -> Self {
2257            let previous = env::var_os(key);
2258            env::set_var(key, value);
2259            Self { key, previous }
2260        }
2261
2262        fn set_str(key: &'static str, value: &str) -> Self {
2263            let previous = env::var_os(key);
2264            env::set_var(key, value);
2265            Self { key, previous }
2266        }
2267
2268        fn unset(key: &'static str) -> Self {
2269            let previous = env::var_os(key);
2270            env::remove_var(key);
2271            Self { key, previous }
2272        }
2273    }
2274
2275    impl Drop for EnvGuard {
2276        fn drop(&mut self) {
2277            match &self.previous {
2278                Some(value) => env::set_var(self.key, value),
2279                None => env::remove_var(self.key),
2280            }
2281        }
2282    }
2283
2284    fn unique_temp_dir(name: &str) -> ScratchDir {
2285        ScratchDir::new(name)
2286    }
2287
2288    fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2289        let dir = unique_temp_dir(name);
2290        let path = dir.join("conn.json");
2291        (dir, path)
2292    }
2293
2294    fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2295        ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2296    }
2297
2298    fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2299        let auth = auth_for(&bound.connection_info);
2300        tokio::spawn(serve_listeners(
2301            bound.listeners,
2302            Arc::new(Router::with_default_self_handler()),
2303            auth,
2304        ))
2305    }
2306
2307    fn expect_bound(outcome: Outcome) -> BoundDaemon {
2308        match outcome {
2309            Outcome::Bound(bound) => bound,
2310            Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2311        }
2312    }
2313
2314    async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2315        let endpoint = conn
2316            .endpoints
2317            .first()
2318            .expect("test connection file should have an endpoint");
2319        let ip: IpAddr = endpoint.host.parse().unwrap();
2320        TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2321    }
2322
2323    fn make_connection_info(port: u16) -> ConnectionInfo {
2324        ConnectionInfo {
2325            schema: SCHEMA_VERSION,
2326            wire_version: Some(PROTOCOL_VERSION),
2327            endpoints: vec![Endpoint {
2328                host: "127.0.0.1".to_owned(),
2329                port,
2330            }],
2331            key: generate_key().unwrap(),
2332            daemon_id: generate_daemon_id().unwrap(),
2333            pid: process::id(),
2334            daemon_ver: "test-subc".to_owned(),
2335        }
2336    }
2337
2338    fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2339        fs::write(path, contents).unwrap();
2340        #[cfg(unix)]
2341        fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2342    }
2343
2344    fn assert_owner_only_connection_file(path: &Path) {
2345        // `path` is only inspected on Unix (mode bits); on Windows the owner-only
2346        // guarantee comes from the inherited %TEMP% ACL, nothing to assert here.
2347        #[cfg(unix)]
2348        {
2349            let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2350            assert_eq!(mode, 0o600);
2351        }
2352        #[cfg(not(unix))]
2353        let _ = path;
2354    }
2355
2356    #[test]
2357    fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2358        let _env_lock = ENV_LOCK.lock().unwrap();
2359        let runtime_dir = unique_temp_dir("xdg-runtime");
2360        let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2361
2362        assert_eq!(
2363            connection_file_path(),
2364            runtime_dir.join(CONNECTION_FILE_NAME)
2365        );
2366    }
2367
2368    #[test]
2369    fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2370        let runtime_dir = OsString::from("/run/user/1000");
2371
2372        let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2373
2374        assert_eq!(
2375            path,
2376            PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2377        );
2378        assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2379    }
2380
2381    #[test]
2382    fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2383        let _env_lock = ENV_LOCK.lock().unwrap();
2384        let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2385
2386        assert_eq!(
2387            connection_file_path(),
2388            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2389        );
2390    }
2391
2392    #[test]
2393    fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2394        let (path, source) = connection_file_path_with_source(None);
2395
2396        assert_eq!(
2397            path,
2398            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2399        );
2400        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2401    }
2402
2403    /// Concurrent callers must derive one token. The former temp-file uid probe
2404    /// could fail transiently (same-tick name collision, fd exhaustion) and send
2405    /// the loser down the env-derived fallback with a different identity for the
2406    /// same user; this fence keeps identity independent of filesystem luck.
2407    #[test]
2408    fn user_connection_token_is_stable_under_concurrent_callers() {
2409        let expected = user_connection_token();
2410        let workers: Vec<_> = (0..32)
2411            .map(|_| {
2412                std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2413            })
2414            .collect();
2415        for worker in workers {
2416            for token in worker.join().expect("probe thread") {
2417                assert_eq!(token, expected, "token diverged under concurrent probes");
2418            }
2419        }
2420    }
2421
2422    #[test]
2423    fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
2424        let (path, source) = connection_file_path_with_source(Some(OsString::new()));
2425
2426        assert_eq!(
2427            path,
2428            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2429        );
2430        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2431    }
2432
2433    /// The shipped binary is the only caller that journals terminal exits into
2434    /// the real run directory, so its constructor must supply that path itself;
2435    /// the in-process default leaves it unset.
2436    #[test]
2437    fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
2438        let _env_lock = ENV_LOCK.lock().unwrap();
2439        let root = unique_temp_dir("daemon-binary-config");
2440        let data_home = root.join("data");
2441        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
2442        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2443        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2444
2445        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
2446
2447        let run_dir = data_home.join("cortexkit").join("run");
2448        assert_eq!(
2449            config.terminal_journal_path,
2450            Some(run_dir.join("terminals.jsonl"))
2451        );
2452        assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
2453        assert_eq!(
2454            BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
2455            None,
2456            "an in-process config must not journal anywhere unless asked to"
2457        );
2458    }
2459
2460    #[test]
2461    fn daemon_binary_config_refuses_a_relative_data_home() {
2462        let _env_lock = ENV_LOCK.lock().unwrap();
2463        let root = unique_temp_dir("daemon-binary-relative-data");
2464        let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
2465        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2466        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2467
2468        let error = BootstrapConfig::from_env_for_daemon_binary()
2469            .expect_err("a relative data home must refuse the daemon binary's config");
2470        assert!(
2471            matches!(error, BootstrapError::RunDir(_)),
2472            "expected a run-directory refusal, got {error}"
2473        );
2474        assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
2475    }
2476
2477    #[test]
2478    fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
2479        let _env_lock = ENV_LOCK.lock().unwrap();
2480        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2481        let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
2482        let config_path = conn_path.with_file_name("subc.jsonc");
2483        for (doc, expected) in [
2484            (None, daemon_config::PrivilegedOperatorPrompt::Log),
2485            (
2486                Some(r#"{"version":1}"#),
2487                daemon_config::PrivilegedOperatorPrompt::Log,
2488            ),
2489            (
2490                Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
2491                daemon_config::PrivilegedOperatorPrompt::Log,
2492            ),
2493            (
2494                Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
2495                daemon_config::PrivilegedOperatorPrompt::Enforce,
2496            ),
2497        ] {
2498            if let Some(doc) = doc {
2499                fs::write(&config_path, doc).unwrap();
2500            }
2501            assert_eq!(
2502                BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2503                    .unwrap()
2504                    .privileged_operator_prompt,
2505                expected
2506            );
2507            assert_eq!(
2508                BootstrapConfig::new(&conn_path, 0)
2509                    .with_daemon_config_path(&config_path)
2510                    .unwrap()
2511                    .privileged_operator_prompt,
2512                expected
2513            );
2514        }
2515    }
2516
2517    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2518    async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
2519        let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
2520        let connection_path = dir.join("connection.json");
2521        let config_path = dir.join("subc.jsonc");
2522        fs::write(
2523            &config_path,
2524            r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
2525        )
2526        .unwrap();
2527        let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
2528            .with_daemon_config_path(&config_path)
2529            .unwrap();
2530        let connection_path = config.connection_file_path.clone();
2531        let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
2532        tokio::time::timeout(Duration::from_secs(5), async {
2533            while !connection_path.exists() {
2534                assert!(
2535                    !daemon.handle.is_finished(),
2536                    "daemon exited before publishing"
2537                );
2538                tokio::task::yield_now().await;
2539            }
2540        })
2541        .await
2542        .unwrap();
2543        let info = connection_file::read(&connection_path).unwrap();
2544        let mut client = connect_from_info(&info).await.unwrap();
2545        authenticate_client(&mut client, &info, Duration::from_secs(5))
2546            .await
2547            .unwrap();
2548        for (correlation, mode, expected) in [
2549            (1, "enforce", Vec::new()),
2550            (2, "log", vec!["privileged_operator_prompt"]),
2551            (3, "log", vec!["privileged_operator_prompt"]),
2552        ] {
2553            fs::write(
2554                &config_path,
2555                serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
2556            )
2557            .unwrap();
2558            let frame = crate::Frame::build(
2559                subc_protocol::FrameType::Request,
2560                subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
2561                0,
2562                0,
2563                correlation,
2564                serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
2565                    preview: true,
2566                })
2567                .unwrap(),
2568            )
2569            .unwrap();
2570            crate::write_frame(&mut client, &frame).await.unwrap();
2571            let reply =
2572                tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
2573                    .await
2574                    .unwrap()
2575                    .unwrap()
2576                    .unwrap();
2577            assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
2578            assert_eq!(reply.header.corr, correlation);
2579            let subc_control::ClientControlResponse::SupervisorRescan { result } =
2580                serde_json::from_slice(&reply.body).unwrap()
2581            else {
2582                panic!("expected a rescan response");
2583            };
2584            assert_eq!(result.restart_required, expected);
2585        }
2586    }
2587
2588    #[test]
2589    fn configured_port_uses_default_config_and_env_override() {
2590        let _env_lock = ENV_LOCK.lock().unwrap();
2591        let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
2592        let config_path = conn_path.with_file_name("subc.jsonc");
2593
2594        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2595        assert_eq!(
2596            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2597                .unwrap()
2598                .port,
2599            DEFAULT_SUBC_PORT
2600        );
2601
2602        fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
2603        assert_eq!(
2604            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2605                .unwrap()
2606                .port,
2607            8123
2608        );
2609
2610        let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
2611        assert_eq!(
2612            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2613                .unwrap()
2614                .port,
2615            9012
2616        );
2617    }
2618
2619    #[tokio::test]
2620    async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
2621        let (_dir, path) = temp_connection_file_path("departing-peer");
2622        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2623        let info = make_connection_info(listener.local_addr().unwrap().port());
2624        write_connection_file(&path, &info, None).unwrap();
2625        let peer = tokio::spawn(async move {
2626            let (mut stream, _) = listener.accept().await.unwrap();
2627            subc_transport::authenticate_server(
2628                &mut stream,
2629                &info.key,
2630                &info.daemon_id,
2631                &info.daemon_ver,
2632                PROBE_AUTH_DEADLINE,
2633            )
2634            .await
2635            .unwrap();
2636        });
2637        let outcome = ensure_singleton(&path, 0).await.unwrap();
2638        peer.await.unwrap();
2639        assert!(
2640            matches!(outcome, Outcome::Bound(_)),
2641            "a departing peer is not AlreadyRunning"
2642        );
2643    }
2644
2645    #[tokio::test]
2646    async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
2647        let (_dir, path) = temp_connection_file_path("already-running");
2648
2649        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2650        let server = start_server(bound);
2651
2652        let second = ensure_singleton(&path, 0).await.unwrap();
2653        assert!(matches!(second, Outcome::AlreadyRunning));
2654
2655        server.abort();
2656        let _ = server.await;
2657    }
2658
2659    #[tokio::test]
2660    async fn daemon_connection_file_publishes_protocol_wire_version() {
2661        let (_dir, path) = temp_connection_file_path("wire-version");
2662        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2663        assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
2664        assert_eq!(
2665            connection_file::read(&path).unwrap().wire_version,
2666            Some(PROTOCOL_VERSION)
2667        );
2668
2669        drop(bound.listeners);
2670    }
2671
2672    #[tokio::test]
2673    async fn stale_unbound_connection_file_is_reclaimed() {
2674        let (_dir, path) = temp_connection_file_path("stale-reclaim");
2675        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2676        let stale_port = stale.local_addr().unwrap().port();
2677        drop(stale);
2678        let stale_info = make_connection_info(stale_port);
2679        write_atomic(&path, &stale_info).unwrap();
2680
2681        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2682        assert_ne!(bound.connection_info.key, stale_info.key);
2683        drop(bound.listeners);
2684    }
2685
2686    #[cfg(unix)]
2687    #[tokio::test]
2688    async fn ensure_singleton_reclaims_insecure_connection_file() {
2689        let (_dir, path) = temp_connection_file_path("insecure-reclaim");
2690        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2691        let stale_port = stale.local_addr().unwrap().port();
2692        drop(stale);
2693        let stale_info = make_connection_info(stale_port);
2694        write_atomic(&path, &stale_info).unwrap();
2695        fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
2696
2697        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2698        assert_ne!(bound.connection_info.key, stale_info.key);
2699        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2700        assert_owner_only_connection_file(&path);
2701
2702        drop(bound.listeners);
2703    }
2704
2705    #[tokio::test]
2706    async fn ensure_singleton_reclaims_non_loopback_connection_file() {
2707        let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
2708        let mut stale_info = make_connection_info(8757);
2709        stale_info.endpoints = vec![Endpoint {
2710            host: "192.0.2.10".to_owned(),
2711            port: 8757,
2712        }];
2713        write_atomic(&path, &stale_info).unwrap();
2714
2715        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2716        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
2717        assert_ne!(bound.connection_info.key, stale_info.key);
2718        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2719        assert!(bound
2720            .connection_info
2721            .endpoints
2722            .iter()
2723            .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
2724        assert_owner_only_connection_file(&path);
2725
2726        drop(bound.listeners);
2727    }
2728
2729    #[tokio::test]
2730    async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
2731        let mut unsupported_schema = make_connection_info(8757);
2732        unsupported_schema.schema = SCHEMA_VERSION + 1;
2733
2734        let mut empty_endpoints = make_connection_info(8757);
2735        empty_endpoints.endpoints.clear();
2736
2737        let mut short_key = make_connection_info(8757);
2738        short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
2739
2740        let cases = vec![
2741            (
2742                "unsupported-schema",
2743                serde_json::to_vec(&unsupported_schema).unwrap(),
2744                Some(unsupported_schema),
2745            ),
2746            (
2747                "empty-endpoints",
2748                serde_json::to_vec(&empty_endpoints).unwrap(),
2749                Some(empty_endpoints),
2750            ),
2751            (
2752                "short-key",
2753                serde_json::to_vec(&short_key).unwrap(),
2754                Some(short_key),
2755            ),
2756            ("invalid-json", b"{not valid connection json".to_vec(), None),
2757        ];
2758
2759        for (label, contents, old_info) in cases {
2760            let (_dir, path) = temp_connection_file_path(label);
2761            write_raw_owner_only_connection_file(&path, &contents);
2762
2763            let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2764            if let Some(old_info) = old_info {
2765                assert_ne!(bound.connection_info.key, old_info.key, "{label}");
2766                assert_ne!(
2767                    bound.connection_info.daemon_id, old_info.daemon_id,
2768                    "{label}"
2769                );
2770            }
2771            assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
2772            assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
2773            assert_owner_only_connection_file(&path);
2774
2775            drop(bound.listeners);
2776        }
2777    }
2778
2779    #[tokio::test]
2780    async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
2781        let (_dir, path) = temp_connection_file_path("foreign-reclaim");
2782        let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2783        let foreign_port = foreign.local_addr().unwrap().port();
2784        write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
2785        let foreign_task = tokio::spawn(async move {
2786            if let Ok((mut stream, _)) = foreign.accept().await {
2787                let mut buf = [0u8; 64];
2788                let _ = stream.read(&mut buf).await;
2789            }
2790        });
2791
2792        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2793        assert!(bound
2794            .connection_info
2795            .endpoints
2796            .iter()
2797            .all(|endpoint| endpoint.port != foreign_port));
2798
2799        drop(bound.listeners);
2800        let _ = foreign_task.await;
2801    }
2802
2803    #[tokio::test]
2804    async fn stale_start_lock_file_is_reclaimable() {
2805        let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
2806        let lock_path = start_lock_path(&path);
2807        drop(open_owner_only_lock(&lock_path).unwrap());
2808        assert!(lock_path.is_file());
2809
2810        let lock = StartLock::acquire(&path).await.unwrap();
2811        assert!(lock_path.is_file());
2812
2813        drop(lock);
2814        assert!(lock_path.is_file());
2815    }
2816
2817    #[tokio::test]
2818    async fn held_start_lock_blocks_second_acquire_until_release() {
2819        let (_dir, path) = temp_connection_file_path("start-lock-held");
2820        let lock_path = start_lock_path(&path);
2821        let first = StartLock::acquire(&path).await.unwrap();
2822
2823        let err = match StartLock::acquire(&path).await {
2824            Ok(_) => panic!("second acquire while held must stay busy"),
2825            Err(err) => err,
2826        };
2827        assert!(matches!(
2828            err,
2829            BootstrapError::StartLockBusy {
2830                ref path,
2831                attempts: START_LOCK_RETRIES,
2832            } if path == &lock_path
2833        ));
2834
2835        drop(first);
2836
2837        let second = StartLock::acquire(&path)
2838            .await
2839            .expect("released advisory lock should be reclaimable");
2840        drop(second);
2841    }
2842
2843    #[tokio::test]
2844    async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
2845        let (_dir, path) = temp_connection_file_path("bind-conflict");
2846        let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2847        let occupied_port = occupied.local_addr().unwrap().port();
2848
2849        let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
2850        assert!(matches!(
2851            err,
2852            BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
2853        ));
2854        assert!(err.to_string().contains("set the port in config"));
2855
2856        drop(occupied);
2857    }
2858
2859    #[tokio::test]
2860    async fn ephemeral_ipv6_collision_still_serves_ipv4() {
2861        let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
2862            let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
2863            let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
2864            drop(occupied);
2865            result
2866        })
2867        .await
2868        .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
2869        assert_eq!(endpoints.len(), 1);
2870        assert_eq!(endpoints[0].host, "127.0.0.1");
2871        TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
2872            .await
2873            .unwrap();
2874        drop(listeners);
2875    }
2876
2877    #[tokio::test]
2878    async fn unpublished_boot_holds_singleton_lock_until_publication() {
2879        let dir = ScratchDir::new("ul");
2880        let path = dir.join("c");
2881        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2882        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2883        assert!(!path.exists());
2884        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2885        assert!(
2886            matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
2887            "an unpublished boot must keep its singleton claim"
2888        );
2889        drop(bound);
2890        FileExt::try_lock(&contender).unwrap();
2891    }
2892
2893    #[tokio::test]
2894    async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
2895        let dir = ScratchDir::new("sl");
2896        let path = dir.join("c");
2897        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2898        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2899        // try_clone shares the open file description just as fork does,
2900        // without relying on the timing of another test's child spawn.
2901        let inherited = bound
2902            .publication_lock
2903            .as_ref()
2904            .unwrap()
2905            ._file
2906            .try_clone()
2907            .unwrap();
2908        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2909        assert!(matches!(
2910            FileExt::try_lock(&contender),
2911            Err(TryLockError::WouldBlock)
2912        ));
2913        drop(bound);
2914        FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
2915        drop(inherited);
2916    }
2917
2918    #[tokio::test]
2919    async fn readiness_probe_requires_an_authenticating_server() {
2920        let dir = ScratchDir::new("unserved-readiness");
2921        let bound = expect_bound(
2922            ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
2923                .await
2924                .unwrap(),
2925        );
2926        assert!(matches!(
2927            verify_serving(&bound.connection_info).await,
2928            Err(BootstrapError::StartupNotServing)
2929        ));
2930    }
2931
2932    #[cfg(unix)]
2933    #[tokio::test]
2934    async fn orphan_sweep_finishes_before_connection_file_publication() {
2935        use crate::live_children::{ExecutableIdentity, LiveChild};
2936        let dir = ScratchDir::new("pas");
2937        // A one-letter connection file name, and no long socket path: the sweep
2938        // this test covers runs before the local endpoint is bound, so a longer
2939        // name would only add a way for the test to fail on something else.
2940        let path = dir.join("c");
2941        let record = dir.join("live-children.json");
2942        let ready = dir.join("ready");
2943        let mut child = std::process::Command::new("/bin/sh")
2944            .args([
2945                "-c",
2946                "trap '' TERM; : > \"$0\"; read -r _",
2947                ready.to_str().unwrap(),
2948            ])
2949            .stdin(std::process::Stdio::piped())
2950            .env("XDG_DATA_HOME", dir.path())
2951            .env("XDG_RUNTIME_DIR", dir.path())
2952            .env("XDG_CONFIG_HOME", dir.path())
2953            .spawn()
2954            .unwrap();
2955        timeout(Duration::from_secs(5), async {
2956            while !ready.exists() {
2957                sleep(Duration::from_millis(5)).await;
2958            }
2959        })
2960        .await
2961        .unwrap();
2962        let observed = subc_os::Process::open(child.id())
2963            .unwrap()
2964            .unwrap()
2965            .observe()
2966            .unwrap();
2967        crate::live_children::write_record(
2968            &record,
2969            &[LiveChild {
2970                container: None,
2971                module_id: "old-sleep".into(),
2972                pid: child.id(),
2973                protocol: subc_control::ModuleProtocol::None,
2974                start_time: Some(observed.start_time),
2975                executable: observed.executable.map(ExecutableIdentity::from),
2976                cgroup_name: None,
2977            }],
2978        )
2979        .unwrap();
2980        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
2981            .with_live_children_record(&record);
2982        let daemon = tokio::spawn(run_with_config(config));
2983        timeout(Duration::from_secs(5), async {
2984            while !path.exists() {
2985                sleep(Duration::from_millis(5)).await;
2986            }
2987        })
2988        .await
2989        .unwrap();
2990        assert!(path.exists());
2991        let ended = child.try_wait().unwrap().is_some();
2992        if !ended {
2993            child.kill().unwrap();
2994        }
2995        child.wait().unwrap();
2996        if ended {
2997            assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
2998        }
2999        daemon.abort();
3000        let _ = daemon.await;
3001        assert!(
3002            ended,
3003            "the published daemon must not still owe its orphan sweep"
3004        );
3005    }
3006
3007    /// Publishing the connection file fsyncs, which can take seconds on a busy
3008    /// disk, so the daemon must do it off the async worker thread. The hook
3009    /// blocks inside the write until another task on the same single-threaded
3010    /// runtime has run. Writing on the worker thread would starve that task: the
3011    /// hook then gives up after its deadline and the test fails, as it does if
3012    /// startup publishes without going through the hooked write at all.
3013    #[tokio::test(flavor = "current_thread")]
3014    async fn connection_file_publication_leaves_the_async_runtime_free() {
3015        use std::sync::atomic::{AtomicBool, Ordering};
3016
3017        let (_dir, path) = temp_connection_file_path("publish-off-runtime");
3018        let write_started = Arc::new(AtomicBool::new(false));
3019        let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
3020        let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
3021        let hook_started = Arc::clone(&write_started);
3022        connection_file_write_hook::install(&path, move || {
3023            hook_started.store(true, Ordering::SeqCst);
3024            let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
3025            let _ = verdict_tx.send(progressed);
3026        });
3027        // Signals only after the write has started, so a message can only come
3028        // from this task running while the write is still blocked.
3029        let ticker = tokio::spawn(async move {
3030            loop {
3031                if write_started.load(Ordering::SeqCst) {
3032                    let _ = progress_tx.send(());
3033                    return;
3034                }
3035                sleep(Duration::from_millis(5)).await;
3036            }
3037        });
3038
3039        let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
3040        let published = timeout(Duration::from_secs(15), async {
3041            while !path.exists() && !daemon.is_finished() {
3042                sleep(Duration::from_millis(5)).await;
3043            }
3044        })
3045        .await;
3046        let daemon_exited = daemon.is_finished();
3047        daemon.abort();
3048        let daemon_result = daemon.await;
3049        ticker.abort();
3050
3051        assert!(
3052            published.is_ok() && !daemon_exited,
3053            "the daemon must start and publish its connection file: {daemon_result:?}"
3054        );
3055        match verdict_rx.try_recv() {
3056            Ok(true) => {}
3057            Ok(false) => panic!(
3058                "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
3059            ),
3060            Err(_) => panic!("startup published the connection file without the hooked write"),
3061        }
3062    }
3063
3064    /// A write that never reports back must stop startup with an error naming
3065    /// the connection file, never be taken as published.
3066    #[tokio::test]
3067    async fn a_panicking_connection_file_write_fails_startup() {
3068        let (_dir, path) = temp_connection_file_path("publish-panics");
3069        connection_file_write_hook::install(&path, || {
3070            panic!("deliberate panic inside the connection-file write")
3071        });
3072
3073        let error = timeout(
3074            Duration::from_secs(15),
3075            run_with_config(BootstrapConfig::new(&path, 0)),
3076        )
3077        .await
3078        .expect("startup must end once the write task fails")
3079        .expect_err("startup must fail when the connection-file write panics");
3080
3081        assert!(
3082            matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
3083            "expected a connection-file write error, got {error:?}"
3084        );
3085        let message = error.to_string();
3086        assert!(message.contains(&path.display().to_string()), "{message}");
3087        assert!(message.contains("panicked"), "{message}");
3088        assert!(!path.exists(), "no connection file may be published");
3089    }
3090
3091    #[cfg(target_os = "macos")]
3092    #[test]
3093    fn nofile_raise_worker() {
3094        let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
3095            return;
3096        };
3097        let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
3098        rlimit::Resource::NOFILE.set(256, hard).unwrap();
3099        raise_nofile_limit_with_ceiling(Some(10240));
3100        std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
3101    }
3102
3103    #[cfg(target_os = "macos")]
3104    #[test]
3105    fn macos_nofile_raise_clamps_to_kernel_ceiling() {
3106        let dir = ScratchDir::new("nofile-ceiling");
3107        let result = dir.join("soft");
3108        let status = std::process::Command::new(std::env::current_exe().unwrap())
3109            .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
3110            .env("SUBC_NOFILE_TEST_FILE", &result)
3111            .env("XDG_DATA_HOME", dir.path())
3112            .env("XDG_RUNTIME_DIR", dir.path())
3113            .env("XDG_CONFIG_HOME", dir.path())
3114            .status()
3115            .unwrap();
3116        assert!(status.success());
3117        let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
3118        assert!(
3119            soft > 256 && soft <= 10240,
3120            "the inherited limit must be raised within the kernel ceiling; got {soft}"
3121        );
3122    }
3123
3124    #[tokio::test]
3125    async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
3126        let (_dir, path) = temp_connection_file_path("key-rotation");
3127        let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3128        let old_info = first.connection_info.clone();
3129        let fixed_port = old_info.endpoints[0].port;
3130        drop(first.listeners);
3131
3132        let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
3133        let new_info = second.connection_info.clone();
3134        assert_ne!(old_info.key, new_info.key);
3135        assert_ne!(old_info.daemon_id, new_info.daemon_id);
3136        let server = start_server(second);
3137
3138        let mut old_stream = connect_from_info(&old_info).await.unwrap();
3139        let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
3140        assert!(
3141            old_auth.is_err(),
3142            "old key must not authenticate after restart"
3143        );
3144
3145        let reread = connection_file::read(&path).unwrap();
3146        let mut new_stream = connect_from_info(&reread).await.unwrap();
3147        authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
3148            .await
3149            .unwrap();
3150
3151        server.abort();
3152        let _ = server.await;
3153    }
3154
3155    #[cfg(unix)]
3156    #[tokio::test]
3157    async fn published_connection_file_permissions_are_owner_only() {
3158        let (_dir, path) = temp_connection_file_path("permissions");
3159        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3160
3161        let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
3162        assert_eq!(mode, 0o600);
3163
3164        drop(bound.listeners);
3165    }
3166
3167    /// A config with the socket in `dir`'s own short subdirectory.
3168    ///
3169    /// A Unix socket path is limited to just over a hundred bytes, and this
3170    /// harness's temporary directories use nearly all of that, so a test that
3171    /// boots a real daemon passes a short directory and a one-letter connection
3172    /// file name; anything longer makes the bind refuse. On Windows there is no
3173    /// socket path to keep short, so the config is returned unchanged.
3174    fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
3175        let dir = config
3176            .connection_file_path
3177            .parent()
3178            .map(Path::to_path_buf)
3179            .unwrap_or_else(|| PathBuf::from("."));
3180        #[cfg(unix)]
3181        let config = BootstrapConfig {
3182            connection_file_path: dir.join("c"),
3183            local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
3184            ..config
3185        };
3186        #[cfg(not(unix))]
3187        let _ = dir;
3188        config
3189    }
3190
3191    /// The shipped binary puts the socket in its own run directory. The file is
3192    /// named from the connection file's path, so a restart finds it again.
3193    #[cfg(unix)]
3194    #[test]
3195    fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
3196        let _env_lock = ENV_LOCK.lock().unwrap();
3197        let root = ScratchDir::new("binary-local-socket");
3198        let data_home = root.join("data");
3199        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3200        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3201        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3202
3203        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3204
3205        let run_dir = data_home.join("cortexkit").join("run");
3206        assert_eq!(
3207            config.local_listener.configured_dir(),
3208            Some(run_dir.as_path()),
3209            "the binary serves from its run directory"
3210        );
3211        let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
3212            .expect("the binary config names a directory");
3213        assert_eq!(socket.parent().unwrap(), run_dir);
3214        assert_eq!(
3215            socket.file_name().unwrap().to_string_lossy(),
3216            "subc.sock",
3217            "a fixed name: the run directory belongs to one data home"
3218        );
3219    }
3220
3221    /// An in-process daemon that names no directory serves TCP only: it publishes
3222    /// no local endpoint, creates nothing under the data home, and says nothing is
3223    /// unavailable, because nothing was asked for.
3224    /// The environment lock covers only the setup: the daemon under test reads
3225    /// its data home from the process environment while it boots, and holding
3226    /// the lock across that await would block every other test that sets one.
3227    #[cfg(unix)]
3228    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3229    async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
3230        let root = ScratchDir::new("in-process-no-local-socket");
3231        let data_home = root.join("data");
3232        fs::create_dir_all(&data_home).unwrap();
3233        let connection_path = root.join("c");
3234        let _data;
3235        let _config;
3236        let _port;
3237        {
3238            let _env_lock = ENV_LOCK.lock().unwrap();
3239            _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3240            _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3241            _port = EnvGuard::unset(SUBC_PORT_ENV);
3242        }
3243
3244        let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
3245
3246        assert!(bound.local_endpoint.is_none());
3247        assert!(bound.local_endpoint_unavailable.is_none());
3248        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3249        assert!(published.local_endpoint.is_none());
3250        assert!(
3251            local_listener::socket_path_for(None, &connection_path).is_none(),
3252            "no configured directory, no socket"
3253        );
3254        let mut leftovers = Vec::new();
3255        let mut pending = vec![data_home.clone()];
3256        while let Some(dir) = pending.pop() {
3257            for entry in fs::read_dir(&dir).unwrap() {
3258                let entry = entry.unwrap();
3259                let path = entry.path();
3260                if entry.file_type().unwrap().is_dir() {
3261                    pending.push(path.clone());
3262                }
3263                if entry.file_name().to_string_lossy().ends_with(".sock") {
3264                    leftovers.push(path);
3265                }
3266            }
3267        }
3268        assert!(
3269            leftovers.is_empty(),
3270            "an in-process daemon must not create a socket in the data home: {leftovers:?}"
3271        );
3272    }
3273
3274    /// A local endpoint that cannot be bound does not stop the daemon: it keeps
3275    /// serving TCP, publishes no local endpoint, and reports the reason where an
3276    /// operator reads daemon facts. Every module stays up for a feature nothing
3277    /// has to use.
3278    #[cfg(unix)]
3279    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3280    async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
3281        let dir = ScratchDir::new("tcp-only");
3282        let canonical = std::fs::canonicalize(&dir).unwrap();
3283        let connection_path = canonical.join("c");
3284        // The socket directory is deep enough that the socket's path is over the
3285        // Unix socket limit, so binding it must fail.
3286        let deep = canonical.join("x".repeat(80));
3287        fs::create_dir(&deep).unwrap();
3288        fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
3289        let config =
3290            BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
3291        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3292        assert!(
3293            bound.local_endpoint.is_none(),
3294            "a socket that cannot be bound is not published"
3295        );
3296        assert!(
3297            bound
3298                .local_endpoint_unavailable
3299                .as_deref()
3300                .is_some_and(|reason| reason.contains("too long")),
3301            "{:?}",
3302            bound.local_endpoint_unavailable
3303        );
3304        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3305        assert!(published.local_endpoint.is_none());
3306        assert!(
3307            !published.info.endpoints.is_empty(),
3308            "TCP is still published"
3309        );
3310
3311        // The daemon serves TCP as usual.
3312        let server = start_server(bound);
3313        let info = connection_file::read(&connection_path).unwrap();
3314        let mut client = connect_from_info(&info).await.unwrap();
3315        authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
3316            .await
3317            .expect("a TCP client still completes the handshake");
3318        server.abort();
3319        let _ = server.await;
3320    }
3321
3322    /// The published file names the local endpoint the daemon actually serves,
3323    /// and a client that trusts the file completes the handshake over it.
3324    #[cfg(unix)]
3325    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3326    async fn published_connection_file_names_the_served_local_endpoint() {
3327        let (_dir, path) = temp_connection_file_path("local-endpoint");
3328        // The socket goes in the system temporary directory rather than inside
3329        // the harness's own directory, whose long lockdown-sandbox path leaves
3330        // no room inside the Unix socket path limit.
3331        let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
3332        let config =
3333            BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
3334        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3335        let endpoint = connection_file::read_with_local_endpoint(&path)
3336            .unwrap()
3337            .local_endpoint
3338            .expect("the daemon serves a local endpoint, so the file names it");
3339        assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
3340        let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
3341            panic!("a Unix daemon publishes a socket path");
3342        };
3343        assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
3344        assert_eq!(
3345            fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
3346            0o600
3347        );
3348
3349        let router = Arc::new(Router::with_default_self_handler());
3350        let auth = auth_for(&bound.connection_info);
3351        let server = tokio::spawn(serve_local_listener(
3352            bound.local_listener.expect("the local listener is bound"),
3353            router,
3354            auth,
3355        ));
3356        let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
3357        authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
3358            .await
3359            .expect("a socket client completes the same handshake as a TCP client");
3360
3361        server.abort();
3362        drop(bound.listeners);
3363    }
3364
3365    /// A configured directory that is not a directory refuses at bind rather
3366    /// than publishing an endpoint nothing serves.
3367    #[cfg(unix)]
3368    #[test]
3369    fn a_socket_directory_that_is_a_file_refuses() {
3370        let dir = ScratchDir::new("local-listener-file-as-dir");
3371        let file = dir.join("not-a-directory");
3372        fs::write(&file, b"x").unwrap();
3373        let error = local_listener::bind(
3374            &LocalListenerConfig::in_socket_dir(file),
3375            &dir.join("connection.json"),
3376        )
3377        .expect_err("a path that is not a directory cannot hold the socket");
3378        assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
3379    }
3380
3381    /// The socket sits directly in the configured directory under one fixed
3382    /// name, and a directory that would put the path over the platform limit is
3383    /// refused rather than shortened around.
3384    #[cfg(unix)]
3385    #[test]
3386    fn the_socket_path_stays_within_the_unix_limit() {
3387        let run_dir = PathBuf::from("/tmp/short-run");
3388        assert!(
3389            local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
3390            "no configured directory means no socket"
3391        );
3392        let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
3393            .expect("a configured directory gets a socket");
3394        assert_eq!(socket, run_dir.join("subc.sock"));
3395        assert_eq!(
3396            socket,
3397            local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
3398            "the name does not come from the connection file"
3399        );
3400
3401        let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
3402        let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
3403            .expect("a configured directory gets a socket");
3404        assert!(
3405            long.as_os_str().len() < 104,
3406            "{} is {} bytes",
3407            long.display(),
3408            long.as_os_str().len()
3409        );
3410    }
3411}