Skip to main content

subc_daemon/
bootstrap.rs

1use std::{
2    collections::BTreeMap,
3    env,
4    error::Error,
5    ffi::OsString,
6    fmt, fs, io,
7    net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
8    path::{Path, PathBuf},
9    process,
10    time::Duration,
11};
12
13use fs4::{FileExt, TryLockError};
14use subc_protocol::PROTOCOL_VERSION;
15pub use subc_transport::user_connection_token;
16use subc_transport::{
17    authenticate_client, connection_file, generate_daemon_id, generate_key,
18    write_atomic_with_local_endpoint, AuthError, ConnectionFileError, ConnectionInfo, Endpoint,
19    LocalEndpoint, SCHEMA_VERSION,
20};
21use tokio::{
22    net::{TcpListener, TcpStream},
23    task::{JoinError, JoinHandle},
24    time::{sleep, timeout},
25};
26use tracing::{error, info, warn};
27
28use crate::{
29    daemon_config::{self, ConfiguredModule, DaemonConfigError},
30    local_listener::{self, BoundLocalListener, LocalListenerConfig},
31    server::{serve_listeners, serve_local_listener, ServerAuth, ServerError},
32    supervise::HealthConfig,
33    ConnectedClients, ControlHandler, DaemonSelfWatchdog, DaemonSelfWatchdogConfig,
34    ForwardingTable, Registry, RestartPolicy, Router, Supervisor, SupervisorHandle,
35    SupervisorProcessLiveness,
36};
37use std::sync::Arc;
38
39#[cfg(windows)]
40mod windows_connection_copy;
41
42pub const DEFAULT_SUBC_PORT: u16 = 8757;
43pub const SUBC_PORT_ENV: &str = "SUBC_PORT";
44use subc_transport::CONNECTION_FILE_NAME;
45const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
46const CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
47const PROBE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
48const START_LOCK_RETRIES: usize = 40;
49const START_LOCK_RETRY_DELAY: Duration = Duration::from_millis(25);
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum ConnectionFileSource {
53    XdgRuntimeDir,
54    TempDirFallback,
55    Explicit,
56}
57
58impl ConnectionFileSource {
59    fn reason(self) -> &'static str {
60        match self {
61            Self::XdgRuntimeDir => "XDG_RUNTIME_DIR set and non-empty",
62            Self::TempDirFallback => "XDG_RUNTIME_DIR unset or empty",
63            Self::Explicit => "configured path",
64        }
65    }
66}
67
68impl fmt::Display for ConnectionFileSource {
69    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
70        formatter.write_str(match self {
71            Self::XdgRuntimeDir => "xdg_runtime_dir",
72            Self::TempDirFallback => "temp_dir_fallback",
73            Self::Explicit => "explicit",
74        })
75    }
76}
77
78/// Runtime bootstrap configuration. Production uses the default fixed port and
79/// optional daemon-config override; tests pass port 0 to let the OS assign a free
80/// loopback port and discover it from the connection file.
81#[derive(Debug, Clone, Default)]
82struct AdmissionFactsConfig {
83    carrier_module_id: Option<String>,
84    targets: Option<Vec<String>>,
85}
86
87/// Controls where module cgroups are prepared.
88///
89/// In-process daemons default to [`Self::Disabled`] so they never derive a
90/// production location from the host process. The shipped daemon explicitly uses
91/// [`Self::Current`]. The `ck-subc` binary accepts
92/// `SUBC_CGROUP_PLACEMENT=disabled` for isolated test processes; an unset
93/// variable retains `Current`, and every other value is rejected at startup.
94/// Tests that exercise placement can own a [`Self::Root`].
95#[derive(Debug, Clone, Default, PartialEq, Eq)]
96pub enum CgroupPlacementConfig {
97    #[default]
98    Disabled,
99    Current,
100    Root(PathBuf),
101}
102
103#[derive(Debug, Clone)]
104pub struct BootstrapConfig {
105    #[cfg(feature = "test-support")]
106    shutdown_intent_window: Duration,
107    pub operator_queue_wait: Duration,
108    pub operator_timeout: Duration,
109    pub operator_stuck_grace: Duration,
110    #[cfg(feature = "test-support")]
111    pub operator_script: Option<PathBuf>,
112    #[cfg(feature = "test-support")]
113    pub operator_events: Option<PathBuf>,
114    pub connection_file_path: PathBuf,
115    pub port: u16,
116    pub daemon_ver: String,
117    configured_modules: Vec<ConfiguredModule>,
118    storage_config: Option<daemon_config::StorageConfig>,
119    admission_facts: AdmissionFactsConfig,
120    /// Module ids whose scopes may carry the attributes that grant authority.
121    scope_authority_owners: Vec<String>,
122    federated_scope_owner: String,
123    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
124    activity_config: daemon_config::ActivityConfig,
125    daemon_config_path: Option<PathBuf>,
126    configured_port: Option<u16>,
127    /// Daemon-wide route.bind relay budget in milliseconds (the fallback for
128    /// any module without a per-module override). `None` = built-in default
129    /// (12s — see `control::DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`).
130    route_bind_relay_default_ms: Option<u64>,
131    reserved_capabilities: BTreeMap<String, String>,
132    watchdog_config: DaemonSelfWatchdogConfig,
133    connection_file_source: ConnectionFileSource,
134    #[cfg(all(test, windows))]
135    profile_override: Option<PathBuf>,
136    /// Where module cgroups are prepared. Disabled unless a caller explicitly
137    /// opts in, because `Current` derives a host location from `/proc/self/cgroup`.
138    cgroup_placement: CgroupPlacementConfig,
139    /// Directory the supervisor writes per-module stdout/stderr capture files
140    /// into. `None` disables capture; the shipped binary supplies its real run
141    /// directory explicitly.
142    capture_logs_dir: Option<PathBuf>,
143    /// File the supervisor appends every module's terminal exits to, so exit
144    /// history survives a daemon restart. `None` keeps terminal history in
145    /// memory only (each module's ring). Absent by default for the same reason
146    /// as `capture_logs_dir`: an in-process daemon booted by a test must not
147    /// append its exits to the operator's real `terminals.jsonl`, where they
148    /// would show up in `ck module terminals`. The shipped binary supplies
149    /// `<run dir>/terminals.jsonl` explicitly.
150    terminal_journal_path: Option<PathBuf>,
151    /// Where the machine id is read from, or minted into when absent. `None`
152    /// serves no machine id. Absent by default for the same reason as
153    /// `capture_logs_dir`: an in-process daemon booted by a test must never
154    /// derive the operator's real data home and mint into it. The shipped binary
155    /// supplies `<data home>/cortexkit/machine-id` explicitly.
156    machine_id_path: Option<PathBuf>,
157
158    /// Where this daemon's same-user local listener lives (a Unix-domain socket
159    /// on macOS and Linux; nothing on Windows, where the pipe name is minted per
160    /// start). The default serves no local endpoint, so an in-process daemon
161    /// never touches the operator's run directory. The shipped binary names its
162    /// run directory; [`BootstrapConfig::with_local_listener_socket_dir`] names
163    /// any other, and an in-process daemon that wants the endpoint names one.
164    local_listener: LocalListenerConfig,
165    /// The live-children record: every supervised process this daemon has
166    /// running, kept so the next daemon can end the ones a crash left behind.
167    /// At startup, before any module is spawned, the previous daemon's record
168    /// here is swept. `None` keeps no record and sweeps nothing, for the same
169    /// reason as `capture_logs_dir`: an in-process daemon booted by a test
170    /// must never signal processes listed in the operator's real run
171    /// directory. The shipped binary supplies `<run dir>/live-children.json`.
172    live_children_path: Option<PathBuf>,
173    privacy_trampoline: Option<PathBuf>,
174}
175
176impl BootstrapConfig {
177    /// Shorten restart intent validity in daemon integration fixtures.
178    #[cfg(feature = "test-support")]
179    pub fn with_shutdown_intent_window(mut self, window: Duration) -> Self {
180        self.shutdown_intent_window = window;
181        self
182    }
183
184    pub fn new(connection_file_path: impl Into<PathBuf>, port: u16) -> Self {
185        Self {
186            #[cfg(feature = "test-support")]
187            shutdown_intent_window: Duration::from_secs(120),
188            connection_file_path: connection_file_path.into(),
189            operator_queue_wait: Duration::from_secs(150),
190            operator_timeout: Duration::from_secs(120),
191            operator_stuck_grace: Duration::from_secs(10),
192            #[cfg(feature = "test-support")]
193            operator_script: None,
194            #[cfg(feature = "test-support")]
195            operator_events: None,
196            port,
197            daemon_ver: DAEMON_VERSION.to_owned(),
198            configured_modules: Vec::new(),
199            storage_config: None,
200            admission_facts: AdmissionFactsConfig::default(),
201            scope_authority_owners: daemon_config::default_scope_authority_owners(),
202            federated_scope_owner: daemon_config::default_federated_scope_owner(),
203            privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt::default(),
204            activity_config: daemon_config::ActivityConfig::default(),
205            daemon_config_path: None,
206            configured_port: None,
207            route_bind_relay_default_ms: None,
208            reserved_capabilities: BTreeMap::new(),
209            watchdog_config: DaemonSelfWatchdogConfig::default(),
210            connection_file_source: ConnectionFileSource::Explicit,
211            #[cfg(all(test, windows))]
212            profile_override: None,
213            cgroup_placement: CgroupPlacementConfig::default(),
214            capture_logs_dir: None,
215            terminal_journal_path: None,
216            machine_id_path: None,
217            local_listener: LocalListenerConfig { socket_dir: None },
218            live_children_path: None,
219            privacy_trampoline: None,
220        }
221    }
222
223    /// Keep the local listener's socket in `dir`. Embedding daemons and tests
224    /// pass a path inside their own fixture tree.
225    pub fn with_local_listener_socket_dir(mut self, dir: impl Into<PathBuf>) -> Self {
226        self.local_listener = LocalListenerConfig::in_socket_dir(dir);
227        self
228    }
229
230    /// Set where the local listener lives, as [`LocalListenerConfig`] describes.
231    pub fn with_local_listener(mut self, config: LocalListenerConfig) -> Self {
232        self.local_listener = config;
233        self
234    }
235
236    /// Keep the live-children record at `path`, and at startup end the
237    /// processes a previous daemon recorded there that are still running.
238    /// Embedding daemons and tests pass a path inside their own fixture tree.
239    pub fn with_live_children_record(mut self, path: impl Into<PathBuf>) -> Self {
240        self.live_children_path = Some(path.into());
241        self
242    }
243
244    /// macOS module launches require an explicit executable implementing the
245    /// subc-os trampoline entry point. Other systems ignore this setting.
246    pub fn with_privacy_trampoline(mut self, path: impl Into<PathBuf>) -> Self {
247        self.privacy_trampoline = Some(path.into());
248        self
249    }
250
251    /// Serve the machine id stored at `path`, minting it there at startup when
252    /// the file is absent. Embedding daemons and tests pass a path inside their
253    /// own fixture tree.
254    pub fn with_machine_id_path(mut self, path: impl Into<PathBuf>) -> Self {
255        self.machine_id_path = Some(path.into());
256        self
257    }
258
259    /// Selects module cgroup placement. The default is disabled.
260    pub fn with_cgroup_placement(mut self, placement: CgroupPlacementConfig) -> Self {
261        self.cgroup_placement = placement;
262        self
263    }
264
265    /// Redirects per-module stdout/stderr capture files out of the real run
266    /// directory. Tests that start an in-process daemon must call this with a
267    /// path inside their fixture tree.
268    pub fn with_capture_logs_dir(mut self, dir: impl Into<PathBuf>) -> Self {
269        self.capture_logs_dir = Some(dir.into());
270        self
271    }
272
273    /// Redirects the daemon-private journal, allowing embedded daemons and tests
274    /// to keep their observations out of the operator's live run directory.
275    pub fn with_terminal_journal_path(mut self, path: impl Into<PathBuf>) -> Self {
276        self.terminal_journal_path = Some(path.into());
277        self
278    }
279
280    pub fn from_env() -> Result<Self, BootstrapError> {
281        Self::from_env_with_daemon_config_path(daemon_config::default_config_path())
282    }
283
284    /// The SHIPPED BINARY's config, which is the only caller that should capture
285    /// child output into the operator's real run directory.
286    ///
287    /// Kept separate from `from_env` deliberately: see `capture_logs_dir` and
288    /// `terminal_journal_path` on this struct for why an absent value must mean
289    /// NO CAPTURE and NO JOURNAL rather than the operator's real run directory.
290    /// A run directory that cannot be resolved (a relative data home) refuses
291    /// startup instead of landing under the working directory.
292    pub fn from_env_for_daemon_binary() -> Result<Self, BootstrapError> {
293        let run_dir = daemon_config::daemon_run_dir().map_err(BootstrapError::RunDir)?;
294        let machine_id_path =
295            crate::machine_id::default_machine_id_path().map_err(BootstrapError::MachineId)?;
296        let config = Self::from_env()?
297            .with_local_listener(LocalListenerConfig::in_socket_dir(&run_dir))
298            .with_capture_logs_dir(run_dir.join("logs"))
299            .with_terminal_journal_path(run_dir.join("terminals.jsonl"))
300            .with_live_children_record(crate::live_children::record_path(&run_dir))
301            .with_machine_id_path(machine_id_path);
302        #[cfg(feature = "test-support")]
303        let config = {
304            let mut config = config;
305            config.operator_script =
306                std::env::var_os("SUBC_TEST_OPERATOR_SCRIPT").map(PathBuf::from);
307            config.operator_events =
308                std::env::var_os("SUBC_TEST_OPERATOR_EVENTS").map(PathBuf::from);
309            for (name, value) in [
310                (
311                    "SUBC_TEST_OPERATOR_QUEUE_WAIT_MS",
312                    &mut config.operator_queue_wait,
313                ),
314                (
315                    "SUBC_TEST_OPERATOR_TIMEOUT_MS",
316                    &mut config.operator_timeout,
317                ),
318                (
319                    "SUBC_TEST_OPERATOR_STUCK_MS",
320                    &mut config.operator_stuck_grace,
321                ),
322            ] {
323                if let Ok(ms) = std::env::var(name) {
324                    if let Ok(ms) = ms.parse::<u64>() {
325                        *value = Duration::from_millis(ms);
326                    }
327                }
328            }
329            config
330        };
331        Ok(config)
332    }
333
334    pub fn from_env_with_daemon_config_path(
335        daemon_config_path: impl AsRef<Path>,
336    ) -> Result<Self, BootstrapError> {
337        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
338        let (daemon_config, activity_config) =
339            daemon_config::load_with_activity(&daemon_config_path)
340                .map_err(BootstrapError::DaemonConfig)?
341                .map(|(config, activity)| (Some(config), activity))
342                .unwrap_or_default();
343        let config_port = daemon_config.as_ref().and_then(|config| config.port);
344        let storage_config = daemon_config
345            .as_ref()
346            .and_then(|config| config.storage.clone());
347        let admission_facts_carrier_module_id = daemon_config
348            .as_ref()
349            .and_then(|config| config.admission_facts_carrier_module_id.clone());
350        let admission_facts_targets = daemon_config
351            .as_ref()
352            .and_then(|config| config.admission_facts_targets.clone());
353        let scope_authority_owners = daemon_config
354            .as_ref()
355            .map(|config| config.scope_authority_owners.clone())
356            .unwrap_or_else(daemon_config::default_scope_authority_owners);
357        let federated_scope_owner = daemon_config
358            .as_ref()
359            .map(|config| config.federated_scope_owner.clone())
360            .unwrap_or_else(daemon_config::default_federated_scope_owner);
361        let privileged_operator_prompt = daemon_config
362            .as_ref()
363            .map(|config| config.privileged_operator_prompt)
364            .unwrap_or_default();
365        let route_bind_relay_default_ms = daemon_config
366            .as_ref()
367            .and_then(|config| config.route_bind_relay_timeout_ms);
368        let reserved_capabilities = daemon_config
369            .as_ref()
370            .map(|config| config.reserved_capabilities.clone())
371            .unwrap_or_default();
372        let configured_modules = daemon_config
373            .map(|config| config.modules)
374            .unwrap_or_default();
375
376        let port = match env::var(SUBC_PORT_ENV) {
377            Ok(raw) if !raw.trim().is_empty() => {
378                let port = raw
379                    .parse::<u16>()
380                    .map_err(|source| BootstrapError::InvalidPort { raw, source })?;
381                if let Some(config_port) = config_port {
382                    info!(
383                        env = SUBC_PORT_ENV,
384                        env_port = port,
385                        config_port,
386                        "SUBC_PORT overrides daemon config port"
387                    );
388                }
389                port
390            }
391            Ok(_) | Err(_) => config_port.unwrap_or(DEFAULT_SUBC_PORT),
392        };
393
394        let (connection_file_path, connection_file_source) =
395            connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR"));
396        Ok(Self::new(connection_file_path, port)
397            .with_configured_modules(configured_modules)
398            .with_storage_config(storage_config)
399            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
400            .with_scope_authority_owners(scope_authority_owners)
401            .with_federated_scope_owner(federated_scope_owner)
402            .with_privileged_operator_prompt(privileged_operator_prompt)
403            .with_activity_config(activity_config)
404            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
405            .with_reserved_capabilities(reserved_capabilities)
406            .with_daemon_config_source(daemon_config_path, config_port)
407            .with_connection_file_source(connection_file_source))
408    }
409
410    pub fn with_daemon_config_path(
411        self,
412        daemon_config_path: impl AsRef<Path>,
413    ) -> Result<Self, BootstrapError> {
414        let daemon_config_path = daemon_config_path.as_ref().to_path_buf();
415        let (daemon_config, activity_config) =
416            daemon_config::load_with_activity(&daemon_config_path)
417                .map_err(BootstrapError::DaemonConfig)?
418                .map(|(config, activity)| (Some(config), activity))
419                .unwrap_or_default();
420        let configured_port = daemon_config.as_ref().and_then(|config| config.port);
421        let storage_config = daemon_config
422            .as_ref()
423            .and_then(|config| config.storage.clone());
424        let admission_facts_carrier_module_id = daemon_config
425            .as_ref()
426            .and_then(|config| config.admission_facts_carrier_module_id.clone());
427        let admission_facts_targets = daemon_config
428            .as_ref()
429            .and_then(|config| config.admission_facts_targets.clone());
430        let scope_authority_owners = daemon_config
431            .as_ref()
432            .map(|config| config.scope_authority_owners.clone())
433            .unwrap_or_else(daemon_config::default_scope_authority_owners);
434        let federated_scope_owner = daemon_config
435            .as_ref()
436            .map(|config| config.federated_scope_owner.clone())
437            .unwrap_or_else(daemon_config::default_federated_scope_owner);
438        let privileged_operator_prompt = daemon_config
439            .as_ref()
440            .map(|config| config.privileged_operator_prompt)
441            .unwrap_or_default();
442        let route_bind_relay_default_ms = daemon_config
443            .as_ref()
444            .and_then(|config| config.route_bind_relay_timeout_ms);
445        let reserved_capabilities = daemon_config
446            .as_ref()
447            .map(|config| config.reserved_capabilities.clone())
448            .unwrap_or_default();
449        let configured_modules = daemon_config
450            .map(|config| config.modules)
451            .unwrap_or_default();
452        Ok(self
453            .with_configured_modules(configured_modules)
454            .with_storage_config(storage_config)
455            .with_admission_facts_config(admission_facts_carrier_module_id, admission_facts_targets)
456            .with_scope_authority_owners(scope_authority_owners)
457            .with_federated_scope_owner(federated_scope_owner)
458            .with_privileged_operator_prompt(privileged_operator_prompt)
459            .with_activity_config(activity_config)
460            .with_route_bind_relay_default_ms(route_bind_relay_default_ms)
461            .with_reserved_capabilities(reserved_capabilities)
462            .with_daemon_config_source(daemon_config_path, configured_port))
463    }
464
465    pub fn with_configured_modules(
466        mut self,
467        modules: impl IntoIterator<Item = ConfiguredModule>,
468    ) -> Self {
469        self.configured_modules = modules.into_iter().collect();
470        self.configured_modules
471            .sort_by(|left, right| left.module_id.cmp(&right.module_id));
472        self
473    }
474
475    pub fn with_storage_config(
476        mut self,
477        storage_config: Option<daemon_config::StorageConfig>,
478    ) -> Self {
479        self.storage_config = storage_config;
480        self
481    }
482
483    pub fn with_admission_facts_config(
484        mut self,
485        carrier_module_id: Option<String>,
486        targets: Option<Vec<String>>,
487    ) -> Self {
488        self.admission_facts = AdmissionFactsConfig {
489            carrier_module_id,
490            targets,
491        };
492        self
493    }
494
495    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
496    /// Absent from a config, this is `daemon_config::default_scope_authority_owners`.
497    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
498        self.scope_authority_owners = owners;
499        self
500    }
501
502    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
503        self.federated_scope_owner = owner;
504        self
505    }
506
507    /// Choose the privileged-operation policy for the daemon's entire lifetime.
508    pub fn with_privileged_operator_prompt(
509        mut self,
510        mode: daemon_config::PrivilegedOperatorPrompt,
511    ) -> Self {
512        self.privileged_operator_prompt = mode;
513        self
514    }
515
516    pub fn with_activity_config(mut self, config: daemon_config::ActivityConfig) -> Self {
517        self.activity_config = config;
518        self
519    }
520
521    /// Set the daemon-wide route.bind relay default (the fallback for any
522    /// module without a per-module override). `None` preserves the built-in
523    /// default (12s). `serve_bound_daemon` reads this at startup and threads
524    /// it into the control handler's daemon-wide field.
525    pub fn with_route_bind_relay_default_ms(mut self, ms: Option<u64>) -> Self {
526        self.route_bind_relay_default_ms = ms;
527        self
528    }
529
530    pub fn with_reserved_capabilities(
531        mut self,
532        reserved_capabilities: BTreeMap<String, String>,
533    ) -> Self {
534        self.reserved_capabilities = reserved_capabilities;
535        self
536    }
537
538    fn with_daemon_config_source(
539        mut self,
540        daemon_config_path: PathBuf,
541        configured_port: Option<u16>,
542    ) -> Self {
543        self.daemon_config_path = Some(daemon_config_path);
544        self.configured_port = configured_port;
545        self
546    }
547
548    fn with_connection_file_source(mut self, source: ConnectionFileSource) -> Self {
549        self.connection_file_source = source;
550        self
551    }
552
553    pub fn with_watchdog_config(mut self, watchdog_config: DaemonSelfWatchdogConfig) -> Self {
554        self.watchdog_config = watchdog_config;
555        self
556    }
557}
558
559/// Result of singleton discovery.
560// Built once per daemon start and immediately matched, so the size gap between
561// the variants costs nothing worth a box on the public shape.
562#[allow(clippy::large_enum_variant)]
563#[derive(Debug)]
564pub enum Outcome {
565    /// A live daemon authenticated from the connection file; this invocation should exit 0.
566    AlreadyRunning,
567    /// This process won the singleton race, owns bound loopback listener(s), and
568    /// has published a fresh connection file.
569    Bound(BoundDaemon),
570}
571
572#[derive(Debug)]
573pub struct BoundDaemon {
574    pub listeners: Vec<TcpListener>,
575    /// The same-user local listener, bound and not yet served. `None` when
576    /// binding it failed, in which case `local_endpoint` is `None` and
577    /// `local_endpoint_unavailable` says why.
578    pub local_listener: Option<BoundLocalListener>,
579    /// Why the daemon serves no local endpoint, when binding it failed. Reported
580    /// by `server.describe` so an operator can see it.
581    pub local_endpoint_unavailable: Option<String>,
582    /// What the connection file publishes under `local_endpoint`, so a client can
583    /// find the local listener without knowing how it was bound.
584    pub local_endpoint: Option<LocalEndpoint>,
585    pub connection_info: ConnectionInfo,
586    pub connection_file_path: PathBuf,
587    pub connection_file_source: ConnectionFileSource,
588    #[cfg(windows)]
589    connection_file_copy: Option<PathBuf>,
590    /// The machine id this daemon serves, established before any connection is
591    /// accepted. `None` when the config named no machine id path.
592    pub machine_id: Option<crate::machine_id::MachineId>,
593    /// Ownership of the run directory, held until the daemon exits so no other
594    /// daemon sweeps or writes this one's run state. `None` when the config
595    /// keeps no live-children record, and so has no run directory to own.
596    run_dir_lock: Option<crate::run_dir_lock::RunDirLock>,
597    /// The singleton start lock, kept when binding deferred publishing the
598    /// connection file. It stays held until the daemon has verified it is serving
599    /// and written the connection file, so a second daemon starting meanwhile
600    /// cannot claim the singleton too. `None` when the connection
601    /// file was already written at bind time.
602    publication_lock: Option<StartLock>,
603}
604
605/// Resolve subc's per-user TCP connection-file path.
606///
607/// `$XDG_RUNTIME_DIR/subc-connection.json` is preferred because the runtime
608/// directory is already per-user on Unix desktops. Without it, subc falls back
609/// to the system temp dir with a per-user token in the filename so different OS
610/// users do not collide on shared temp directories.
611pub fn connection_file_path() -> PathBuf {
612    connection_file_path_with_source(non_empty_os_var("XDG_RUNTIME_DIR")).0
613}
614
615fn connection_file_path_with_source(
616    runtime_dir: Option<OsString>,
617) -> (PathBuf, ConnectionFileSource) {
618    if let Some(runtime_dir) = runtime_dir.filter(|value| !value.is_empty()) {
619        return (
620            PathBuf::from(runtime_dir).join(CONNECTION_FILE_NAME),
621            ConnectionFileSource::XdgRuntimeDir,
622        );
623    }
624
625    (
626        env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token())),
627        ConnectionFileSource::TempDirFallback,
628    )
629}
630
631/// Resolve, claim, and serve the per-user daemon singleton.
632///
633/// A second invocation is successful: if a live daemon authenticates from the
634/// existing connection file, this returns `Ok(())` after logging and the caller
635/// exits with status 0.
636pub async fn run() -> Result<(), BootstrapError> {
637    // `run` is a binary entry point, so it opts into the production cgroup and
638    // child-log locations. In-process callers keep both features disabled by default.
639    run_with_config(
640        BootstrapConfig::from_env_for_daemon_binary()?
641            .with_cgroup_placement(CgroupPlacementConfig::Current),
642    )
643    .await
644}
645
646/// Serve a daemon from an explicit config. This is the entry point the twelve
647/// sibling repos use to boot an in-process daemon in their integration tests.
648///
649/// # THIS INSTALLS NO TRACING SUBSCRIBER, SO THE DAEMON IS SILENT BY DEFAULT
650///
651/// The daemon's own diagnostics go through `tracing`, and `tracing` DISCARDS
652/// every event when no subscriber is installed. The shipped binary installs one
653/// in `main` (`init_tracing`); this function deliberately does not, because a
654/// library that installs a global subscriber fights with whatever the host
655/// process already set up.
656///
657/// The consequence for a test harness is not "less verbose": it is that the
658/// daemon has NOTHING TO SAY about any failure, and a missing instrument reads
659/// exactly like a clean one. PLEX found this on 2026-09-18 while trying to
660/// capture daemon logs beside an intermittent bind failure, and discovered the
661/// daemon had been silent in every conformance run that repo had ever done --
662/// so the one client-side error string was all the evidence that could exist,
663/// and they had spent a real investigation on a failure whose second source was
664/// never being recorded.
665///
666/// Install one in the harness before calling this, and assert it did something
667/// (they measured 236 daemon lines with the subscriber installed, 0 with the
668/// call commented out) -- otherwise the fix is itself unverified.
669pub async fn run_with_config(config: BootstrapConfig) -> Result<(), BootstrapError> {
670    #[cfg(feature = "test-support")]
671    let shutdown_intent_window = config.shutdown_intent_window;
672    let forwarding = Arc::new(ForwardingTable::default());
673    forwarding.operator_confirms().configure_bootstrap(&config);
674    let configured_modules = config.configured_modules.clone();
675    let storage_config = config.storage_config.clone();
676    let admission_facts = config.admission_facts.clone();
677    let scope_authority_owners = config.scope_authority_owners.clone();
678    let federated_scope_owner = config.federated_scope_owner.clone();
679    let privileged_operator_prompt = config.privileged_operator_prompt;
680    let activity_config = config.activity_config;
681    let daemon_config_path = config.daemon_config_path.clone();
682    let configured_port = config.configured_port;
683    let route_bind_relay_default_ms = config.route_bind_relay_default_ms;
684    let reserved_capabilities = config.reserved_capabilities.clone();
685    let watchdog_config = config.watchdog_config.clone();
686    let cgroup_placement_config = config.cgroup_placement.clone();
687    let capture_logs_dir = config.capture_logs_dir.clone();
688    let terminal_journal_path = config.terminal_journal_path.clone();
689    let live_children_path = config.live_children_path.clone();
690    let privacy_trampoline = config.privacy_trampoline.clone();
691    match ensure_singleton_inner(config, false).await? {
692        Outcome::AlreadyRunning => {
693            info!("subc daemon already running");
694            Ok(())
695        }
696        Outcome::Bound(bound) => {
697            #[cfg(target_os = "linux")]
698            let cgroup_placement = prepare_cgroup_placement(&cgroup_placement_config);
699            #[cfg(not(target_os = "linux"))]
700            let _ = cgroup_placement_config;
701            serve_bound_daemon(
702                bound,
703                forwarding,
704                configured_modules,
705                storage_config,
706                admission_facts,
707                scope_authority_owners,
708                federated_scope_owner,
709                privileged_operator_prompt,
710                activity_config,
711                daemon_config_path,
712                configured_port,
713                route_bind_relay_default_ms,
714                reserved_capabilities,
715                watchdog_config,
716                capture_logs_dir,
717                terminal_journal_path,
718                live_children_path,
719                privacy_trampoline,
720                #[cfg(feature = "test-support")]
721                shutdown_intent_window,
722                #[cfg(target_os = "linux")]
723                cgroup_placement,
724            )
725            .await
726        }
727    }
728}
729
730#[cfg(target_os = "linux")]
731fn prepare_cgroup_placement(config: &CgroupPlacementConfig) -> Option<subc_cgroup::Placement> {
732    let result = match config {
733        CgroupPlacementConfig::Disabled => return None,
734        CgroupPlacementConfig::Current => subc_cgroup::prepare_current(),
735        CgroupPlacementConfig::Root(root) => subc_cgroup::prepare_at(root),
736    };
737
738    match result {
739        Ok(Some(placement)) => Some(placement),
740        Ok(None) => {
741            warn!(
742                placement = ?config,
743                "module cgroup placement is disabled: configured cgroup root is not delegated"
744            );
745            None
746        }
747        Err(error) => {
748            warn!(
749                placement = ?config,
750                error = %error,
751                "module cgroup placement is disabled by an unexpected cgroup probe error"
752            );
753            None
754        }
755    }
756}
757
758/// Target soft limit for open file descriptors, applied to the daemon before any
759/// module is spawned so children inherit it. Multi-root modules (one process
760/// aggregating every project root's sqlite stores, index caches, watchers, and
761/// LSP pipes) trivially exceed the macOS default soft limit of 256; a launchd
762/// user agent does not pass login-shell ulimits through, so the raise must
763/// happen in-process.
764#[cfg(unix)]
765const NOFILE_TARGET: u64 = 65536;
766
767/// Raise RLIMIT_NOFILE to `NOFILE_TARGET` (clamped to the hard limit).
768/// Best-effort: failure is logged and never fatal, since the daemon can run
769/// under the inherited limit — modules with few roots just have less headroom.
770#[cfg(unix)]
771fn raise_nofile_limit() {
772    #[cfg(target_os = "macos")]
773    let ceiling = std::process::Command::new("/usr/sbin/sysctl")
774        .args(["-n", "kern.maxfilesperproc"])
775        .output()
776        .ok()
777        .filter(|output| output.status.success())
778        .and_then(|output| String::from_utf8(output.stdout).ok())
779        .and_then(|value| value.trim().parse::<u64>().ok())
780        .filter(|value| *value > 0);
781    #[cfg(not(target_os = "macos"))]
782    let ceiling = None;
783    raise_nofile_limit_with_ceiling(ceiling);
784}
785
786#[cfg(unix)]
787fn raise_nofile_limit_with_ceiling(kernel_ceiling: Option<u64>) {
788    match rlimit::Resource::NOFILE.get() {
789        Ok((soft, hard)) => {
790            // On XNU an infinite hard limit does not remove the separate
791            // per-process kernel ceiling. Exceeding it makes setrlimit fail
792            // without raising the inherited (often 256) soft limit at all.
793            let target = NOFILE_TARGET
794                .min(hard)
795                .min(kernel_ceiling.unwrap_or(u64::MAX));
796            if soft >= target {
797                return;
798            }
799            match rlimit::Resource::NOFILE.set(target, hard) {
800                Ok(()) => info!(
801                    previous_soft = soft,
802                    new_soft = target,
803                    hard,
804                    "raised open-file soft limit for daemon and module children"
805                ),
806                Err(err) => warn!(
807                    soft,
808                    hard,
809                    error = %err,
810                    "could not raise open-file soft limit; multi-root modules may exhaust descriptors"
811                ),
812            }
813        }
814        Err(err) => warn!(error = %err, "could not read open-file limit"),
815    }
816}
817
818/// CRT stdio-stream target on Windows (the `_setmaxstdio` maximum). Win32
819/// HANDLEs — what Rust `File`, tokio sockets, and SQLite's Win32 VFS actually
820/// consume — have a per-process quota in the millions and need no raise; the
821/// C-runtime stream table (default 512) is the only low ceiling, and it is
822/// per-process rather than inherited, so supervised modules linking the CRT
823/// must raise their own. Raising it here covers the daemon itself.
824#[cfg(windows)]
825fn raise_nofile_limit() {
826    const MAXSTDIO_TARGET: u32 = 8192;
827    let current = rlimit::getmaxstdio();
828    if current >= MAXSTDIO_TARGET {
829        return;
830    }
831    match rlimit::setmaxstdio(MAXSTDIO_TARGET) {
832        Ok(new_max) => info!(
833            previous = current,
834            new_max, "raised CRT stdio-stream limit for daemon"
835        ),
836        Err(err) => warn!(
837            current,
838            error = %err,
839            "could not raise CRT stdio-stream limit"
840        ),
841    }
842}
843
844#[cfg(not(any(unix, windows)))]
845fn raise_nofile_limit() {}
846
847pub async fn run_with_daemon_config_path(
848    config: BootstrapConfig,
849    daemon_config_path: impl AsRef<Path>,
850) -> Result<(), BootstrapError> {
851    run_with_config(config.with_daemon_config_path(daemon_config_path)?).await
852}
853
854#[allow(clippy::too_many_arguments)]
855async fn serve_bound_daemon(
856    bound: BoundDaemon,
857    forwarding: Arc<ForwardingTable>,
858    configured_modules: Vec<ConfiguredModule>,
859    storage_config: Option<daemon_config::StorageConfig>,
860    admission_facts: AdmissionFactsConfig,
861    scope_authority_owners: Vec<String>,
862    federated_scope_owner: String,
863    privileged_operator_prompt: daemon_config::PrivilegedOperatorPrompt,
864    activity_config: daemon_config::ActivityConfig,
865    daemon_config_path: Option<PathBuf>,
866    configured_port: Option<u16>,
867    route_bind_relay_default_ms: Option<u64>,
868    reserved_capabilities: BTreeMap<String, String>,
869    watchdog_config: DaemonSelfWatchdogConfig,
870    capture_logs_dir: Option<PathBuf>,
871    terminal_journal_path: Option<PathBuf>,
872    live_children_path: Option<PathBuf>,
873    privacy_trampoline: Option<PathBuf>,
874    #[cfg(feature = "test-support")] shutdown_intent_window: Duration,
875    #[cfg(target_os = "linux")] cgroup_placement: Option<subc_cgroup::Placement>,
876) -> Result<(), BootstrapError> {
877    #[cfg(unix)]
878    let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
879        .map_err(BootstrapError::Signal)?;
880    // Windows has no SIGTERM. Ctrl-C is a different event, not an equivalent
881    // service-stop contract, so this shutdown handler is intentionally Unix-only.
882    raise_nofile_limit();
883    #[cfg(windows)]
884    if let Err(error) = subc_jobobject::ensure_root_job() {
885        // Keep the control plane available. The root-job API caches setup
886        // failures, so later module spawns refuse instead of running unprotected.
887        warn!(%error, "module launches disabled: Windows root job setup failed");
888    }
889
890    info!(
891        connection_file = %bound.connection_file_path.display(),
892        connection_file_source = %bound.connection_file_source,
893        connection_file_source_reason = bound.connection_file_source.reason(),
894        endpoints = ?bound.connection_info.endpoints,
895        configured_modules = configured_modules.len(),
896        machine_id = bound.machine_id.as_ref().map(|id| id.as_str()).unwrap_or("none"),
897        "subc daemon starting"
898    );
899
900    // Before anything can spawn a module (the configured modules below, or a
901    // client's start request once the listeners are served): a previous daemon
902    // that died without its shutdown stop may have left children running, and
903    // a fresh copy beside one would fight it for its port and stores. The
904    // record is not a running daemon's because this daemon holds the lock on
905    // the run directory the record lives in, and a running daemon holds that
906    // lock for its whole life. (Claiming the singleton is not enough: it is
907    // keyed on the connection file, which can live in a different runtime
908    // directory from the run directory.) The lock stays held until this
909    // function returns, which is when the daemon exits.
910    let run_dir_lock = bound.run_dir_lock;
911    let publication_lock = bound.publication_lock;
912    if let Some(owner) = &run_dir_lock {
913        #[cfg(target_os = "linux")]
914        crate::live_children::sweep_orphans_with_placement(
915            owner,
916            &crate::live_children::AdoptedPids::none(),
917            crate::live_children::SweepBounds::default(),
918            cgroup_placement.as_ref(),
919        )
920        .await;
921        #[cfg(not(target_os = "linux"))]
922        crate::live_children::sweep_orphans(
923            owner,
924            &crate::live_children::AdoptedPids::none(),
925            crate::live_children::SweepBounds::default(),
926        )
927        .await;
928    }
929
930    let registry = Arc::new(Registry::default());
931    let process_liveness = Arc::new(SupervisorProcessLiveness::new());
932    let supervisor_handle = SupervisorHandle::new();
933    #[cfg(feature = "test-support")]
934    supervisor_handle.set_shutdown_intent_window(shutdown_intent_window);
935    let connected_clients = ConnectedClients::new();
936    let daemon_incarnation = format!(
937        "{:032x}",
938        u128::from_be_bytes(bound.connection_info.daemon_id)
939    );
940    let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
941        .with_process_liveness(process_liveness.clone())
942        .with_forwarding(Arc::clone(&forwarding))
943        .with_handle(supervisor_handle.clone())
944        .with_connection_file_path(bound.connection_file_path.clone())
945        .with_daemon_incarnation(daemon_incarnation.clone());
946    let supervisor = match privacy_trampoline {
947        Some(path) => supervisor.with_privacy_trampoline(path),
948        None => supervisor,
949    };
950    // ABSENT MEANS NO CAPTURE AND NO JOURNAL, NOT "THE REAL RUN DIRECTORY", and
951    // the difference is a production-corruption hazard rather than a preference.
952    // Both fields below follow the same rule: `None` for `capture_logs_dir`
953    // means supervised output is not captured, and `None` for
954    // `terminal_journal_path` means terminal history lives only in each
955    // module's in-memory ring (`supervisor.terminals` still answers, with the
956    // journal counters at zero).
957    //
958    // The terminal journal used to fall back to
959    // `daemon_run_dir().join("terminals.jsonl")`, so an in-process test daemon
960    // appended its fixture exits to the operator's journal, where they then
961    // appeared in `ck module terminals`.
962    //
963    // The capture line used to be `unwrap_or_else(|| daemon_run_dir().join("logs"))`,
964    // so ANY caller that did not set the field captured supervised children into
965    // the operator's live `~/.local/share/cortexkit/run/logs/`. That is twelve
966    // sibling repos whose integration tests boot an in-process daemon through
967    // `run_with_config` -- none of which asked for it, and none of which can see
968    // it from their side.
969    //
970    // Harmless while fixture module ids are fixture-shaped: this host carries 20
971    // zero-byte files from subc's own tests (good-aft, missing-aft,
972    // preview-consumer...). THE HAZARD IS A COLLISION. A fixture named "broca"
973    // or "aft" appends to a PRODUCTION capture file that operators read
974    // forensically and that placement gates count lines in -- with no residue to
975    // notice, because the file legitimately exists and legitimately grows.
976    //
977    // Found by BROCA (2026-09-19) from the other side: their rigs spawn the
978    // SHIPPED ck-subc and set XDG_CONFIG_HOME + XDG_RUNTIME_DIR but not
979    // XDG_DATA_HOME, so every local rig run supervised a module named "broca"
980    // and captured it into production's broca.stderr.log -- the same file I
981    // count seal lines in before and after placing their binaries.
982    //
983    // The supervisor already treats `None` as no-capture and no-journal, so
984    // this only removes invented defaults. The binary keeps capturing and
985    // journaling via `BootstrapConfig::from_env_for_daemon_binary`.
986
987    // Keep both ledgers in the explicitly selected run directory. An embedded
988    // daemon without a journal path must not write to the operator's live ledger.
989    let activity_run_dir = terminal_journal_path
990        .as_ref()
991        .and_then(|path| path.parent())
992        .map(Path::to_path_buf);
993    let supervisor = match terminal_journal_path {
994        Some(path) => supervisor.with_terminal_journal(path, daemon_incarnation.clone()),
995        None => supervisor,
996    };
997    let supervisor = match capture_logs_dir {
998        Some(dir) => supervisor.with_capture_logs_dir(dir),
999        None => supervisor,
1000    };
1001    let supervisor = match live_children_path {
1002        Some(path) => supervisor.with_live_children_record(path),
1003        None => supervisor,
1004    };
1005    #[cfg(target_os = "linux")]
1006    let supervisor = supervisor.with_cgroup_placement(cgroup_placement);
1007    // Collect per-module route.bind relay overrides BEFORE handing the
1008    // `configured_modules` vector to the supervisor (which only needs each
1009    // module's `drain_timeout_ms`). Each entry was filled in by parse-time
1010    // resolution (per-module > daemon-wide > absent), so modules with no
1011    // override are absent from this map and the daemon-wide default applies.
1012    let route_bind_relay_timeouts = configured_modules
1013        .iter()
1014        .filter_map(|module| {
1015            module
1016                .route_bind_relay_timeout_ms
1017                .map(|ms| (module.module_id.clone(), Duration::from_millis(ms)))
1018        })
1019        .collect::<std::collections::BTreeMap<_, _>>();
1020    let control_start_clock = crate::clock::StartClock::capture();
1021    let mut control = ControlHandler::with_forwarding(Arc::clone(&registry), forwarding)
1022        .with_daemon_version(bound.connection_info.daemon_ver.clone())
1023        .with_process_liveness(process_liveness)
1024        .with_supervisor(supervisor_handle)
1025        // The same flag `begin_daemon_shutdown` sets below, so requirement
1026        // changes while modules exit during the shutdown log at INFO.
1027        .with_daemon_shutdown_flag(supervisor.daemon_shutdown_flag())
1028        .with_connected_clients(connected_clients.clone())
1029        .with_local_endpoint_unavailable(bound.local_endpoint_unavailable.clone())
1030        .with_storage_config(storage_config)
1031        .with_machine_id(bound.machine_id.clone())
1032        .with_admission_facts_config(admission_facts.carrier_module_id, admission_facts.targets)
1033        .with_scope_authority_owners(scope_authority_owners)
1034        .with_federated_scope_owner(federated_scope_owner)
1035        .with_privileged_operator_prompt(privileged_operator_prompt)
1036        .with_route_bind_relay_timeouts(route_bind_relay_timeouts)
1037        .with_daemon_provenance(
1038            bound.connection_info.pid,
1039            control_start_clock.started_at_ms(),
1040            std::env::current_exe().ok(),
1041            normalized_build_provenance(env!("SUBC_BUILD_GIT_SHA")),
1042            normalized_build_provenance(env!("SUBC_BUILD_LOCK_DIGEST")),
1043        )
1044        .with_daemon_start_clock(control_start_clock)
1045        .with_capability_config(
1046            configured_modules
1047                .iter()
1048                .map(|module| (module.module_id.clone(), module.enabled)),
1049            reserved_capabilities,
1050        );
1051    if let Some(run_dir) = activity_run_dir {
1052        control =
1053            control.with_activity_ledger_config(&run_dir, daemon_incarnation, activity_config);
1054    }
1055    if let Some(ms) = route_bind_relay_default_ms {
1056        // A daemon-wide config value overrides the built-in default; a
1057        // `None` here leaves the ControlHandler's 12s default in place.
1058        control = control.with_route_bind_relay_timeout(Duration::from_millis(ms));
1059    }
1060    if let Some(config_path) = daemon_config_path {
1061        control = control.with_supervisor_rescan(supervisor.clone(), config_path, configured_port);
1062    }
1063    let control = Arc::new(control);
1064    let router = Arc::new(Router::with_control_handler(Arc::clone(&control)));
1065    let auth = ServerAuth::new(
1066        bound.connection_info.key.clone(),
1067        bound.connection_info.daemon_id,
1068        bound.connection_info.daemon_ver.clone(),
1069    )
1070    .with_connected_clients(connected_clients);
1071
1072    // Discovery must expose every accepted configured module, including disabled
1073    // ones. Registration creates status handles but neither starts monitors
1074    // nor launches processes: children need the published file for `--subc`.
1075    let mut registered_modules = Vec::with_capacity(configured_modules.len());
1076    for configured in configured_modules {
1077        let enabled = configured.enabled;
1078        let health = configured.health.clone();
1079        let module_id = configured.module_id.clone();
1080        let registered = match supervisor.register_configured_with_health(
1081            configured.module_spec(),
1082            enabled,
1083            health.clone(),
1084            configured.drain_timeout_ms,
1085            configured.restart,
1086            configured.register_timeout,
1087        ) {
1088            Ok(registered) => registered,
1089            Err(err) => {
1090                error!(module_id = %module_id, error = %err, "failed to supervise configured module; continuing daemon startup");
1091                continue;
1092            }
1093        };
1094        registered_modules.push(registered);
1095        // A relaxed threshold can silently outlive the temporary allowance
1096        // that required it, so report the extra tolerance on every boot.
1097        let default_threshold = HealthConfig::default().failure_threshold;
1098        if enabled && health.failure_threshold > default_threshold {
1099            warn!(
1100                module_id = %module_id,
1101                failure_threshold = health.failure_threshold,
1102                default_threshold,
1103                tolerance_secs = health.cadence.as_secs() * u64::from(health.failure_threshold),
1104                "health failure threshold is relaxed above the default; a wedged module stays unflagged for longer"
1105            );
1106        }
1107        info!(module_id = %module_id, enabled, "configured module registered");
1108    }
1109    control.refresh_capability_requirements();
1110
1111    // The local endpoint is served beside TCP and answers the same handshake.
1112    // It serves clients only, which the server enforces per connection.
1113    let local_serve_task = bound.local_listener.map(|local_listener| {
1114        AbortOnDrop::new(tokio::spawn(serve_local_listener(
1115            local_listener,
1116            Arc::clone(&router),
1117            auth.clone(),
1118        )))
1119    });
1120    // On Unix the task is dropped below to stop accepting at shutdown. Windows
1121    // leaves it to the process exit; naming it keeps the binding uniform.
1122    #[cfg(windows)]
1123    let _local_serve_task = local_serve_task;
1124    let mut serve_task =
1125        AbortOnDrop::new(tokio::spawn(serve_listeners(bound.listeners, router, auth)));
1126    verify_serving(&bound.connection_info).await?;
1127    // A published endpoint promises that authentication can complete now.
1128    // Orphan cleanup and control-plane construction can take longer than an
1129    // existing-daemon probe's deadline, so finish both before discovery.
1130    // The write fsyncs, so it runs on tokio's blocking pool (see
1131    // `publish_connection_file`). The singleton lock travels with it and comes
1132    // back only once the file is in place.
1133    let (published, publication_lock) = publish_connection_file(
1134        &bound.connection_file_path,
1135        #[cfg(windows)]
1136        bound.connection_file_copy.as_deref(),
1137        &bound.connection_info,
1138        bound.local_endpoint.as_ref(),
1139        publication_lock,
1140    )
1141    .await;
1142    published?;
1143    #[cfg(test)]
1144    startup_publication_hook::run(&bound.connection_file_path).await;
1145    drop(publication_lock);
1146    let _clock_step_task = AbortOnDrop::new(crate::watchdog::spawn_clock_step_monitor());
1147    // Off the startup path: it runs `systemctl`, and only ever logs.
1148    #[cfg(target_os = "linux")]
1149    let _kill_mode_check = AbortOnDrop::new(tokio::spawn(
1150        crate::systemd_kill_mode::warn_if_kill_mode_defeats_ordered_shutdown(),
1151    ));
1152    let _watchdog_task = AbortOnDrop::new(
1153        DaemonSelfWatchdog::new(
1154            bound.connection_info.clone(),
1155            bound.connection_file_path.clone(),
1156        )
1157        .with_config(watchdog_config)
1158        .spawn(),
1159    );
1160
1161    for registered in registered_modules {
1162        match supervisor.start_registered_configured(registered, None, true) {
1163            Ok(module) => {
1164                info!(module_id = %module.module_id(), "configured module supervised");
1165            }
1166            Err(err) => {
1167                error!(error = %err, "failed to start configured module; continuing daemon startup");
1168            }
1169        }
1170    }
1171
1172    control.refresh_capability_requirements();
1173    Arc::clone(&control).spawn_capability_deadline_loop();
1174    Arc::clone(&control).spawn_scope_expiry_loop();
1175    control.spawn_resource_sampler();
1176
1177    #[cfg(unix)]
1178    {
1179        tokio::select! {
1180            result = serve_task.join() => {
1181                return result.map_err(BootstrapError::ServeJoin)?.map_err(BootstrapError::Serve);
1182            }
1183            _ = terminate.recv() => {}
1184        }
1185        // First, before the notice, the drain, or any connection close: from
1186        // here on a module exit is recorded as `daemon_shutdown` and never
1187        // respawned. Also before allowing a second signal to cut the bounded
1188        // wait short, so the journal marker is always written.
1189        supervisor.begin_daemon_shutdown();
1190        // Stop the self-watchdog before closing the listener. Its next tick would
1191        // connect to that listener, fail, and log an ERROR indistinguishable from
1192        // a wedged daemon, once for every interval a planned stop lasts.
1193        drop(_watchdog_task);
1194        // Dropping the listener stops new accepts, not established connections:
1195        // their detached tasks must remain live throughout notice and drain. The
1196        // local endpoint stops accepting the same way.
1197        drop(serve_task);
1198        drop(local_serve_task);
1199        let second_sigterm = async {
1200            terminate.recv().await;
1201            std::time::Instant::now()
1202        };
1203        tokio::pin!(second_sigterm);
1204        let mut escalation_started = tokio::select! {
1205            biased;
1206            at = second_sigterm.as_mut() => {
1207                info!("second SIGTERM: abandoning daemon shutdown wait");
1208                Some(at)
1209            }
1210            result = supervisor.drain_for_daemon_shutdown() => {
1211                if let Err(error) = result {
1212                    warn!(%error, "daemon shutdown drain failed; exiting anyway");
1213                }
1214                None
1215            }
1216        };
1217        // Supervised modules lead their own process groups, so a service
1218        // manager's kill of this process's group does not reach them. (A
1219        // systemd unit with KillMode=control-group kills by cgroup instead and
1220        // does reach them; see `systemd_kill_mode`.) The daemon ends them
1221        // itself: EOF (or SIGTERM for a protocol none child) first, then
1222        // signals at each child's own drain deadline.
1223        let already_escalated = escalation_started.is_some();
1224        supervisor
1225            .end_children_for_daemon_shutdown(already_escalated, async {
1226                if already_escalated {
1227                    std::future::pending::<()>().await;
1228                } else {
1229                    escalation_started = Some(second_sigterm.await);
1230                }
1231            })
1232            .await;
1233        if let Some(at) = escalation_started {
1234            info!(
1235                second_sigterm_elapsed_ms = at.elapsed().as_millis() as u64,
1236                "daemon shutdown wait ended after second SIGTERM"
1237            );
1238        }
1239        info!("daemon exiting: SIGTERM shutdown complete");
1240        Ok(())
1241    }
1242    #[cfg(not(unix))]
1243    serve_task
1244        .join()
1245        .await
1246        .map_err(BootstrapError::ServeJoin)?
1247        .map_err(BootstrapError::Serve)
1248}
1249
1250fn normalized_build_provenance(value: &str) -> Option<String> {
1251    match value.trim() {
1252        "" | "unavailable" => None,
1253        value => Some(value.to_string()),
1254    }
1255}
1256
1257/// Find an existing daemon or atomically bind loopback TCP for this daemon.
1258///
1259/// The algorithm is intentionally connect-first: an endpoint from the connection
1260/// file is treated as live only after the TCP+key server-proof authenticates for
1261/// that file's key and daemon_id. Stale or foreign connection files are reclaimed
1262/// only while holding the per-user start lock; the TCP port is never the
1263/// singleton primitive.
1264pub async fn ensure_singleton(
1265    connection_file_path: impl AsRef<Path>,
1266    port: u16,
1267) -> Result<Outcome, BootstrapError> {
1268    ensure_singleton_with_config(BootstrapConfig::new(connection_file_path.as_ref(), port)).await
1269}
1270
1271pub async fn ensure_singleton_with_config(
1272    config: BootstrapConfig,
1273) -> Result<Outcome, BootstrapError> {
1274    ensure_singleton_inner(config, true).await
1275}
1276
1277async fn ensure_singleton_inner(
1278    config: BootstrapConfig,
1279    publish: bool,
1280) -> Result<Outcome, BootstrapError> {
1281    #[cfg(windows)]
1282    let copy_path = windows_connection_copy::resolve(&config);
1283    let path = config.connection_file_path;
1284
1285    if stable_peer(&path).await? {
1286        return Ok(Outcome::AlreadyRunning);
1287    }
1288
1289    let lock = StartLock::acquire(&path).await?;
1290
1291    // Re-probe after acquiring the start lock so a peer that won the race between
1292    // our first failed probe and the lock acquisition is observed instead of
1293    // overwritten.
1294    if stable_peer(&path).await? {
1295        return Ok(Outcome::AlreadyRunning);
1296    }
1297
1298    // The start lock and the probe above only rule out a daemon using this
1299    // connection file. A daemon started with another runtime directory but
1300    // the same data home shares the run directory while passing both, so the
1301    // run directory needs its own owner: refuse at once if a live daemon holds
1302    // it, before anything below reads or writes run or data-home state.
1303    let run_dir_lock = config
1304        .live_children_path
1305        .as_deref()
1306        .map(crate::run_dir_lock::RunDirLock::acquire)
1307        .transpose()?;
1308
1309    remove_stale_connection_file_if_present(&path)?;
1310    #[cfg(windows)]
1311    let copy_path = windows_connection_copy::reclaim(&path, copy_path).await;
1312
1313    // Established under the start lock, and under the run-directory lock when
1314    // there is one, and before binding: a corrupt file stops boot before
1315    // anything is published, and two daemons racing to start cannot both mint.
1316    //
1317    // Minting a new machine id fsyncs its file, so loading or minting runs on
1318    // tokio's blocking pool. Both locks travel with that work, so neither is
1319    // released while a mint is in flight.
1320    let (machine_id, lock, run_dir_lock) = match config.machine_id_path {
1321        None => (None, lock, run_dir_lock),
1322        Some(machine_id_path) => {
1323            let task_path = machine_id_path.clone();
1324            match run_blocking_holding((lock, run_dir_lock), move || {
1325                crate::machine_id::load_or_mint(&task_path)
1326            })
1327            .await
1328            {
1329                Ok((Ok(id), (lock, run_dir_lock))) => (Some(id), lock, run_dir_lock),
1330                Ok((Err(error), _locks)) => return Err(BootstrapError::MachineId(error)),
1331                Err(join_error) => {
1332                    return Err(BootstrapError::MachineId(
1333                        crate::machine_id::MachineIdFileError::Write {
1334                            path: machine_id_path,
1335                            source: io::Error::other(describe_blocking_task_failure(
1336                                "loading or minting the machine id",
1337                                &join_error,
1338                            )),
1339                        },
1340                    ))
1341                }
1342            }
1343        }
1344    };
1345
1346    let (listeners, endpoints) = bind_loopback(config.port).await?;
1347    // The local endpoint is an addition to the daemon, not a condition for it:
1348    // a bind failure leaves clients on TCP only, with the connection file naming
1349    // no local endpoint, and the reason reported by `server.describe`. A daemon
1350    // that refused to start would take every module down for a feature nothing
1351    // has to use yet.
1352    let (local_listener, local_endpoint, local_endpoint_unavailable) =
1353        if config.local_listener.configured_dir().is_some() {
1354            match local_listener::bind(&config.local_listener, &path) {
1355                Ok(local_listener) => {
1356                    let endpoint = local_listener.endpoint();
1357                    (Some(local_listener), Some(endpoint), None)
1358                }
1359                Err(error) => {
1360                    let reason = error.to_string();
1361                    warn!(
1362                        error = %reason,
1363                        "the local endpoint is unavailable; serving loopback TCP only"
1364                    );
1365                    (None, None, Some(reason))
1366                }
1367            }
1368        } else {
1369            (None, None, None)
1370        };
1371    let connection_info = ConnectionInfo {
1372        schema: SCHEMA_VERSION,
1373        wire_version: Some(PROTOCOL_VERSION),
1374        endpoints,
1375        key: generate_key().map_err(BootstrapError::GenerateConnectionFile)?,
1376        daemon_id: generate_daemon_id().map_err(BootstrapError::GenerateConnectionFile)?,
1377        pid: process::id(),
1378        daemon_ver: config.daemon_ver,
1379    };
1380
1381    // The low-level binding API historically publishes for callers that serve
1382    // these listeners themselves. The daemon entry point defers publication
1383    // until its own authentication server is running.
1384    let publication_lock = if publish {
1385        let (published, lock) = publish_connection_file(
1386            &path,
1387            #[cfg(windows)]
1388            copy_path.as_deref(),
1389            &connection_info,
1390            local_endpoint.as_ref(),
1391            lock,
1392        )
1393        .await;
1394        if let Err(error) = published {
1395            if let Some(local_listener) = local_listener.as_ref() {
1396                local_listener::release(local_listener);
1397            }
1398            drop(listeners);
1399            return Err(error);
1400        }
1401        drop(lock);
1402        None
1403    } else {
1404        Some(lock)
1405    };
1406
1407    Ok(Outcome::Bound(BoundDaemon {
1408        listeners,
1409        local_listener,
1410        local_endpoint,
1411        local_endpoint_unavailable,
1412        connection_info,
1413        connection_file_path: path,
1414        connection_file_source: config.connection_file_source,
1415        #[cfg(windows)]
1416        connection_file_copy: copy_path,
1417        machine_id,
1418        run_dir_lock,
1419        publication_lock,
1420    }))
1421}
1422
1423#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1424enum Probe {
1425    Live,
1426    StaleOrAbsent,
1427}
1428
1429async fn stable_peer(path: &Path) -> Result<bool, BootstrapError> {
1430    if !matches!(probe_existing(path).await?, Probe::Live) {
1431        return Ok(false);
1432    }
1433    // A terminating peer can still authenticate while its listeners drain.
1434    // Require another proof after a short settling interval before reporting
1435    // a successful duplicate start. If the peer left, proceed with startup;
1436    // any remaining bind/ownership conflict is an error, not exit 0 (which
1437    // would suppress launchd's restart-on-failure policy).
1438    tokio::time::sleep(Duration::from_millis(1500)).await;
1439    Ok(matches!(probe_existing(path).await?, Probe::Live))
1440}
1441
1442async fn probe_existing(path: &Path) -> Result<Probe, BootstrapError> {
1443    let info = match connection_file::read(path) {
1444        Ok(info) => info,
1445        Err(source) if is_absent_or_stale_connection_file(&source) => {
1446            return Ok(Probe::StaleOrAbsent)
1447        }
1448        Err(source) => {
1449            return Err(BootstrapError::ConnectionFileRead {
1450                path: path.to_path_buf(),
1451                source,
1452            })
1453        }
1454    };
1455
1456    for endpoint in &info.endpoints {
1457        if matches!(probe_endpoint(&info, endpoint).await, Probe::Live) {
1458            return Ok(Probe::Live);
1459        }
1460    }
1461
1462    Ok(Probe::StaleOrAbsent)
1463}
1464
1465async fn probe_endpoint(info: &ConnectionInfo, endpoint: &Endpoint) -> Probe {
1466    let Ok(ip) = endpoint.host.parse::<IpAddr>() else {
1467        return Probe::StaleOrAbsent;
1468    };
1469    if !ip.is_loopback() {
1470        return Probe::StaleOrAbsent;
1471    }
1472    let addr = SocketAddr::new(ip, endpoint.port);
1473
1474    let mut stream = match timeout(CONNECT_TIMEOUT, TcpStream::connect(addr)).await {
1475        Ok(Ok(stream)) => stream,
1476        Ok(Err(_)) | Err(_) => return Probe::StaleOrAbsent,
1477    };
1478
1479    match authenticate_client(&mut stream, info, PROBE_AUTH_DEADLINE).await {
1480        Ok(()) => Probe::Live,
1481        Err(AuthError::DaemonIdMismatch)
1482        | Err(AuthError::InvalidServerProof)
1483        | Err(AuthError::UnexpectedEof { .. })
1484        | Err(AuthError::Timeout { .. })
1485        | Err(AuthError::JsonEncode { .. })
1486        | Err(AuthError::JsonDecode { .. })
1487        | Err(AuthError::Io { .. })
1488        | Err(AuthError::MessageTooLarge { .. })
1489        | Err(AuthError::KeyTooShort { .. })
1490        | Err(AuthError::Random(_))
1491        | Err(AuthError::InvalidClientAuth) => Probe::StaleOrAbsent,
1492    }
1493}
1494
1495async fn verify_serving(info: &ConnectionInfo) -> Result<(), BootstrapError> {
1496    // Scheduling a server task is not a readiness barrier. Authenticate once
1497    // against the mandatory IPv4 listener before exposing discovery to peers.
1498    // This uses the normal probe deadline, with no retries or widened budget.
1499    if let Some(endpoint) = info.endpoints.first() {
1500        if matches!(probe_endpoint(info, endpoint).await, Probe::Live) {
1501            return Ok(());
1502        }
1503    }
1504    Err(BootstrapError::StartupNotServing)
1505}
1506
1507/// Run blocking filesystem work (an fsync, a rename) on tokio's blocking-thread
1508/// pool rather than on the async worker thread that awaits it. On a slow or
1509/// busy disk an fsync can take seconds, and a runtime has few worker threads
1510/// (an in-process test daemon may share two with everything else), so doing it
1511/// inline stalls unrelated tasks for that long.
1512///
1513/// `guards` (lock handles) move into the blocking task and are handed back
1514/// with the work's result. If the awaiting future is dropped first, the work
1515/// still runs to completion on its thread and the guards are released only
1516/// after it ends, so a lock is never given up while the work it protects is
1517/// still writing.
1518///
1519/// `Err` means the task panicked, or was cancelled before it started because
1520/// the runtime is shutting down; the guards have then been released already.
1521async fn run_blocking_holding<G, T>(
1522    guards: G,
1523    work: impl FnOnce() -> T + Send + 'static,
1524) -> Result<(T, G), JoinError>
1525where
1526    G: Send + 'static,
1527    T: Send + 'static,
1528{
1529    tokio::task::spawn_blocking(move || {
1530        let output = work();
1531        (output, guards)
1532    })
1533    .await
1534}
1535
1536/// Explain, for a startup error, why blocking work produced no result.
1537fn describe_blocking_task_failure(work: &str, join_error: &JoinError) -> String {
1538    if join_error.is_panic() {
1539        format!("the background thread {work} panicked before finishing: {join_error}")
1540    } else {
1541        format!(
1542            "the background thread {work} was cancelled before finishing, because the async runtime is shutting down: {join_error}"
1543        )
1544    }
1545}
1546
1547/// Publish the connection file: owner-only permissions, written to a temporary
1548/// file, fsynced and renamed over `path` (all done by `write_atomic`). The
1549/// write runs on tokio's blocking pool through [`run_blocking_holding`], so the
1550/// fsync never holds an async worker thread.
1551///
1552/// `guard` is the singleton start lock (or whatever the caller must keep held
1553/// across publication); it is returned with the result. It comes back `None`
1554/// only when the write task panicked or was cancelled, in which case the
1555/// result is an error and the file must be treated as not published.
1556///
1557/// On Windows, `copy_path` asks for the profile copy as well. Its start lock is
1558/// taken before the primary write. Once `path` is written, an error returned
1559/// while preparing or writing the copy only logs a warning, and the result is
1560/// still success. The two writes share one blocking task but aren't one atomic
1561/// step, and a panic or cancellation of that task is still a publication
1562/// error.
1563async fn publish_connection_file<G: Send + 'static>(
1564    path: &Path,
1565    #[cfg(windows)] copy_path: Option<&Path>,
1566    info: &ConnectionInfo,
1567    local_endpoint: Option<&LocalEndpoint>,
1568    guard: G,
1569) -> (Result<(), BootstrapError>, Option<G>) {
1570    let task_path = path.to_path_buf();
1571    let task_info = info.clone();
1572    let task_endpoint = local_endpoint.cloned();
1573    #[cfg(windows)]
1574    let copy = windows_connection_copy::claim_for_publication(path, copy_path, info).await;
1575    let outcome = run_blocking_holding(guard, move || {
1576        write_connection_file(&task_path, &task_info, task_endpoint.as_ref())?;
1577        #[cfg(windows)]
1578        if let Some((copy_path, _lock)) = copy {
1579            if let Err(error) =
1580                write_connection_file(&copy_path, &task_info, task_endpoint.as_ref())
1581            {
1582                windows_connection_copy::warn_unavailable(&task_path, Some(&copy_path), &error);
1583            }
1584        }
1585        Ok(())
1586    })
1587    .await;
1588    match outcome {
1589        Ok((Ok(()), guard)) => (Ok(()), Some(guard)),
1590        Ok((Err(source), guard)) => (
1591            Err(BootstrapError::ConnectionFileWrite {
1592                path: path.to_path_buf(),
1593                source,
1594            }),
1595            Some(guard),
1596        ),
1597        Err(join_error) => (
1598            Err(BootstrapError::ConnectionFileWrite {
1599                path: path.to_path_buf(),
1600                source: ConnectionFileError::Io {
1601                    op: "write",
1602                    path: path.to_path_buf(),
1603                    source: io::Error::other(describe_blocking_task_failure(
1604                        "writing the connection file",
1605                        &join_error,
1606                    )),
1607                },
1608            }),
1609            None,
1610        ),
1611    }
1612}
1613
1614/// The synchronous connection-file write, including its fsync. Call it only
1615/// through [`publish_connection_file`], which keeps it off the async workers.
1616///
1617/// `local_endpoint` is written through subc-transport's endpoint-aware writer,
1618/// so the file carries `local_endpoint` when there is a local listener and is
1619/// byte-identical to before when there is none.
1620fn write_connection_file(
1621    path: &Path,
1622    info: &ConnectionInfo,
1623    local_endpoint: Option<&LocalEndpoint>,
1624) -> Result<(), ConnectionFileError> {
1625    #[cfg(test)]
1626    connection_file_write_hook::run(path);
1627    write_atomic_with_local_endpoint(path, info, local_endpoint)
1628}
1629
1630/// Lets a test run code inside the connection-file write, on whichever thread
1631/// performs it, for example to block there and check what else can still run.
1632#[cfg(test)]
1633mod connection_file_write_hook {
1634    use std::{
1635        path::{Path, PathBuf},
1636        sync::{Mutex, PoisonError},
1637    };
1638
1639    type Hook = Box<dyn FnOnce() + Send>;
1640
1641    /// Keyed by connection-file path, so a hook fires only for the test that
1642    /// installed it while other tests publish their own files in parallel.
1643    static HOOKS: Mutex<Vec<(PathBuf, Hook)>> = Mutex::new(Vec::new());
1644
1645    /// Run `hook` once, at the start of the next write of `path`.
1646    pub(super) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) {
1647        HOOKS
1648            .lock()
1649            .unwrap_or_else(PoisonError::into_inner)
1650            .push((path.to_path_buf(), Box::new(hook)));
1651    }
1652
1653    pub(super) fn run(path: &Path) {
1654        let hook = {
1655            let mut hooks = HOOKS.lock().unwrap_or_else(PoisonError::into_inner);
1656            hooks
1657                .iter()
1658                .position(|(hooked, _)| hooked == path)
1659                .map(|index| hooks.swap_remove(index).1)
1660        };
1661        if let Some(hook) = hook {
1662            hook();
1663        }
1664    }
1665}
1666
1667/// A path-scoped barrier after publication and before any configured activation.
1668/// The async wait leaves the listeners free to answer discovery-time requests.
1669#[cfg(test)]
1670mod startup_publication_hook {
1671    use std::{
1672        path::{Path, PathBuf},
1673        sync::{Mutex, PoisonError},
1674    };
1675    use tokio::sync::oneshot;
1676
1677    type Barrier = (oneshot::Sender<()>, oneshot::Receiver<()>);
1678    static BARRIERS: Mutex<Vec<(PathBuf, Barrier)>> = Mutex::new(Vec::new());
1679
1680    pub(super) fn install(path: &Path) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
1681        let (entered_tx, entered_rx) = oneshot::channel();
1682        let (release_tx, release_rx) = oneshot::channel();
1683        BARRIERS
1684            .lock()
1685            .unwrap_or_else(PoisonError::into_inner)
1686            .push((path.to_path_buf(), (entered_tx, release_rx)));
1687        (entered_rx, release_tx)
1688    }
1689
1690    pub(super) async fn run(path: &Path) {
1691        let barrier = {
1692            let mut barriers = BARRIERS.lock().unwrap_or_else(PoisonError::into_inner);
1693            barriers
1694                .iter()
1695                .position(|(hooked, _)| hooked == path)
1696                .map(|index| barriers.swap_remove(index).1)
1697        };
1698        if let Some((entered, release)) = barrier {
1699            let _ = entered.send(());
1700            let _ = release.await;
1701        }
1702    }
1703}
1704
1705fn is_absent_or_stale_connection_file(err: &ConnectionFileError) -> bool {
1706    match err {
1707        ConnectionFileError::Io { source, .. } if source.kind() == io::ErrorKind::NotFound => true,
1708        ConnectionFileError::JsonRead { .. }
1709        | ConnectionFileError::UnsupportedSchema { .. }
1710        | ConnectionFileError::Invalid { .. }
1711        | ConnectionFileError::KeyTooShort { .. }
1712        // A live daemon always publishes the file owner-only (0600), so a file
1713        // with insecure permissions is never a daemon we should defer to: treat it
1714        // as stale and take over (which republishes a correct 0600 file).
1715        | ConnectionFileError::InsecurePermissions { .. } => true,
1716        ConnectionFileError::MissingParent { .. }
1717        | ConnectionFileError::MissingFileName { .. }
1718        // A writable ancestor is an operator misconfiguration, never evidence
1719        // about whether a daemon is live. Reclaiming the file would republish key
1720        // material into the same directory the refusal is about.
1721        | ConnectionFileError::InsecureParentDirectory { .. }
1722        | ConnectionFileError::Io { .. }
1723        | ConnectionFileError::JsonWrite { .. }
1724        | ConnectionFileError::Random(_)
1725        // A wire mismatch may identify a newer live daemon, so never reclaim its
1726        // connection file merely because this binary cannot speak its envelope.
1727        | ConnectionFileError::WireVersionMismatch { .. } => false,
1728    }
1729}
1730
1731fn remove_stale_connection_file_if_present(path: &Path) -> Result<(), BootstrapError> {
1732    match fs::remove_file(path) {
1733        Ok(()) => Ok(()),
1734        Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
1735        Err(source) => Err(BootstrapError::RemoveStale {
1736            path: path.to_path_buf(),
1737            source,
1738        }),
1739    }
1740}
1741
1742async fn bind_loopback(port: u16) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError> {
1743    bind_loopback_with(port, |port| TcpListener::bind((Ipv6Addr::LOCALHOST, port))).await
1744}
1745
1746async fn bind_loopback_with<F, Fut>(
1747    port: u16,
1748    mut bind_v6: F,
1749) -> Result<(Vec<TcpListener>, Vec<Endpoint>), BootstrapError>
1750where
1751    F: FnMut(u16) -> Fut,
1752    Fut: std::future::Future<Output = io::Result<TcpListener>>,
1753{
1754    let v4_host = Ipv4Addr::LOCALHOST;
1755    let v4 = TcpListener::bind((v4_host, port))
1756        .await
1757        .map_err(|source| BootstrapError::Bind {
1758            host: v4_host.to_string(),
1759            port,
1760            source,
1761        })?;
1762    let actual_port = v4
1763        .local_addr()
1764        .map_err(|source| BootstrapError::LocalAddr {
1765            host: v4_host.to_string(),
1766            source,
1767        })?
1768        .port();
1769
1770    let mut listeners = vec![v4];
1771    let mut endpoints = vec![Endpoint {
1772        host: v4_host.to_string(),
1773        port: actual_port,
1774    }];
1775
1776    let v6_host = Ipv6Addr::LOCALHOST;
1777    match bind_v6(actual_port).await {
1778        Ok(v6) => {
1779            listeners.push(v6);
1780            endpoints.push(Endpoint {
1781                host: v6_host.to_string(),
1782                port: actual_port,
1783            });
1784        }
1785        Err(err)
1786            if ipv6_loopback_unavailable(&err)
1787                || (port == 0 && err.kind() == io::ErrorKind::AddrInUse) =>
1788        {
1789            warn!(
1790                port = actual_port,
1791                error = %err,
1792                "IPv6 loopback unavailable; serving only IPv4 loopback"
1793            );
1794        }
1795        Err(source) => {
1796            drop(listeners);
1797            return Err(BootstrapError::Bind {
1798                host: v6_host.to_string(),
1799                port: actual_port,
1800                source,
1801            });
1802        }
1803    }
1804
1805    Ok((listeners, endpoints))
1806}
1807
1808fn ipv6_loopback_unavailable(err: &io::Error) -> bool {
1809    matches!(
1810        err.kind(),
1811        io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported
1812    ) || matches!(err.raw_os_error(), Some(47) | Some(49) | Some(97))
1813}
1814
1815struct AbortOnDrop<T> {
1816    handle: JoinHandle<T>,
1817}
1818
1819impl<T> AbortOnDrop<T> {
1820    fn new(handle: JoinHandle<T>) -> Self {
1821        Self { handle }
1822    }
1823
1824    async fn join(&mut self) -> Result<T, JoinError> {
1825        (&mut self.handle).await
1826    }
1827}
1828
1829impl<T> Drop for AbortOnDrop<T> {
1830    fn drop(&mut self) {
1831        if !self.handle.is_finished() {
1832            self.handle.abort();
1833        }
1834    }
1835}
1836
1837#[derive(Debug)]
1838struct StartLock {
1839    // Keep the locked file handle alive for the duration of bootstrap.
1840    _file: fs::File,
1841}
1842
1843impl Drop for StartLock {
1844    fn drop(&mut self) {
1845        // Closing alone leaves the lock held if a concurrent fork inherited
1846        // the open file description and has not exec'd yet. Unlock explicitly
1847        // so a restart can reclaim the singleton as soon as bootstrap ends.
1848        if let Err(error) = FileExt::unlock(&self._file) {
1849            tracing::warn!(%error, "failed to release the singleton start lock");
1850        }
1851    }
1852}
1853
1854impl StartLock {
1855    async fn acquire(connection_file_path: &Path) -> Result<Self, BootstrapError> {
1856        let path = start_lock_path(connection_file_path);
1857        for _ in 0..START_LOCK_RETRIES {
1858            let file = match open_owner_only_lock(&path) {
1859                Ok(file) => file,
1860                Err(source) => return Err(BootstrapError::StartLockCreate { path, source }),
1861            };
1862            match FileExt::try_lock(&file) {
1863                Ok(()) => return Ok(Self { _file: file }),
1864                Err(TryLockError::WouldBlock) => sleep(START_LOCK_RETRY_DELAY).await,
1865                Err(TryLockError::Error(source)) => {
1866                    return Err(BootstrapError::StartLockCreate { path, source });
1867                }
1868            }
1869        }
1870
1871        Err(BootstrapError::StartLockBusy {
1872            path,
1873            attempts: START_LOCK_RETRIES,
1874        })
1875    }
1876}
1877
1878pub(crate) fn open_owner_only_lock(path: &Path) -> io::Result<fs::File> {
1879    let mut options = fs::OpenOptions::new();
1880    options.read(true).write(true).create(true);
1881    #[cfg(unix)]
1882    {
1883        use std::os::unix::fs::OpenOptionsExt;
1884        options.mode(0o600);
1885    }
1886    options.open(path)
1887}
1888
1889fn start_lock_path(connection_file_path: &Path) -> PathBuf {
1890    let file_name = connection_file_path
1891        .file_name()
1892        .map(|name| name.to_string_lossy())
1893        .unwrap_or_else(|| CONNECTION_FILE_NAME.into());
1894    let lock_name = format!("{file_name}.start-lock");
1895    connection_file_path
1896        .parent()
1897        .filter(|parent| !parent.as_os_str().is_empty())
1898        .unwrap_or_else(|| Path::new("."))
1899        .join(lock_name)
1900}
1901
1902fn non_empty_os_var(key: &str) -> Option<OsString> {
1903    let value = env::var_os(key)?;
1904    if value.is_empty() {
1905        None
1906    } else {
1907        Some(value)
1908    }
1909}
1910
1911/// Bootstrap-layer errors are deliberately typed so startup never panics for
1912/// ordinary daemon-discovery races or stale filesystem state.
1913#[derive(Debug)]
1914pub enum BootstrapError {
1915    /// The bound server could not authenticate its own readiness probe.
1916    StartupNotServing,
1917    #[cfg(unix)]
1918    Signal(io::Error),
1919    InvalidPort {
1920        raw: String,
1921        source: std::num::ParseIntError,
1922    },
1923    ConnectionFileRead {
1924        path: PathBuf,
1925        source: ConnectionFileError,
1926    },
1927    ConnectionFileWrite {
1928        path: PathBuf,
1929        source: ConnectionFileError,
1930    },
1931    GenerateConnectionFile(ConnectionFileError),
1932    StartLockCreate {
1933        path: PathBuf,
1934        source: io::Error,
1935    },
1936    StartLockBusy {
1937        path: PathBuf,
1938        attempts: usize,
1939    },
1940    /// The run-directory lock file could not be created, opened or locked.
1941    RunDirLockCreate {
1942        path: PathBuf,
1943        source: io::Error,
1944    },
1945    /// Another live process, almost certainly a daemon started with a
1946    /// different runtime directory over the same data home, owns the run
1947    /// directory. The daemon does not start. `holder_pid` is read from the
1948    /// lock file when possible and is informational only.
1949    RunDirBusy {
1950        path: PathBuf,
1951        holder_pid: Option<u32>,
1952    },
1953    RemoveStale {
1954        path: PathBuf,
1955        source: io::Error,
1956    },
1957    Bind {
1958        host: String,
1959        port: u16,
1960        source: io::Error,
1961    },
1962    LocalAddr {
1963        host: String,
1964        source: io::Error,
1965    },
1966    DaemonConfig(DaemonConfigError),
1967    /// The machine id could not be established: its file is corrupt, unreadable
1968    /// or unwritable, or the data home is relative. The daemon does not start.
1969    MachineId(crate::machine_id::MachineIdFileError),
1970    /// The daemon run directory could not be resolved because the data home is
1971    /// relative. The daemon does not start.
1972    RunDir(daemon_config::DaemonRunDirError),
1973    Serve(ServerError),
1974    ServeJoin(tokio::task::JoinError),
1975}
1976
1977impl From<ServerError> for BootstrapError {
1978    fn from(error: ServerError) -> Self {
1979        Self::Serve(error)
1980    }
1981}
1982
1983impl fmt::Display for BootstrapError {
1984    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1985        match self {
1986            Self::StartupNotServing => write!(f, "bound daemon did not authenticate its readiness probe; connection file was not published"),
1987            #[cfg(unix)]
1988            Self::Signal(error) => write!(f, "failed to register SIGTERM handler: {error}"),
1989            Self::InvalidPort { raw, source } => {
1990                write!(f, "invalid {SUBC_PORT_ENV} value '{raw}': {source}")
1991            }
1992            Self::ConnectionFileRead { path, source } => write!(
1993                f,
1994                "failed to read connection file {}: {source}",
1995                path.display()
1996            ),
1997            Self::ConnectionFileWrite { path, source } => write!(
1998                f,
1999                "failed to publish connection file {}: {source}",
2000                path.display()
2001            ),
2002            Self::GenerateConnectionFile(err) => {
2003                write!(f, "failed to generate connection-file auth material: {err}")
2004            }
2005            Self::StartLockCreate { path, source } => {
2006                write!(
2007                    f,
2008                    "failed to create start lock {}: {source}",
2009                    path.display()
2010                )
2011            }
2012            Self::StartLockBusy { path, attempts } => write!(
2013                f,
2014                "start lock {} remained busy after {attempts} attempts",
2015                path.display()
2016            ),
2017            Self::RunDirLockCreate { path, source } => write!(
2018                f,
2019                "refusing to start: failed to lock run directory via {}: {source}",
2020                path.display()
2021            ),
2022            Self::RunDirBusy { path, holder_pid } => {
2023                write!(
2024                    f,
2025                    "refusing to start: run directory lock {} is held by another daemon",
2026                    path.display()
2027                )?;
2028                if let Some(pid) = holder_pid {
2029                    write!(f, " (pid {pid})")?;
2030                }
2031                write!(
2032                    f,
2033                    "; a live daemon owns this data home's run state (typically one started with a different XDG_RUNTIME_DIR)"
2034                )
2035            }
2036            Self::RemoveStale { path, source } => write!(
2037                f,
2038                "failed to remove stale connection file {}: {source}",
2039                path.display()
2040            ),
2041            Self::Bind { host, port, source } if source.kind() == io::ErrorKind::AddrInUse => {
2042                write!(
2043                    f,
2044                    "port {port} in use on loopback {host}: {source}; set the port in config"
2045                )
2046            }
2047            Self::Bind { host, port, source } => {
2048                write!(f, "failed to bind loopback TCP {host}:{port}: {source}")
2049            }
2050            Self::LocalAddr { host, source } => {
2051                write!(f, "failed to read local address for {host}: {source}")
2052            }
2053            Self::DaemonConfig(err) => write!(f, "failed to load daemon config: {err}"),
2054            Self::MachineId(err) => write!(f, "refusing to start: {err}"),
2055            Self::RunDir(err) => write!(f, "refusing to start: {err}"),
2056            Self::Serve(err) => write!(f, "daemon server failed: {err}"),
2057            Self::ServeJoin(err) => write!(f, "daemon server task failed: {err}"),
2058        }
2059    }
2060}
2061
2062impl Error for BootstrapError {
2063    fn source(&self) -> Option<&(dyn Error + 'static)> {
2064        match self {
2065            Self::StartupNotServing => None,
2066            #[cfg(unix)]
2067            Self::Signal(source) => Some(source),
2068            Self::InvalidPort { source, .. } => Some(source),
2069            Self::ConnectionFileRead { source, .. }
2070            | Self::ConnectionFileWrite { source, .. }
2071            | Self::GenerateConnectionFile(source) => Some(source),
2072            Self::StartLockCreate { source, .. }
2073            | Self::RunDirLockCreate { source, .. }
2074            | Self::RemoveStale { source, .. }
2075            | Self::Bind { source, .. }
2076            | Self::LocalAddr { source, .. } => Some(source),
2077            Self::DaemonConfig(err) => Some(err),
2078            Self::MachineId(err) => Some(err),
2079            Self::RunDir(err) => Some(err),
2080            Self::Serve(err) => Some(err),
2081            Self::ServeJoin(err) => Some(err),
2082            Self::StartLockBusy { .. } | Self::RunDirBusy { .. } => None,
2083        }
2084    }
2085}
2086
2087#[cfg(test)]
2088mod tests {
2089    mod activity {
2090        include!("activity_bootstrap_tests.rs");
2091    }
2092    mod startup_order {
2093        include!("startup_order_tests.rs");
2094    }
2095    use super::*;
2096    use crate::server::ServerAuth;
2097    use cortexkit_test_support::ScratchDir;
2098    #[cfg(target_os = "linux")]
2099    use std::collections::BTreeSet;
2100    use std::sync::Mutex;
2101    #[cfg(target_os = "linux")]
2102    use subc_control::ModuleProtocol;
2103    use subc_transport::{write_atomic, MIN_KEY_LEN};
2104    use tokio::io::AsyncReadExt;
2105    use tokio::task::JoinHandle;
2106
2107    #[cfg(unix)]
2108    use std::os::unix::fs::PermissionsExt;
2109
2110    static ENV_LOCK: Mutex<()> = Mutex::new(());
2111
2112    #[test]
2113    fn normalized_build_provenance_preserves_real_values() {
2114        assert_eq!(normalized_build_provenance("abc"), Some("abc".to_string()));
2115    }
2116
2117    #[test]
2118    fn normalized_build_provenance_omits_unavailable_and_empty_values() {
2119        assert_eq!(normalized_build_provenance("unavailable"), None);
2120        assert_eq!(normalized_build_provenance(""), None);
2121    }
2122
2123    #[cfg(target_os = "linux")]
2124    fn current_cgroup_path_for_test() -> io::Result<PathBuf> {
2125        let cgroups = fs::read_to_string("/proc/self/cgroup")?;
2126        let relative = cgroups
2127            .lines()
2128            .find_map(|line| line.strip_prefix("0::"))
2129            .ok_or_else(|| {
2130                io::Error::new(io::ErrorKind::Unsupported, "cgroup v2 is unavailable")
2131            })?;
2132        Ok(Path::new("/sys/fs/cgroup").join(relative.trim_start_matches('/')))
2133    }
2134
2135    #[cfg(target_os = "linux")]
2136    fn module_cgroup_directories() -> io::Result<BTreeSet<OsString>> {
2137        let modules = current_cgroup_path_for_test()?.join("subc-modules");
2138        let entries = match fs::read_dir(modules) {
2139            Ok(entries) => entries,
2140            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
2141            Err(error) => return Err(error),
2142        };
2143        let mut directories = BTreeSet::new();
2144        for entry in entries {
2145            let entry = entry?;
2146            if entry.file_type()?.is_dir() {
2147                directories.insert(entry.file_name());
2148            }
2149        }
2150        Ok(directories)
2151    }
2152
2153    #[cfg(target_os = "linux")]
2154    async fn wait_for_path(path: &Path, task: &JoinHandle<Result<(), BootstrapError>>) {
2155        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2156        while !path.exists() && tokio::time::Instant::now() < deadline {
2157            assert!(
2158                !task.is_finished(),
2159                "daemon exited before creating {}",
2160                path.display()
2161            );
2162            sleep(Duration::from_millis(10)).await;
2163        }
2164        assert!(path.exists(), "daemon did not create {}", path.display());
2165    }
2166
2167    /// An in-process daemon with the default (disabled) cgroup placement must
2168    /// leave the host's live module cgroup tree exactly as it found it.
2169    /// Red-checking this test (making it fail) creates a
2170    /// `cgroup-isolation-probe-*` directory in the live cgroup tree, which must be removed by hand.
2171    #[cfg(target_os = "linux")]
2172    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2173    async fn run_with_config_does_not_reconcile_the_ambient_cgroup_by_default() {
2174        let temp = unique_temp_dir("bootstrap-cgroup-default-disabled");
2175        let module_id = format!("cgroup-isolation-probe-{}", process::id());
2176        let before = module_cgroup_directories().expect("read ambient module cgroups before boot");
2177        assert!(
2178            !before.contains(&OsString::from(&module_id)),
2179            "isolation probe cgroup already exists before this daemon starts"
2180        );
2181        let capture = temp.join("logs").join(format!("{module_id}.stderr.log"));
2182        let module = ConfiguredModule {
2183            carry_containment: false,
2184            module_id,
2185            program: PathBuf::from("sh"),
2186            args: vec!["-c".to_string(), "sleep 30".to_string()],
2187            env: Vec::new(),
2188            log: None,
2189            enabled: true,
2190            reserved: false,
2191            reserved_prefixes: Vec::new(),
2192            protocol: ModuleProtocol::None,
2193            overlap: Default::default(),
2194            health: HealthConfig::default(),
2195            drain_timeout_ms: None,
2196            register_timeout: crate::supervise::DEFAULT_REGISTER_TIMEOUT,
2197            route_bind_relay_timeout_ms: None,
2198            restart: RestartPolicy::default(),
2199        };
2200        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2201            .with_configured_modules([module])
2202            .with_capture_logs_dir(temp.join("logs"))
2203            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2204        let task = tokio::spawn(run_with_config(config));
2205
2206        wait_for_path(&capture, &task).await;
2207        let after = module_cgroup_directories().expect("read ambient module cgroups after boot");
2208
2209        task.abort();
2210        assert!(task
2211            .await
2212            .expect_err("aborted daemon task must cancel")
2213            .is_cancelled());
2214        assert_eq!(
2215            before, after,
2216            "an in-process daemon must not create or reconcile ambient module cgroups"
2217        );
2218    }
2219
2220    #[cfg(target_os = "linux")]
2221    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2222    async fn explicit_cgroup_root_is_prepared_inside_the_fixture_tree() {
2223        let temp = unique_temp_dir("bootstrap-cgroup-explicit-root");
2224        let cgroup_root = temp.join("cgroup");
2225        fs::create_dir(&cgroup_root).expect("create scratch cgroup root");
2226        fs::write(cgroup_root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
2227        let modules = cgroup_root.join("subc-modules");
2228        let config = config_with_a_short_socket(BootstrapConfig::new(temp.join("c"), 0))
2229            .with_cgroup_placement(CgroupPlacementConfig::Root(cgroup_root))
2230            .with_terminal_journal_path(temp.join("terminals.jsonl"));
2231        let task = tokio::spawn(run_with_config(config));
2232
2233        wait_for_path(&modules, &task).await;
2234
2235        task.abort();
2236        assert!(task
2237            .await
2238            .expect_err("aborted daemon task must cancel")
2239            .is_cancelled());
2240        assert!(
2241            fs::read_dir(&modules)
2242                .expect("read prepared modules directory")
2243                .next()
2244                .is_none(),
2245            "the delegation probe must clean up after itself"
2246        );
2247    }
2248
2249    struct EnvGuard {
2250        key: &'static str,
2251        previous: Option<OsString>,
2252    }
2253
2254    impl EnvGuard {
2255        fn set(key: &'static str, value: &Path) -> Self {
2256            let previous = env::var_os(key);
2257            env::set_var(key, value);
2258            Self { key, previous }
2259        }
2260
2261        fn set_str(key: &'static str, value: &str) -> Self {
2262            let previous = env::var_os(key);
2263            env::set_var(key, value);
2264            Self { key, previous }
2265        }
2266
2267        fn unset(key: &'static str) -> Self {
2268            let previous = env::var_os(key);
2269            env::remove_var(key);
2270            Self { key, previous }
2271        }
2272    }
2273
2274    impl Drop for EnvGuard {
2275        fn drop(&mut self) {
2276            match &self.previous {
2277                Some(value) => env::set_var(self.key, value),
2278                None => env::remove_var(self.key),
2279            }
2280        }
2281    }
2282
2283    fn unique_temp_dir(name: &str) -> ScratchDir {
2284        ScratchDir::new(name)
2285    }
2286
2287    fn temp_connection_file_path(name: &str) -> (ScratchDir, PathBuf) {
2288        let dir = unique_temp_dir(name);
2289        let path = dir.join("conn.json");
2290        (dir, path)
2291    }
2292
2293    fn auth_for(info: &ConnectionInfo) -> ServerAuth {
2294        ServerAuth::new(info.key.clone(), info.daemon_id, info.daemon_ver.clone())
2295    }
2296
2297    fn start_server(bound: BoundDaemon) -> JoinHandle<Result<(), ServerError>> {
2298        let auth = auth_for(&bound.connection_info);
2299        tokio::spawn(serve_listeners(
2300            bound.listeners,
2301            Arc::new(Router::with_default_self_handler()),
2302            auth,
2303        ))
2304    }
2305
2306    fn expect_bound(outcome: Outcome) -> BoundDaemon {
2307        match outcome {
2308            Outcome::Bound(bound) => bound,
2309            Outcome::AlreadyRunning => panic!("fresh connection file unexpectedly had a daemon"),
2310        }
2311    }
2312
2313    async fn connect_from_info(conn: &ConnectionInfo) -> io::Result<TcpStream> {
2314        let endpoint = conn
2315            .endpoints
2316            .first()
2317            .expect("test connection file should have an endpoint");
2318        let ip: IpAddr = endpoint.host.parse().unwrap();
2319        TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await
2320    }
2321
2322    fn make_connection_info(port: u16) -> ConnectionInfo {
2323        ConnectionInfo {
2324            schema: SCHEMA_VERSION,
2325            wire_version: Some(PROTOCOL_VERSION),
2326            endpoints: vec![Endpoint {
2327                host: "127.0.0.1".to_owned(),
2328                port,
2329            }],
2330            key: generate_key().unwrap(),
2331            daemon_id: generate_daemon_id().unwrap(),
2332            pid: process::id(),
2333            daemon_ver: "test-subc".to_owned(),
2334        }
2335    }
2336
2337    fn write_raw_owner_only_connection_file(path: &Path, contents: &[u8]) {
2338        fs::write(path, contents).unwrap();
2339        #[cfg(unix)]
2340        fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap();
2341    }
2342
2343    fn assert_owner_only_connection_file(path: &Path) {
2344        // `path` is only inspected on Unix (mode bits); on Windows the owner-only
2345        // guarantee comes from the inherited %TEMP% ACL, nothing to assert here.
2346        #[cfg(unix)]
2347        {
2348            let mode = fs::metadata(path).unwrap().permissions().mode() & 0o777;
2349            assert_eq!(mode, 0o600);
2350        }
2351        #[cfg(not(unix))]
2352        let _ = path;
2353    }
2354
2355    #[test]
2356    fn connection_file_path_uses_xdg_runtime_dir_when_set() {
2357        let _env_lock = ENV_LOCK.lock().unwrap();
2358        let runtime_dir = unique_temp_dir("xdg-runtime");
2359        let _xdg = EnvGuard::set("XDG_RUNTIME_DIR", runtime_dir.path());
2360
2361        assert_eq!(
2362            connection_file_path(),
2363            runtime_dir.join(CONNECTION_FILE_NAME)
2364        );
2365    }
2366
2367    #[test]
2368    fn connection_file_path_source_is_xdg_runtime_dir_when_set() {
2369        let runtime_dir = OsString::from("/run/user/1000");
2370
2371        let (path, source) = connection_file_path_with_source(Some(runtime_dir));
2372
2373        assert_eq!(
2374            path,
2375            PathBuf::from("/run/user/1000").join(CONNECTION_FILE_NAME)
2376        );
2377        assert_eq!(source, ConnectionFileSource::XdgRuntimeDir);
2378    }
2379
2380    #[test]
2381    fn connection_file_path_falls_back_to_temp_dir_with_user_token_when_xdg_unset() {
2382        let _env_lock = ENV_LOCK.lock().unwrap();
2383        let _xdg = EnvGuard::unset("XDG_RUNTIME_DIR");
2384
2385        assert_eq!(
2386            connection_file_path(),
2387            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2388        );
2389    }
2390
2391    #[test]
2392    fn connection_file_path_source_is_temp_dir_when_xdg_unset() {
2393        let (path, source) = connection_file_path_with_source(None);
2394
2395        assert_eq!(
2396            path,
2397            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2398        );
2399        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2400    }
2401
2402    /// Concurrent callers must derive one token. The former temp-file uid probe
2403    /// could fail transiently (same-tick name collision, fd exhaustion) and send
2404    /// the loser down the env-derived fallback with a different identity for the
2405    /// same user; this fence keeps identity independent of filesystem luck.
2406    #[test]
2407    fn user_connection_token_is_stable_under_concurrent_callers() {
2408        let expected = user_connection_token();
2409        let workers: Vec<_> = (0..32)
2410            .map(|_| {
2411                std::thread::spawn(|| (0..40).map(|_| user_connection_token()).collect::<Vec<_>>())
2412            })
2413            .collect();
2414        for worker in workers {
2415            for token in worker.join().expect("probe thread") {
2416                assert_eq!(token, expected, "token diverged under concurrent probes");
2417            }
2418        }
2419    }
2420
2421    #[test]
2422    fn connection_file_path_source_is_temp_dir_when_xdg_empty() {
2423        let (path, source) = connection_file_path_with_source(Some(OsString::new()));
2424
2425        assert_eq!(
2426            path,
2427            env::temp_dir().join(format!("subc-{}.connection.json", user_connection_token()))
2428        );
2429        assert_eq!(source, ConnectionFileSource::TempDirFallback);
2430    }
2431
2432    /// The shipped binary is the only caller that journals terminal exits into
2433    /// the real run directory, so its constructor must supply that path itself;
2434    /// the in-process default leaves it unset.
2435    #[test]
2436    fn daemon_binary_config_journals_and_captures_into_the_run_dir() {
2437        let _env_lock = ENV_LOCK.lock().unwrap();
2438        let root = unique_temp_dir("daemon-binary-config");
2439        let data_home = root.join("data");
2440        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
2441        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2442        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2443
2444        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
2445
2446        let run_dir = data_home.join("cortexkit").join("run");
2447        assert_eq!(
2448            config.terminal_journal_path,
2449            Some(run_dir.join("terminals.jsonl"))
2450        );
2451        assert_eq!(config.capture_logs_dir, Some(run_dir.join("logs")));
2452        assert_eq!(
2453            BootstrapConfig::new(root.join("connection.json"), 0).terminal_journal_path,
2454            None,
2455            "an in-process config must not journal anywhere unless asked to"
2456        );
2457    }
2458
2459    #[test]
2460    fn daemon_binary_config_refuses_a_relative_data_home() {
2461        let _env_lock = ENV_LOCK.lock().unwrap();
2462        let root = unique_temp_dir("daemon-binary-relative-data");
2463        let _data = EnvGuard::set_str("XDG_DATA_HOME", "relative-data-home");
2464        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
2465        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2466
2467        let error = BootstrapConfig::from_env_for_daemon_binary()
2468            .expect_err("a relative data home must refuse the daemon binary's config");
2469        assert!(
2470            matches!(error, BootstrapError::RunDir(_)),
2471            "expected a run-directory refusal, got {error}"
2472        );
2473        assert!(error.to_string().contains("XDG_DATA_HOME"), "{error}");
2474    }
2475
2476    #[test]
2477    fn privileged_operator_prompt_is_loaded_by_both_bootstrap_config_paths() {
2478        let _env_lock = ENV_LOCK.lock().unwrap();
2479        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2480        let (_dir, conn_path) = temp_connection_file_path("privileged-operator-prompt");
2481        let config_path = conn_path.with_file_name("subc.jsonc");
2482        for (doc, expected) in [
2483            (None, daemon_config::PrivilegedOperatorPrompt::Log),
2484            (
2485                Some(r#"{"version":1}"#),
2486                daemon_config::PrivilegedOperatorPrompt::Log,
2487            ),
2488            (
2489                Some(r#"{"version":1,"privileged_operator_prompt":"log"}"#),
2490                daemon_config::PrivilegedOperatorPrompt::Log,
2491            ),
2492            (
2493                Some(r#"{"version":1,"privileged_operator_prompt":"enforce"}"#),
2494                daemon_config::PrivilegedOperatorPrompt::Enforce,
2495            ),
2496        ] {
2497            if let Some(doc) = doc {
2498                fs::write(&config_path, doc).unwrap();
2499            }
2500            assert_eq!(
2501                BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2502                    .unwrap()
2503                    .privileged_operator_prompt,
2504                expected
2505            );
2506            assert_eq!(
2507                BootstrapConfig::new(&conn_path, 0)
2508                    .with_daemon_config_path(&config_path)
2509                    .unwrap()
2510                    .privileged_operator_prompt,
2511                expected
2512            );
2513        }
2514    }
2515
2516    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2517    async fn privileged_operator_prompt_is_snapshotted_by_running_daemon() {
2518        let dir = unique_temp_dir("privileged-operator-prompt-bootstrap");
2519        let connection_path = dir.join("connection.json");
2520        let config_path = dir.join("subc.jsonc");
2521        fs::write(
2522            &config_path,
2523            r#"{"version":1,"privileged_operator_prompt":"enforce"}"#,
2524        )
2525        .unwrap();
2526        let config = config_with_a_short_socket(BootstrapConfig::new(&connection_path, 0))
2527            .with_daemon_config_path(&config_path)
2528            .unwrap();
2529        let connection_path = config.connection_file_path.clone();
2530        let daemon = AbortOnDrop::new(tokio::spawn(run_with_config(config)));
2531        tokio::time::timeout(Duration::from_secs(5), async {
2532            while !connection_path.exists() {
2533                assert!(
2534                    !daemon.handle.is_finished(),
2535                    "daemon exited before publishing"
2536                );
2537                tokio::task::yield_now().await;
2538            }
2539        })
2540        .await
2541        .unwrap();
2542        let info = connection_file::read(&connection_path).unwrap();
2543        let mut client = connect_from_info(&info).await.unwrap();
2544        authenticate_client(&mut client, &info, Duration::from_secs(5))
2545            .await
2546            .unwrap();
2547        for (correlation, mode, expected) in [
2548            (1, "enforce", Vec::new()),
2549            (2, "log", vec!["privileged_operator_prompt"]),
2550            (3, "log", vec!["privileged_operator_prompt"]),
2551        ] {
2552            fs::write(
2553                &config_path,
2554                serde_json::json!({"version":1,"privileged_operator_prompt":mode}).to_string(),
2555            )
2556            .unwrap();
2557            let frame = crate::Frame::build(
2558                subc_protocol::FrameType::Request,
2559                subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
2560                0,
2561                0,
2562                correlation,
2563                serde_json::to_vec(&subc_control::ClientControlRequest::SupervisorRescan {
2564                    preview: true,
2565                })
2566                .unwrap(),
2567            )
2568            .unwrap();
2569            crate::write_frame(&mut client, &frame).await.unwrap();
2570            let reply =
2571                tokio::time::timeout(Duration::from_secs(5), crate::read_frame(&mut client))
2572                    .await
2573                    .unwrap()
2574                    .unwrap()
2575                    .unwrap();
2576            assert_eq!(reply.header.ty, subc_protocol::FrameType::Response);
2577            assert_eq!(reply.header.corr, correlation);
2578            let subc_control::ClientControlResponse::SupervisorRescan { result } =
2579                serde_json::from_slice(&reply.body).unwrap()
2580            else {
2581                panic!("expected a rescan response");
2582            };
2583            assert_eq!(result.restart_required, expected);
2584        }
2585    }
2586
2587    #[test]
2588    fn configured_port_uses_default_config_and_env_override() {
2589        let _env_lock = ENV_LOCK.lock().unwrap();
2590        let (_dir, conn_path) = temp_connection_file_path("daemon-config-port");
2591        let config_path = conn_path.with_file_name("subc.jsonc");
2592
2593        let _port = EnvGuard::unset(SUBC_PORT_ENV);
2594        assert_eq!(
2595            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2596                .unwrap()
2597                .port,
2598            DEFAULT_SUBC_PORT
2599        );
2600
2601        fs::write(&config_path, r#"{ "version": 1, "port": 8123 }"#).unwrap();
2602        assert_eq!(
2603            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2604                .unwrap()
2605                .port,
2606            8123
2607        );
2608
2609        let _port = EnvGuard::set_str(SUBC_PORT_ENV, "9012");
2610        assert_eq!(
2611            BootstrapConfig::from_env_with_daemon_config_path(&config_path)
2612                .unwrap()
2613                .port,
2614            9012
2615        );
2616    }
2617
2618    #[tokio::test]
2619    async fn singleton_does_not_accept_a_peer_that_answers_once_then_exits() {
2620        let (_dir, path) = temp_connection_file_path("departing-peer");
2621        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2622        let info = make_connection_info(listener.local_addr().unwrap().port());
2623        write_connection_file(&path, &info, None).unwrap();
2624        let peer = tokio::spawn(async move {
2625            let (mut stream, _) = listener.accept().await.unwrap();
2626            subc_transport::authenticate_server(
2627                &mut stream,
2628                &info.key,
2629                &info.daemon_id,
2630                &info.daemon_ver,
2631                PROBE_AUTH_DEADLINE,
2632            )
2633            .await
2634            .unwrap();
2635        });
2636        let outcome = ensure_singleton(&path, 0).await.unwrap();
2637        peer.await.unwrap();
2638        assert!(
2639            matches!(outcome, Outcome::Bound(_)),
2640            "a departing peer is not AlreadyRunning"
2641        );
2642    }
2643
2644    #[tokio::test]
2645    async fn second_singleton_probe_against_served_tcp_daemon_reports_already_running() {
2646        let (_dir, path) = temp_connection_file_path("already-running");
2647
2648        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2649        let server = start_server(bound);
2650
2651        let second = ensure_singleton(&path, 0).await.unwrap();
2652        assert!(matches!(second, Outcome::AlreadyRunning));
2653
2654        server.abort();
2655        let _ = server.await;
2656    }
2657
2658    #[tokio::test]
2659    async fn daemon_connection_file_publishes_protocol_wire_version() {
2660        let (_dir, path) = temp_connection_file_path("wire-version");
2661        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2662        assert_eq!(bound.connection_info.wire_version, Some(PROTOCOL_VERSION));
2663        assert_eq!(
2664            connection_file::read(&path).unwrap().wire_version,
2665            Some(PROTOCOL_VERSION)
2666        );
2667
2668        drop(bound.listeners);
2669    }
2670
2671    #[tokio::test]
2672    async fn stale_unbound_connection_file_is_reclaimed() {
2673        let (_dir, path) = temp_connection_file_path("stale-reclaim");
2674        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2675        let stale_port = stale.local_addr().unwrap().port();
2676        drop(stale);
2677        let stale_info = make_connection_info(stale_port);
2678        write_atomic(&path, &stale_info).unwrap();
2679
2680        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2681        assert_ne!(bound.connection_info.key, stale_info.key);
2682        drop(bound.listeners);
2683    }
2684
2685    #[cfg(unix)]
2686    #[tokio::test]
2687    async fn ensure_singleton_reclaims_insecure_connection_file() {
2688        let (_dir, path) = temp_connection_file_path("insecure-reclaim");
2689        let stale = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2690        let stale_port = stale.local_addr().unwrap().port();
2691        drop(stale);
2692        let stale_info = make_connection_info(stale_port);
2693        write_atomic(&path, &stale_info).unwrap();
2694        fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
2695
2696        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2697        assert_ne!(bound.connection_info.key, stale_info.key);
2698        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2699        assert_owner_only_connection_file(&path);
2700
2701        drop(bound.listeners);
2702    }
2703
2704    #[tokio::test]
2705    async fn ensure_singleton_reclaims_non_loopback_connection_file() {
2706        let (_dir, path) = temp_connection_file_path("non-loopback-reclaim");
2707        let mut stale_info = make_connection_info(8757);
2708        stale_info.endpoints = vec![Endpoint {
2709            host: "192.0.2.10".to_owned(),
2710            port: 8757,
2711        }];
2712        write_atomic(&path, &stale_info).unwrap();
2713
2714        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2715        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
2716        assert_ne!(bound.connection_info.key, stale_info.key);
2717        assert_ne!(bound.connection_info.daemon_id, stale_info.daemon_id);
2718        assert!(bound
2719            .connection_info
2720            .endpoints
2721            .iter()
2722            .all(|endpoint| endpoint.host.parse::<IpAddr>().unwrap().is_loopback()));
2723        assert_owner_only_connection_file(&path);
2724
2725        drop(bound.listeners);
2726    }
2727
2728    #[tokio::test]
2729    async fn ensure_singleton_reclaims_invalid_connection_file_shapes() {
2730        let mut unsupported_schema = make_connection_info(8757);
2731        unsupported_schema.schema = SCHEMA_VERSION + 1;
2732
2733        let mut empty_endpoints = make_connection_info(8757);
2734        empty_endpoints.endpoints.clear();
2735
2736        let mut short_key = make_connection_info(8757);
2737        short_key.key = vec![0x5A; MIN_KEY_LEN - 1];
2738
2739        let cases = vec![
2740            (
2741                "unsupported-schema",
2742                serde_json::to_vec(&unsupported_schema).unwrap(),
2743                Some(unsupported_schema),
2744            ),
2745            (
2746                "empty-endpoints",
2747                serde_json::to_vec(&empty_endpoints).unwrap(),
2748                Some(empty_endpoints),
2749            ),
2750            (
2751                "short-key",
2752                serde_json::to_vec(&short_key).unwrap(),
2753                Some(short_key),
2754            ),
2755            ("invalid-json", b"{not valid connection json".to_vec(), None),
2756        ];
2757
2758        for (label, contents, old_info) in cases {
2759            let (_dir, path) = temp_connection_file_path(label);
2760            write_raw_owner_only_connection_file(&path, &contents);
2761
2762            let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2763            if let Some(old_info) = old_info {
2764                assert_ne!(bound.connection_info.key, old_info.key, "{label}");
2765                assert_ne!(
2766                    bound.connection_info.daemon_id, old_info.daemon_id,
2767                    "{label}"
2768                );
2769            }
2770            assert!(bound.connection_info.key.len() >= MIN_KEY_LEN, "{label}");
2771            assert_ne!(bound.connection_info.daemon_id, [0u8; 16], "{label}");
2772            assert_owner_only_connection_file(&path);
2773
2774            drop(bound.listeners);
2775        }
2776    }
2777
2778    #[tokio::test]
2779    async fn foreign_reused_port_connection_file_is_reclaimed_after_auth_probe_fails() {
2780        let (_dir, path) = temp_connection_file_path("foreign-reclaim");
2781        let foreign = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2782        let foreign_port = foreign.local_addr().unwrap().port();
2783        write_atomic(&path, &make_connection_info(foreign_port)).unwrap();
2784        let foreign_task = tokio::spawn(async move {
2785            if let Ok((mut stream, _)) = foreign.accept().await {
2786                let mut buf = [0u8; 64];
2787                let _ = stream.read(&mut buf).await;
2788            }
2789        });
2790
2791        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
2792        assert!(bound
2793            .connection_info
2794            .endpoints
2795            .iter()
2796            .all(|endpoint| endpoint.port != foreign_port));
2797
2798        drop(bound.listeners);
2799        let _ = foreign_task.await;
2800    }
2801
2802    #[tokio::test]
2803    async fn stale_start_lock_file_is_reclaimable() {
2804        let (_dir, path) = temp_connection_file_path("start-lock-stale-file");
2805        let lock_path = start_lock_path(&path);
2806        drop(open_owner_only_lock(&lock_path).unwrap());
2807        assert!(lock_path.is_file());
2808
2809        let lock = StartLock::acquire(&path).await.unwrap();
2810        assert!(lock_path.is_file());
2811
2812        drop(lock);
2813        assert!(lock_path.is_file());
2814    }
2815
2816    #[tokio::test]
2817    async fn held_start_lock_blocks_second_acquire_until_release() {
2818        let (_dir, path) = temp_connection_file_path("start-lock-held");
2819        let lock_path = start_lock_path(&path);
2820        let first = StartLock::acquire(&path).await.unwrap();
2821
2822        let err = match StartLock::acquire(&path).await {
2823            Ok(_) => panic!("second acquire while held must stay busy"),
2824            Err(err) => err,
2825        };
2826        assert!(matches!(
2827            err,
2828            BootstrapError::StartLockBusy {
2829                ref path,
2830                attempts: START_LOCK_RETRIES,
2831            } if path == &lock_path
2832        ));
2833
2834        drop(first);
2835
2836        let second = StartLock::acquire(&path)
2837            .await
2838            .expect("released advisory lock should be reclaimable");
2839        drop(second);
2840    }
2841
2842    #[tokio::test]
2843    async fn bind_conflict_on_fixed_port_fails_loud_without_reselecting() {
2844        let (_dir, path) = temp_connection_file_path("bind-conflict");
2845        let occupied = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
2846        let occupied_port = occupied.local_addr().unwrap().port();
2847
2848        let err = ensure_singleton(&path, occupied_port).await.unwrap_err();
2849        assert!(matches!(
2850            err,
2851            BootstrapError::Bind { ref source, .. } if source.kind() == io::ErrorKind::AddrInUse
2852        ));
2853        assert!(err.to_string().contains("set the port in config"));
2854
2855        drop(occupied);
2856    }
2857
2858    #[tokio::test]
2859    async fn ephemeral_ipv6_collision_still_serves_ipv4() {
2860        let (listeners, endpoints) = bind_loopback_with(0, |port| async move {
2861            let occupied = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await?;
2862            let result = TcpListener::bind((Ipv6Addr::LOCALHOST, port)).await;
2863            drop(occupied);
2864            result
2865        })
2866        .await
2867        .expect("an incidental IPv6 collision must not prevent an ephemeral daemon from starting");
2868        assert_eq!(endpoints.len(), 1);
2869        assert_eq!(endpoints[0].host, "127.0.0.1");
2870        TcpStream::connect((Ipv4Addr::LOCALHOST, endpoints[0].port))
2871            .await
2872            .unwrap();
2873        drop(listeners);
2874    }
2875
2876    #[tokio::test]
2877    async fn unpublished_boot_holds_singleton_lock_until_publication() {
2878        let dir = ScratchDir::new("ul");
2879        let path = dir.join("c");
2880        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2881        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2882        assert!(!path.exists());
2883        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2884        assert!(
2885            matches!(FileExt::try_lock(&contender), Err(TryLockError::WouldBlock)),
2886            "an unpublished boot must keep its singleton claim"
2887        );
2888        drop(bound);
2889        FileExt::try_lock(&contender).unwrap();
2890    }
2891
2892    #[tokio::test]
2893    async fn start_lock_drop_releases_claim_with_a_shared_file_description() {
2894        let dir = ScratchDir::new("sl");
2895        let path = dir.join("c");
2896        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0));
2897        let bound = expect_bound(ensure_singleton_inner(config, false).await.unwrap());
2898        // try_clone shares the open file description just as fork does,
2899        // without relying on the timing of another test's child spawn.
2900        let inherited = bound
2901            .publication_lock
2902            .as_ref()
2903            .unwrap()
2904            ._file
2905            .try_clone()
2906            .unwrap();
2907        let contender = open_owner_only_lock(&start_lock_path(&path)).unwrap();
2908        assert!(matches!(
2909            FileExt::try_lock(&contender),
2910            Err(TryLockError::WouldBlock)
2911        ));
2912        drop(bound);
2913        FileExt::try_lock(&contender).expect("dropping bootstrap releases even an inherited lock");
2914        drop(inherited);
2915    }
2916
2917    #[tokio::test]
2918    async fn readiness_probe_requires_an_authenticating_server() {
2919        let dir = ScratchDir::new("unserved-readiness");
2920        let bound = expect_bound(
2921            ensure_singleton_inner(BootstrapConfig::new(dir.join("connection.json"), 0), false)
2922                .await
2923                .unwrap(),
2924        );
2925        assert!(matches!(
2926            verify_serving(&bound.connection_info).await,
2927            Err(BootstrapError::StartupNotServing)
2928        ));
2929    }
2930
2931    #[cfg(unix)]
2932    #[tokio::test]
2933    async fn orphan_sweep_finishes_before_connection_file_publication() {
2934        use crate::live_children::{ExecutableIdentity, LiveChild};
2935        let dir = ScratchDir::new("pas");
2936        // A one-letter connection file name, and no long socket path: the sweep
2937        // this test covers runs before the local endpoint is bound, so a longer
2938        // name would only add a way for the test to fail on something else.
2939        let path = dir.join("c");
2940        let record = dir.join("live-children.json");
2941        let ready = dir.join("ready");
2942        let mut child = std::process::Command::new("/bin/sh")
2943            .args([
2944                "-c",
2945                "trap '' TERM; : > \"$0\"; read -r _",
2946                ready.to_str().unwrap(),
2947            ])
2948            .stdin(std::process::Stdio::piped())
2949            .env("XDG_DATA_HOME", dir.path())
2950            .env("XDG_RUNTIME_DIR", dir.path())
2951            .env("XDG_CONFIG_HOME", dir.path())
2952            .spawn()
2953            .unwrap();
2954        timeout(Duration::from_secs(5), async {
2955            while !ready.exists() {
2956                sleep(Duration::from_millis(5)).await;
2957            }
2958        })
2959        .await
2960        .unwrap();
2961        let observed = subc_os::Process::open(child.id())
2962            .unwrap()
2963            .unwrap()
2964            .observe()
2965            .unwrap();
2966        crate::live_children::write_record(
2967            &record,
2968            &[LiveChild {
2969                container: None,
2970                module_id: "old-sleep".into(),
2971                pid: child.id(),
2972                protocol: subc_control::ModuleProtocol::None,
2973                start_time: Some(observed.start_time),
2974                executable: observed.executable.map(ExecutableIdentity::from),
2975                cgroup_name: None,
2976            }],
2977        )
2978        .unwrap();
2979        let config = config_with_a_short_socket(BootstrapConfig::new(&path, 0))
2980            .with_live_children_record(&record);
2981        let daemon = tokio::spawn(run_with_config(config));
2982        timeout(Duration::from_secs(5), async {
2983            while !path.exists() {
2984                sleep(Duration::from_millis(5)).await;
2985            }
2986        })
2987        .await
2988        .unwrap();
2989        assert!(path.exists());
2990        let ended = child.try_wait().unwrap().is_some();
2991        if !ended {
2992            child.kill().unwrap();
2993        }
2994        child.wait().unwrap();
2995        if ended {
2996            assert!(matches!(probe_existing(&path).await.unwrap(), Probe::Live));
2997        }
2998        daemon.abort();
2999        let _ = daemon.await;
3000        assert!(
3001            ended,
3002            "the published daemon must not still owe its orphan sweep"
3003        );
3004    }
3005
3006    /// Publishing the connection file fsyncs, which can take seconds on a busy
3007    /// disk, so the daemon must do it off the async worker thread. The hook
3008    /// blocks inside the write until another task on the same single-threaded
3009    /// runtime has run. Writing on the worker thread would starve that task: the
3010    /// hook then gives up after its deadline and the test fails, as it does if
3011    /// startup publishes without going through the hooked write at all.
3012    #[tokio::test(flavor = "current_thread")]
3013    async fn connection_file_publication_leaves_the_async_runtime_free() {
3014        use std::sync::atomic::{AtomicBool, Ordering};
3015
3016        let (_dir, path) = temp_connection_file_path("publish-off-runtime");
3017        let write_started = Arc::new(AtomicBool::new(false));
3018        let (progress_tx, progress_rx) = std::sync::mpsc::channel::<()>();
3019        let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::<bool>();
3020        let hook_started = Arc::clone(&write_started);
3021        connection_file_write_hook::install(&path, move || {
3022            hook_started.store(true, Ordering::SeqCst);
3023            let progressed = progress_rx.recv_timeout(Duration::from_secs(5)).is_ok();
3024            let _ = verdict_tx.send(progressed);
3025        });
3026        // Signals only after the write has started, so a message can only come
3027        // from this task running while the write is still blocked.
3028        let ticker = tokio::spawn(async move {
3029            loop {
3030                if write_started.load(Ordering::SeqCst) {
3031                    let _ = progress_tx.send(());
3032                    return;
3033                }
3034                sleep(Duration::from_millis(5)).await;
3035            }
3036        });
3037
3038        let daemon = tokio::spawn(run_with_config(BootstrapConfig::new(&path, 0)));
3039        let published = timeout(Duration::from_secs(15), async {
3040            while !path.exists() && !daemon.is_finished() {
3041                sleep(Duration::from_millis(5)).await;
3042            }
3043        })
3044        .await;
3045        let daemon_exited = daemon.is_finished();
3046        daemon.abort();
3047        let daemon_result = daemon.await;
3048        ticker.abort();
3049
3050        assert!(
3051            published.is_ok() && !daemon_exited,
3052            "the daemon must start and publish its connection file: {daemon_result:?}"
3053        );
3054        match verdict_rx.try_recv() {
3055            Ok(true) => {}
3056            Ok(false) => panic!(
3057                "no other task on the runtime ran while the connection file was being written: the write is blocking the async worker thread"
3058            ),
3059            Err(_) => panic!("startup published the connection file without the hooked write"),
3060        }
3061    }
3062
3063    /// A write that never reports back must stop startup with an error naming
3064    /// the connection file, never be taken as published.
3065    #[tokio::test]
3066    async fn a_panicking_connection_file_write_fails_startup() {
3067        let (_dir, path) = temp_connection_file_path("publish-panics");
3068        connection_file_write_hook::install(&path, || {
3069            panic!("deliberate panic inside the connection-file write")
3070        });
3071
3072        let error = timeout(
3073            Duration::from_secs(15),
3074            run_with_config(BootstrapConfig::new(&path, 0)),
3075        )
3076        .await
3077        .expect("startup must end once the write task fails")
3078        .expect_err("startup must fail when the connection-file write panics");
3079
3080        assert!(
3081            matches!(&error, BootstrapError::ConnectionFileWrite { path: failed, .. } if failed == &path),
3082            "expected a connection-file write error, got {error:?}"
3083        );
3084        let message = error.to_string();
3085        assert!(message.contains(&path.display().to_string()), "{message}");
3086        assert!(message.contains("panicked"), "{message}");
3087        assert!(!path.exists(), "no connection file may be published");
3088    }
3089
3090    #[cfg(target_os = "macos")]
3091    #[test]
3092    fn nofile_raise_worker() {
3093        let Some(path) = std::env::var_os("SUBC_NOFILE_TEST_FILE") else {
3094            return;
3095        };
3096        let (_, hard) = rlimit::Resource::NOFILE.get().unwrap();
3097        rlimit::Resource::NOFILE.set(256, hard).unwrap();
3098        raise_nofile_limit_with_ceiling(Some(10240));
3099        std::fs::write(path, rlimit::Resource::NOFILE.get().unwrap().0.to_string()).unwrap();
3100    }
3101
3102    #[cfg(target_os = "macos")]
3103    #[test]
3104    fn macos_nofile_raise_clamps_to_kernel_ceiling() {
3105        let dir = ScratchDir::new("nofile-ceiling");
3106        let result = dir.join("soft");
3107        let status = std::process::Command::new(std::env::current_exe().unwrap())
3108            .args(["--exact", "bootstrap::tests::nofile_raise_worker"])
3109            .env("SUBC_NOFILE_TEST_FILE", &result)
3110            .env("XDG_DATA_HOME", dir.path())
3111            .env("XDG_RUNTIME_DIR", dir.path())
3112            .env("XDG_CONFIG_HOME", dir.path())
3113            .status()
3114            .unwrap();
3115        assert!(status.success());
3116        let soft: u64 = std::fs::read_to_string(result).unwrap().parse().unwrap();
3117        assert!(
3118            soft > 256 && soft <= 10240,
3119            "the inherited limit must be raised within the kernel ceiling; got {soft}"
3120        );
3121    }
3122
3123    #[tokio::test]
3124    async fn key_rotation_republishes_new_material_and_old_file_fails_auth() {
3125        let (_dir, path) = temp_connection_file_path("key-rotation");
3126        let first = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3127        let old_info = first.connection_info.clone();
3128        let fixed_port = old_info.endpoints[0].port;
3129        drop(first.listeners);
3130
3131        let second = expect_bound(ensure_singleton(&path, fixed_port).await.unwrap());
3132        let new_info = second.connection_info.clone();
3133        assert_ne!(old_info.key, new_info.key);
3134        assert_ne!(old_info.daemon_id, new_info.daemon_id);
3135        let server = start_server(second);
3136
3137        let mut old_stream = connect_from_info(&old_info).await.unwrap();
3138        let old_auth = authenticate_client(&mut old_stream, &old_info, PROBE_AUTH_DEADLINE).await;
3139        assert!(
3140            old_auth.is_err(),
3141            "old key must not authenticate after restart"
3142        );
3143
3144        let reread = connection_file::read(&path).unwrap();
3145        let mut new_stream = connect_from_info(&reread).await.unwrap();
3146        authenticate_client(&mut new_stream, &reread, PROBE_AUTH_DEADLINE)
3147            .await
3148            .unwrap();
3149
3150        server.abort();
3151        let _ = server.await;
3152    }
3153
3154    #[cfg(unix)]
3155    #[tokio::test]
3156    async fn published_connection_file_permissions_are_owner_only() {
3157        let (_dir, path) = temp_connection_file_path("permissions");
3158        let bound = expect_bound(ensure_singleton(&path, 0).await.unwrap());
3159
3160        let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777;
3161        assert_eq!(mode, 0o600);
3162
3163        drop(bound.listeners);
3164    }
3165
3166    /// A config with the socket in `dir`'s own short subdirectory.
3167    ///
3168    /// A Unix socket path is limited to just over a hundred bytes, and this
3169    /// harness's temporary directories use nearly all of that, so a test that
3170    /// boots a real daemon passes a short directory and a one-letter connection
3171    /// file name; anything longer makes the bind refuse. On Windows there is no
3172    /// socket path to keep short, so the config is returned unchanged.
3173    fn config_with_a_short_socket(config: BootstrapConfig) -> BootstrapConfig {
3174        let dir = config
3175            .connection_file_path
3176            .parent()
3177            .map(Path::to_path_buf)
3178            .unwrap_or_else(|| PathBuf::from("."));
3179        #[cfg(unix)]
3180        let config = BootstrapConfig {
3181            connection_file_path: dir.join("c"),
3182            local_listener: LocalListenerConfig::in_socket_dir(dir.join("s")),
3183            ..config
3184        };
3185        #[cfg(not(unix))]
3186        let _ = dir;
3187        config
3188    }
3189
3190    /// The shipped binary puts the socket in its own run directory. The file is
3191    /// named from the connection file's path, so a restart finds it again.
3192    #[cfg(unix)]
3193    #[test]
3194    fn the_daemon_binary_puts_its_socket_in_the_run_directory() {
3195        let _env_lock = ENV_LOCK.lock().unwrap();
3196        let root = ScratchDir::new("binary-local-socket");
3197        let data_home = root.join("data");
3198        let _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3199        let _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3200        let _port = EnvGuard::unset(SUBC_PORT_ENV);
3201
3202        let config = BootstrapConfig::from_env_for_daemon_binary().unwrap();
3203
3204        let run_dir = data_home.join("cortexkit").join("run");
3205        assert_eq!(
3206            config.local_listener.configured_dir(),
3207            Some(run_dir.as_path()),
3208            "the binary serves from its run directory"
3209        );
3210        let socket = local_listener::socket_path_for(Some(&run_dir), &config.connection_file_path)
3211            .expect("the binary config names a directory");
3212        assert_eq!(socket.parent().unwrap(), run_dir);
3213        assert_eq!(
3214            socket.file_name().unwrap().to_string_lossy(),
3215            "subc.sock",
3216            "a fixed name: the run directory belongs to one data home"
3217        );
3218    }
3219
3220    /// An in-process daemon that names no directory serves TCP only: it publishes
3221    /// no local endpoint, creates nothing under the data home, and says nothing is
3222    /// unavailable, because nothing was asked for.
3223    /// The environment lock covers only the setup: the daemon under test reads
3224    /// its data home from the process environment while it boots, and holding
3225    /// the lock across that await would block every other test that sets one.
3226    #[cfg(unix)]
3227    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3228    async fn an_in_process_daemon_without_a_configured_directory_serves_no_local_endpoint() {
3229        let root = ScratchDir::new("in-process-no-local-socket");
3230        let data_home = root.join("data");
3231        fs::create_dir_all(&data_home).unwrap();
3232        let connection_path = root.join("c");
3233        let _data;
3234        let _config;
3235        let _port;
3236        {
3237            let _env_lock = ENV_LOCK.lock().unwrap();
3238            _data = EnvGuard::set("XDG_DATA_HOME", &data_home);
3239            _config = EnvGuard::set("XDG_CONFIG_HOME", &root.join("config"));
3240            _port = EnvGuard::unset(SUBC_PORT_ENV);
3241        }
3242
3243        let bound = expect_bound(ensure_singleton(&connection_path, 0).await.unwrap());
3244
3245        assert!(bound.local_endpoint.is_none());
3246        assert!(bound.local_endpoint_unavailable.is_none());
3247        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3248        assert!(published.local_endpoint.is_none());
3249        assert!(
3250            local_listener::socket_path_for(None, &connection_path).is_none(),
3251            "no configured directory, no socket"
3252        );
3253        let mut leftovers = Vec::new();
3254        let mut pending = vec![data_home.clone()];
3255        while let Some(dir) = pending.pop() {
3256            for entry in fs::read_dir(&dir).unwrap() {
3257                let entry = entry.unwrap();
3258                let path = entry.path();
3259                if entry.file_type().unwrap().is_dir() {
3260                    pending.push(path.clone());
3261                }
3262                if entry.file_name().to_string_lossy().ends_with(".sock") {
3263                    leftovers.push(path);
3264                }
3265            }
3266        }
3267        assert!(
3268            leftovers.is_empty(),
3269            "an in-process daemon must not create a socket in the data home: {leftovers:?}"
3270        );
3271    }
3272
3273    /// A local endpoint that cannot be bound does not stop the daemon: it keeps
3274    /// serving TCP, publishes no local endpoint, and reports the reason where an
3275    /// operator reads daemon facts. Every module stays up for a feature nothing
3276    /// has to use.
3277    #[cfg(unix)]
3278    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3279    async fn a_daemon_whose_socket_path_is_too_long_still_serves_tcp() {
3280        let dir = ScratchDir::new("tcp-only");
3281        let canonical = std::fs::canonicalize(&dir).unwrap();
3282        let connection_path = canonical.join("c");
3283        // The socket directory is deep enough that the socket's path is over the
3284        // Unix socket limit, so binding it must fail.
3285        let deep = canonical.join("x".repeat(80));
3286        fs::create_dir(&deep).unwrap();
3287        fs::set_permissions(&deep, fs::Permissions::from_mode(0o700)).unwrap();
3288        let config =
3289            BootstrapConfig::new(&connection_path, 0).with_local_listener_socket_dir(&deep);
3290        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3291        assert!(
3292            bound.local_endpoint.is_none(),
3293            "a socket that cannot be bound is not published"
3294        );
3295        assert!(
3296            bound
3297                .local_endpoint_unavailable
3298                .as_deref()
3299                .is_some_and(|reason| reason.contains("too long")),
3300            "{:?}",
3301            bound.local_endpoint_unavailable
3302        );
3303        let published = connection_file::read_with_local_endpoint(&connection_path).unwrap();
3304        assert!(published.local_endpoint.is_none());
3305        assert!(
3306            !published.info.endpoints.is_empty(),
3307            "TCP is still published"
3308        );
3309
3310        // The daemon serves TCP as usual.
3311        let server = start_server(bound);
3312        let info = connection_file::read(&connection_path).unwrap();
3313        let mut client = connect_from_info(&info).await.unwrap();
3314        authenticate_client(&mut client, &info, PROBE_AUTH_DEADLINE)
3315            .await
3316            .expect("a TCP client still completes the handshake");
3317        server.abort();
3318        let _ = server.await;
3319    }
3320
3321    /// The published file names the local endpoint the daemon actually serves,
3322    /// and a client that trusts the file completes the handshake over it.
3323    #[cfg(unix)]
3324    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3325    async fn published_connection_file_names_the_served_local_endpoint() {
3326        let (_dir, path) = temp_connection_file_path("local-endpoint");
3327        // The socket goes in the system temporary directory rather than inside
3328        // the harness's own directory, whose long lockdown-sandbox path leaves
3329        // no room inside the Unix socket path limit.
3330        let short_socket_dir = std::env::temp_dir().join(format!("subc-bt-{}", std::process::id()));
3331        let config =
3332            BootstrapConfig::new(&path, 0).with_local_listener_socket_dir(&short_socket_dir);
3333        let bound = expect_bound(ensure_singleton_with_config(config).await.unwrap());
3334        let endpoint = connection_file::read_with_local_endpoint(&path)
3335            .unwrap()
3336            .local_endpoint
3337            .expect("the daemon serves a local endpoint, so the file names it");
3338        assert_eq!(Some(&endpoint), bound.local_endpoint.as_ref());
3339        let LocalEndpoint::Unix { path: socket_path } = &endpoint else {
3340            panic!("a Unix daemon publishes a socket path");
3341        };
3342        assert_eq!(socket_path.parent().unwrap(), short_socket_dir);
3343        assert_eq!(
3344            fs::metadata(socket_path).unwrap().permissions().mode() & 0o777,
3345            0o600
3346        );
3347
3348        let router = Arc::new(Router::with_default_self_handler());
3349        let auth = auth_for(&bound.connection_info);
3350        let server = tokio::spawn(serve_local_listener(
3351            bound.local_listener.expect("the local listener is bound"),
3352            router,
3353            auth,
3354        ));
3355        let mut client = tokio::net::UnixStream::connect(socket_path).await.unwrap();
3356        authenticate_client(&mut client, &bound.connection_info, Duration::from_secs(5))
3357            .await
3358            .expect("a socket client completes the same handshake as a TCP client");
3359
3360        server.abort();
3361        drop(bound.listeners);
3362    }
3363
3364    /// A configured directory that is not a directory refuses at bind rather
3365    /// than publishing an endpoint nothing serves.
3366    #[cfg(unix)]
3367    #[test]
3368    fn a_socket_directory_that_is_a_file_refuses() {
3369        let dir = ScratchDir::new("local-listener-file-as-dir");
3370        let file = dir.join("not-a-directory");
3371        fs::write(&file, b"x").unwrap();
3372        let error = local_listener::bind(
3373            &LocalListenerConfig::in_socket_dir(file),
3374            &dir.join("connection.json"),
3375        )
3376        .expect_err("a path that is not a directory cannot hold the socket");
3377        assert!(matches!(error, ServerError::LocalAccept { .. }), "{error}");
3378    }
3379
3380    /// The socket sits directly in the configured directory under one fixed
3381    /// name, and a directory that would put the path over the platform limit is
3382    /// refused rather than shortened around.
3383    #[cfg(unix)]
3384    #[test]
3385    fn the_socket_path_stays_within_the_unix_limit() {
3386        let run_dir = PathBuf::from("/tmp/short-run");
3387        assert!(
3388            local_listener::socket_path_for(None, &run_dir.join("one.json")).is_none(),
3389            "no configured directory means no socket"
3390        );
3391        let socket = local_listener::socket_path_for(Some(&run_dir), &run_dir.join("one.json"))
3392            .expect("a configured directory gets a socket");
3393        assert_eq!(socket, run_dir.join("subc.sock"));
3394        assert_eq!(
3395            socket,
3396            local_listener::socket_path_for(Some(&run_dir), &run_dir.join("two.json")).unwrap(),
3397            "the name does not come from the connection file"
3398        );
3399
3400        let deep = PathBuf::from(format!("/tmp/{}", "x".repeat(60)));
3401        let long = local_listener::socket_path_for(Some(&deep), &deep.join("conn.json"))
3402            .expect("a configured directory gets a socket");
3403        assert!(
3404            long.as_os_str().len() < 104,
3405            "{} is {} bytes",
3406            long.display(),
3407            long.as_os_str().len()
3408        );
3409    }
3410}