Skip to main content

khive_runtime/events_split/
daemon.rs

1use super::*;
2
3/// Advisory lock guaranteeing at most one events daemon per socket path.
4/// Held for the daemon's lifetime; a second daemon exits instead of stealing
5/// the socket path from the live one.
6#[cfg(unix)]
7pub struct EventsDaemonGuard {
8    pub(super) _file: std::fs::File,
9}
10
11/// Try to become the events daemon for `socket_path`, returning `None` for
12/// contention or any refusal. Callers must validate the parent with
13/// `ensure_socket_dir_is_trusted` before lock-path operations in that directory.
14#[cfg(unix)]
15pub fn try_acquire_events_daemon_guard(socket_path: &Path) -> Option<EventsDaemonGuard> {
16    match acquire_events_daemon_guard_outcome(socket_path) {
17        EventsDaemonGuardAcquisition::Held(guard) => Some(guard),
18        _ => None,
19    }
20}
21
22/// Why a non-blocking daemon-lock acquisition did or did not succeed.
23/// Refusals retain the I/O error instead of treating every failure as contention.
24#[cfg(unix)]
25pub(super) enum EventsDaemonGuardAcquisition {
26    Held(EventsDaemonGuard),
27    Contended,
28    OpenFailed(std::io::Error),
29    HardeningRefused(std::io::Error),
30}
31
32#[cfg(unix)]
33impl std::fmt::Debug for EventsDaemonGuardAcquisition {
34    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
35        match self {
36            Self::Held(_) => f.write_str("Held"),
37            Self::Contended => f.write_str("Contended"),
38            Self::OpenFailed(error) => f.debug_tuple("OpenFailed").field(error).finish(),
39            Self::HardeningRefused(error) => {
40                f.debug_tuple("HardeningRefused").field(error).finish()
41            }
42        }
43    }
44}
45
46/// Try to become the events daemon for `socket_path`. Every refusal means
47/// the caller must not serve; only `Contended` identifies a held lock.
48/// Callers that need the socket directory validated must run
49/// `ensure_socket_dir_is_trusted` on the parent BEFORE calling this, so no
50/// lock-path operation happens in an untrusted directory.
51#[cfg(unix)]
52pub(super) fn acquire_events_daemon_guard_outcome(
53    socket_path: &Path,
54) -> EventsDaemonGuardAcquisition {
55    use EventsDaemonGuardAcquisition::{Contended, HardeningRefused, Held, OpenFailed};
56
57    let lock_path = socket_path.with_extension("lock");
58    if let Some(parent) = lock_path.parent() {
59        if let Err(error) = std::fs::create_dir_all(parent) {
60            return OpenFailed(error);
61        }
62    }
63    // `O_NOFOLLOW` pins the open to the final component: a symlink planted
64    // at the lock name is refused instead of redirecting the open (and the
65    // chmod below) to an attacker-selected target.
66    let opened = khive_fs::opened_file::open_writable_file_nofollow(
67        &lock_path,
68        khive_fs::fd_relative::OpenFileOptions {
69            read_write: false,
70            create: khive_fs::fd_relative::Create::IfMissing,
71            nonblock: false,
72            mode: 0o600,
73        },
74    );
75    let file = match opened {
76        Ok(file) => file,
77        Err(error) if error.raw_os_error() == Some(libc::ELOOP) => {
78            return HardeningRefused(error);
79        }
80        Err(error) => return OpenFailed(error),
81    };
82    // `mode` applies only at creation; tighten a pre-existing lock file too.
83    // Descriptor-based (`fchmod` on the handle just opened), never a second
84    // path lookup — and fail closed: with the inode pinned, a failed chmod
85    // is abnormal, and serving behind a lock file another user can open is
86    // exactly what the hardening exists to refuse.
87    if let Err(error) = file.set_permissions(
88        <std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
89    ) {
90        return HardeningRefused(error);
91    }
92    use std::os::fd::AsRawFd;
93    // SAFETY: `fd` is a live descriptor owned by `file` for the duration of
94    // the call; `flock` reads nothing else.
95    let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
96    if rc == 0 {
97        Held(EventsDaemonGuard { _file: file })
98    } else {
99        let error = std::io::Error::last_os_error();
100        if error.raw_os_error() == Some(libc::EWOULDBLOCK)
101            || error.raw_os_error() == Some(libc::EAGAIN)
102        {
103            Contended
104        } else {
105            HardeningRefused(error)
106        }
107    }
108}
109
110pub(super) fn events_db_targets(db_path: &Path) -> [PathBuf; 3] {
111    ["", "-wal", "-shm"].map(|suffix| {
112        let mut name = db_path.as_os_str().to_os_string();
113        name.push(suffix);
114        PathBuf::from(name)
115    })
116}
117
118/// Refuse to serve an events database whose path — or whose `-wal`/`-shm`
119/// sidecar path — is a pre-existing symlink. These paths are derived, never
120/// user-chosen (`events_db_path_beside` canonicalizes the main database
121/// spelling first), so a link here is a planted redirect, not an alias:
122/// permission hardening and SQLite would otherwise follow it and tighten or
123/// write event rows through to whatever file the link's author chose
124/// (CWE-59). Runs before any open on both the embedded and daemon arms; a
125/// link planted after admission is bounded by the daemon's trusted-directory
126/// contract on the socket parent.
127pub(super) fn refuse_events_db_symlinks(db_path: &Path) -> anyhow::Result<()> {
128    for path in events_db_targets(db_path) {
129        match std::fs::symlink_metadata(&path) {
130            Ok(meta) if meta.file_type().is_symlink() => anyhow::bail!(
131                "refusing to serve events: {} is a symlink; the events database and its \
132                 sidecars must be regular files",
133                path.display()
134            ),
135            Ok(_) => {}
136            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
137            Err(e) => anyhow::bail!(
138                "refusing to serve events: cannot inspect {}: {e}",
139                path.display()
140            ),
141        }
142    }
143    Ok(())
144}
145
146#[cfg(unix)]
147#[derive(Clone, Copy, Debug, PartialEq, Eq)]
148pub(super) struct EventsFileIdentity {
149    pub(super) device: u64,
150    pub(super) inode: u64,
151}
152
153#[cfg(unix)]
154impl EventsFileIdentity {
155    pub(super) fn from_metadata(metadata: &std::fs::Metadata) -> Self {
156        use std::os::unix::fs::MetadataExt;
157        Self {
158            device: metadata.dev(),
159            inode: metadata.ino(),
160        }
161    }
162}
163
164/// Main/WAL/SHM identities, in `events_db_targets` order. None means the
165/// target was absent when pre-open hardening tried to open it.
166#[cfg(unix)]
167pub(super) type EventsDbIdentities = [Option<EventsFileIdentity>; 3];
168
169/// Require the directory holding the events database to be trusted before
170/// anything opens a path inside it. The symlink pre-checks and the fd-pinned
171/// chmod close the races they can see, but SQLite's own open is path-based
172/// and cannot be inode-pinned from here — so the remaining defense is the
173/// same one the daemon socket uses: no untrusted local user may be able to
174/// write (or swap components of) the directory the path traverses. Delegates
175/// to the shared trust walk with store-specific diagnostics, validating every
176/// ancestor the kernel will visit with the same ownership and sticky-bit rules.
177#[cfg(unix)]
178pub(super) fn ensure_events_db_parent_trusted(db_path: &Path) -> anyhow::Result<()> {
179    let db_path = absolutize(db_path);
180    let parent = db_path.parent().filter(|p| !p.as_os_str().is_empty());
181    match parent {
182        Some(dir) => crate::daemon::ensure_store_dir_is_trusted(dir, &db_path).map_err(|e| {
183            anyhow::anyhow!("refusing to serve events from an untrusted directory: {e}")
184        }),
185        None => Ok(()),
186    }
187}
188
189/// Create the events database file owner-only if it does not exist yet.
190/// The 0600 socket and peer-uid admission bound who can *talk to* the
191/// daemon; they bound nothing if the database file itself is readable by
192/// other local users, so the daemon creates it 0600 before SQLite ever
193/// opens it (SQLite would otherwise create it at the process umask).
194#[cfg(unix)]
195pub(super) fn ensure_events_db_owner_only(db_path: &Path) -> anyhow::Result<()> {
196    use std::os::unix::fs::OpenOptionsExt;
197    refuse_events_db_symlinks(db_path)?;
198    if let Some(parent) = db_path.parent() {
199        std::fs::create_dir_all(parent)?;
200    }
201    // After creation so a fresh parent can be validated; a directory this
202    // process just created in a trusted ancestor passes by construction.
203    ensure_events_db_parent_trusted(db_path)?;
204    match std::fs::OpenOptions::new()
205        .write(true)
206        .create_new(true)
207        .mode(0o600)
208        .open(db_path)
209    {
210        // A zero-byte file is a valid empty SQLite database.
211        Ok(_created) => Ok(()),
212        Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()),
213        Err(e) => Err(anyhow::anyhow!(
214            "refusing to serve events: cannot create {} owner-only: {e}",
215            db_path.display()
216        )),
217    }
218}
219
220/// Tighten the events database and its SQLite sidecars to owner-only.
221/// Fail closed, same contract as the socket chmod: a daemon that cannot
222/// keep its database owner-only must not serve it.
223///
224/// PRECONDITION: no SQLite connection to `db_path` may be open in this
225/// process. This function opens and closes a descriptor on the database and
226/// on each sidecar, and POSIX advisory locks are per process and per inode,
227/// released by the close of ANY descriptor for the inode (fcntl(2)): called
228/// on a live database it silently drops the SHARED lock SQLite keeps in WAL
229/// mode and the `-shm` DMS lock, while the connection believes it still
230/// holds them. Both callers run it before their open; keep it that way. The
231/// post-open check that opens nothing is `verify_events_db_owner_only_unopened`.
232#[cfg(unix)]
233pub(super) fn harden_events_db_sidecars(db_path: &Path) -> anyhow::Result<EventsDbIdentities> {
234    use khive_fs::opened_file::ContainedOpenError;
235    use std::os::unix::fs::PermissionsExt;
236    let mut identities = [None; 3];
237    for (index, path) in events_db_targets(db_path).into_iter().enumerate() {
238        // Pin the inode before touching it: `O_NOFOLLOW` makes the open
239        // itself refuse a symlink at the final component, and the chmod is
240        // then issued on the returned handle (fchmod), so no path re-lookup
241        // exists between validation and the permission change for a swapped
242        // entry to exploit. A path-based lstat-then-chmod pair here would
243        // re-open the exact race it is defending against.
244        let file = match khive_fs::opened_file::open_regular_file_nofollow(&path) {
245            Ok(file) => file,
246            Err(ContainedOpenError::Open(e)) if e.kind() == std::io::ErrorKind::NotFound => {
247                continue;
248            }
249            Err(ContainedOpenError::Open(e)) => anyhow::bail!(
250                "refusing to serve events: cannot open {} without following symlinks: \
251                 {e}. The events database and its sidecars must be regular files.",
252                path.display()
253            ),
254            Err(ContainedOpenError::Metadata(e)) => return Err(e.into()),
255            Err(_) => anyhow::bail!(
256                "refusing to serve events: {} is not a regular file. The events \
257                 database and its sidecars must be regular files.",
258                path.display()
259            ),
260        };
261        // The shared helper has checked the opened file's regular type.
262        // Read this same descriptor's metadata for the pre-open identity
263        // snapshot; no pathname is reopened before the chmod.
264        let metadata = file.metadata()?;
265        if !metadata.file_type().is_file() {
266            anyhow::bail!(
267                "refusing to serve events: {} is not a regular file. The events \
268                 database and its sidecars must be regular files.",
269                path.display()
270            );
271        }
272        file.set_permissions(std::fs::Permissions::from_mode(0o600))
273            .map_err(|e| {
274                anyhow::anyhow!(
275                    "refusing to serve events: cannot chmod 0600 {}: {e}. The events \
276                     database and its sidecars must be owner-only.",
277                    path.display()
278                )
279            })?;
280        // The same descriptor supplied validation and fchmod; record its
281        // identity before closing it, never by re-looking up the path.
282        identities[index] = Some(EventsFileIdentity::from_metadata(&metadata));
283    }
284    Ok(identities)
285}
286
287/// Check, after SQLite has opened the database, that it and its sidecars are
288/// owner-only regular files WITHOUT opening any of them: `lstat` takes no
289/// descriptor, so it cannot release the advisory locks the open connections
290/// hold (see the precondition on `harden_events_db_sidecars`). Sidecars
291/// SQLite creates inherit the database file's mode, so a failure here means
292/// something else changed the file set; refuse to serve rather than tighten
293/// through a path-based chmod and its lookup race. Present paths must retain
294/// their pre-open device/inode when one was observed. A missing main database
295/// is refused: serving an unlinked database would detach writes from its path.
296/// Sidecars may legitimately be absent or newly created by SQLite, so an absent
297/// pre- or post-observation does not assert identity continuity for that sidecar.
298///
299/// This is a two-observation check, not a pin of SQLite's own descriptors. It
300/// cannot detect swaps restored between observations, inode reuse, or changes
301/// after the final lstat. Even a legitimate sidecar delete/recreate is refused
302/// if both observations see different identities; a later startup can retry
303/// with a fresh snapshot. Trusted parent directories remain required. Unix only.
304#[cfg(unix)]
305pub(super) fn verify_events_db_owner_only_unopened(
306    db_path: &Path,
307    before: &EventsDbIdentities,
308) -> anyhow::Result<()> {
309    use std::os::unix::fs::PermissionsExt;
310    for (index, path) in events_db_targets(db_path).into_iter().enumerate() {
311        let metadata = match std::fs::symlink_metadata(&path) {
312            Ok(metadata) => metadata,
313            Err(e) if e.kind() == std::io::ErrorKind::NotFound && index != 0 => continue,
314            Err(e) if e.kind() == std::io::ErrorKind::NotFound => anyhow::bail!(
315                "refusing to serve events: main database {} disappeared after SQLite open; \
316                 refusing to serve an unlinked database",
317                path.display()
318            ),
319            Err(e) => {
320                anyhow::bail!(
321                    "refusing to serve events: cannot stat {}: {e}",
322                    path.display()
323                )
324            }
325        };
326        if !metadata.file_type().is_file() {
327            anyhow::bail!(
328                "refusing to serve events: {} is not a regular file. The events database \
329                 and its sidecars must be regular files.",
330                path.display()
331            );
332        }
333        let mode = metadata.permissions().mode() & 0o777;
334        if mode & 0o077 != 0 {
335            anyhow::bail!(
336                "refusing to serve events: {} is mode {mode:03o}, not owner-only. The events \
337                 database and its sidecars must be owner-only.",
338                path.display()
339            );
340        }
341        if before[index]
342            .is_some_and(|identity| identity != EventsFileIdentity::from_metadata(&metadata))
343        {
344            anyhow::bail!(
345                "refusing to serve events: {} changed identity between pre-open hardening \
346                 and post-open verification",
347                path.display()
348            );
349        }
350    }
351    Ok(())
352}
353
354/// How long a daemon holds requests at startup for the events daemon it
355/// supervises to answer before it refuses them ([`EventsReadiness`]).
356#[cfg(unix)]
357pub const EVENTS_READINESS_WINDOW: Duration = Duration::from_secs(10);
358
359/// Startup readiness of the events daemon this process supervises (ADR-170),
360/// as seen by the request path.
361///
362/// A daemon with events forwarding enabled commits every write's audit batch
363/// through the supervised events daemon. That child is spawned in the
364/// background after the main socket is bound, and until it binds its own
365/// socket every audit commit fails: a write verb then reports a refusal
366/// while its domain write is already committed. The daemon therefore holds
367/// each request at this gate until the events socket has answered once.
368///
369/// The window is measured from the moment supervision starts, not per
370/// request: once it has elapsed without an answer, requests are refused at
371/// once instead of each waiting again, and a later answer (a respawned child
372/// binding) admits them again. The gate is a startup barrier only; it stays
373/// open after the first answer.
374#[cfg(unix)]
375#[derive(Clone, Debug)]
376pub struct EventsReadiness {
377    pub(super) socket: PathBuf,
378    pub(super) deadline: tokio::time::Instant,
379    pub(super) ready: tokio::sync::watch::Receiver<bool>,
380}
381
382/// The supervisor's side of an [`EventsReadiness`] gate.
383#[cfg(unix)]
384#[derive(Debug)]
385pub struct EventsReadinessSignal {
386    pub(super) deadline: tokio::time::Instant,
387    pub(super) ready: tokio::sync::watch::Sender<bool>,
388}
389
390/// Why an [`EventsReadiness`] gate refused a request.
391#[cfg(unix)]
392#[derive(Clone, Debug, PartialEq, Eq)]
393pub enum EventsNotReadyReason {
394    /// The startup window elapsed without the events socket answering.
395    WindowElapsed,
396    /// The supervisor stopped before the events socket answered.
397    SupervisorStopped,
398}
399
400/// A request refused by an [`EventsReadiness`] gate; names the socket that
401/// did not answer.
402#[cfg(unix)]
403#[derive(Clone, Debug, PartialEq, Eq)]
404pub struct EventsNotReady {
405    pub socket: PathBuf,
406    pub reason: EventsNotReadyReason,
407}
408
409#[cfg(unix)]
410impl std::fmt::Display for EventsNotReady {
411    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
412        match self.reason {
413            EventsNotReadyReason::WindowElapsed => write!(
414                f,
415                "events daemon at {} has not answered since this daemon started; \
416                 the request was refused before dispatch",
417                self.socket.display()
418            ),
419            EventsNotReadyReason::SupervisorStopped => write!(
420                f,
421                "events daemon supervision for {} stopped before the socket answered; \
422                 the request was refused before dispatch",
423                self.socket.display()
424            ),
425        }
426    }
427}
428
429/// Create a readiness gate for the events daemon at `socket`, whose startup
430/// window of `window` starts now.
431#[cfg(unix)]
432pub fn events_readiness(
433    socket: PathBuf,
434    window: Duration,
435) -> (EventsReadinessSignal, EventsReadiness) {
436    let deadline = tokio::time::Instant::now() + window;
437    let (sender, receiver) = tokio::sync::watch::channel(false);
438    (
439        EventsReadinessSignal {
440            deadline,
441            ready: sender,
442        },
443        EventsReadiness {
444            socket,
445            deadline,
446            ready: receiver,
447        },
448    )
449}
450
451#[cfg(unix)]
452impl EventsReadinessSignal {
453    /// Record that the events socket answered; opens the gate for good.
454    pub fn mark_ready(&self) {
455        self.ready
456            .send_if_modified(|ready| !std::mem::replace(ready, true));
457    }
458
459    /// Whether requests may still be waiting on this gate: not yet ready and
460    /// the startup window still open.
461    pub fn awaiting(&self) -> bool {
462        !*self.ready.borrow() && tokio::time::Instant::now() < self.deadline
463    }
464
465    /// Whether the events socket has answered.
466    pub fn is_ready(&self) -> bool {
467        *self.ready.borrow()
468    }
469}
470
471#[cfg(unix)]
472impl EventsReadiness {
473    /// The events socket this gate waits on.
474    pub fn socket(&self) -> &Path {
475        &self.socket
476    }
477
478    /// Whether the events socket has answered.
479    pub fn is_ready(&self) -> bool {
480        *self.ready.borrow()
481    }
482
483    /// Wait until the events socket has answered, or refuse once the startup
484    /// window has elapsed or the supervisor has stopped.
485    pub async fn wait(&self) -> Result<(), EventsNotReady> {
486        let mut ready = self.ready.clone();
487        let outcome = tokio::time::timeout_at(self.deadline, ready.wait_for(|ready| *ready))
488            .await
489            .map(|answered| answered.map(|_| ()));
490        let refused = |reason| EventsNotReady {
491            socket: self.socket.clone(),
492            reason,
493        };
494        match outcome {
495            Ok(Ok(())) => Ok(()),
496            Ok(Err(_)) => Err(refused(EventsNotReadyReason::SupervisorStopped)),
497            Err(_) if self.is_ready() => Ok(()),
498            Err(_) => Err(refused(EventsNotReadyReason::WindowElapsed)),
499        }
500    }
501}
502
503/// The startup window for a new [`EventsReadiness`] gate:
504/// [`EVENTS_READINESS_WINDOW`], or `KHIVE_TEST_EVENTS_READY_WINDOW_MS` under
505/// `KHIVE_TEST_HARNESS=1` so integration tests can reach the refusal quickly.
506#[cfg(unix)]
507pub fn events_readiness_window() -> Duration {
508    harness_duration_ms("KHIVE_TEST_EVENTS_READY_WINDOW_MS").unwrap_or(EVENTS_READINESS_WINDOW)
509}
510
511/// A test-only duration in milliseconds read from the variable `name`,
512/// honoured only under `KHIVE_TEST_HARNESS=1`. Unset, malformed or outside
513/// the harness it is `None`.
514#[cfg(unix)]
515pub(crate) fn harness_duration_ms(name: &str) -> Option<Duration> {
516    if std::env::var("KHIVE_TEST_HARNESS").as_deref() != Ok("1") {
517        return None;
518    }
519    std::env::var(name)
520        .ok()?
521        .trim()
522        .parse::<u64>()
523        .ok()
524        .map(Duration::from_millis)
525}
526
527/// Supervise the events daemon from the main daemon process: probe the
528/// socket periodically and (re)spawn the daemon subcommand when unreachable.
529///
530/// The spawned command contract is fixed here once: the current executable
531/// re-invoked as `events-daemon --db <db> --socket <socket>` with the resolved
532/// WAL ceiling bytes and source — the subcommand the kernel binary registers
533/// for [`run_events_daemon`]. The child holds the per-socket advisory lock,
534/// so a probe/spawn race resolves to one survivor.
535///
536/// Lifecycle: the loop observes the process-wide daemon shutdown token, so
537/// `drain()` never waits on it forever, and it retains the handle of the
538/// child it spawned — reaping it with `try_wait` on every probe (no zombie
539/// accumulation during a persistent startup failure) and never stacking a
540/// second spawn on a still-live child. On shutdown, a child this supervisor
541/// spawned is killed and reaped; a pre-existing events daemon it never
542/// spawned is left alone.
543///
544/// The reachability probe uses the peer-verified connect: a socket answered
545/// by a foreign-uid process is treated as UNREACHABLE (and logged loudly),
546/// so a pre-bound spoof socket triggers a real-daemon spawn instead of being
547/// reported healthy.
548///
549/// This standalone entry resolves its environment once before supervision.
550/// Hosts with an opened main backend pass its resolved policies through
551/// [`supervise_events_daemon_with_policies`].
552#[cfg(unix)]
553pub async fn supervise_events_daemon(db_path: PathBuf, socket_path: PathBuf) {
554    match standalone_daemon_policies(&db_path) {
555        Ok((wal_ceiling, disk_guard, volume_lock_dir)) => {
556            supervise_events_daemon_with_policies(
557                db_path,
558                socket_path,
559                wal_ceiling,
560                disk_guard,
561                volume_lock_dir,
562            )
563            .await;
564        }
565        Err(error) => {
566            tracing::warn!(%error, "invalid events daemon policy; events supervisor not started");
567        }
568    }
569}
570
571/// Supervise using the main pool's captured writer policies and lock directory.
572#[cfg(unix)]
573pub async fn supervise_events_daemon_with_policies(
574    db_path: PathBuf,
575    socket_path: PathBuf,
576    wal_ceiling: WalCeilingPolicy,
577    disk_guard: khive_db::EffectiveDiskGuardConfig,
578    volume_lock_dir: PathBuf,
579) {
580    let (readiness, _unobserved) = events_readiness(socket_path.clone(), EVENTS_READINESS_WINDOW);
581    supervise_events_daemon_with_readiness(
582        db_path,
583        socket_path,
584        wal_ceiling,
585        disk_guard,
586        volume_lock_dir,
587        readiness,
588    )
589    .await;
590}
591
592/// [`supervise_events_daemon_with_policies`], reporting the first answer of
593/// the events socket through `readiness` so the host can hold requests until
594/// then ([`EventsReadiness`]).
595///
596/// While the gate is waiting, the socket is probed every
597/// `FAST_PROBE_INTERVAL` instead of every `PROBE_INTERVAL`, so a request is
598/// admitted within milliseconds of the child binding. Once the startup window
599/// has elapsed without an answer, it is probed every
600/// `UNANSWERED_PROBE_INTERVAL`, so a late bind reopens the gate within a
601/// second instead of up to `PROBE_INTERVAL` later. A spawn still happens at
602/// most once per `PROBE_INTERVAL`, so a child that keeps failing at startup
603/// is respawned no faster than before.
604#[cfg(unix)]
605pub async fn supervise_events_daemon_with_readiness(
606    db_path: PathBuf,
607    socket_path: PathBuf,
608    wal_ceiling: WalCeilingPolicy,
609    disk_guard: khive_db::EffectiveDiskGuardConfig,
610    volume_lock_dir: PathBuf,
611    readiness: EventsReadinessSignal,
612) {
613    if let Err(error) = disk_guard.validate() {
614        tracing::warn!(%error, "invalid disk policy; events supervisor not started");
615        return;
616    }
617    if let Err(error) = wal_ceiling.validate_static(true, true, false) {
618        tracing::warn!(error = %error, "invalid WAL ceiling; events supervisor not started");
619        return;
620    }
621    const PROBE_INTERVAL: Duration = Duration::from_secs(15);
622    const FAST_PROBE_INTERVAL: Duration = Duration::from_millis(20);
623    const UNANSWERED_PROBE_INTERVAL: Duration = Duration::from_secs(1);
624    let shutdown = crate::daemon::daemon_shutdown_token();
625    let mut child: Option<std::process::Child> = None;
626    let mut respawns: u64 = 0;
627    let mut last_spawn: Option<std::time::Instant> = None;
628    loop {
629        // Reap first: a child that exited (crashed, lost the advisory lock
630        // race, refused an untrusted directory) must not linger as a zombie,
631        // and clearing the slot is what re-arms the spawn below.
632        if let Some(c) = child.as_mut() {
633            match c.try_wait() {
634                Ok(Some(status)) => {
635                    tracing::info!(%status, "events daemon child exited");
636                    child = None;
637                }
638                Ok(None) => {}
639                Err(error) => {
640                    tracing::warn!(error = %error, "cannot poll events daemon child; dropping handle");
641                    child = None;
642                }
643            }
644        }
645
646        let reachable = match connect_verified(&socket_path).await {
647            Ok(_stream) => true,
648            Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
649                tracing::warn!(
650                    socket = %socket_path.display(),
651                    error = %error,
652                    "events socket answered by a foreign uid; treating as unreachable"
653                );
654                false
655            }
656            Err(_) => false,
657        };
658        if reachable {
659            readiness.mark_ready();
660        }
661        let spawn_due = last_spawn.is_none_or(|at| at.elapsed() >= PROBE_INTERVAL);
662        if !reachable && child.is_none() && spawn_due {
663            last_spawn = Some(std::time::Instant::now());
664            match std::env::current_exe() {
665                Ok(exe) => {
666                    let spawned = events_daemon_command(
667                        &exe,
668                        &db_path,
669                        &socket_path,
670                        wal_ceiling,
671                        disk_guard,
672                        &volume_lock_dir,
673                    )
674                    .spawn();
675                    match spawned {
676                        Ok(spawned_child) => {
677                            respawns += 1;
678                            tracing::info!(
679                                pid = spawned_child.id(),
680                                respawns,
681                                socket = %socket_path.display(),
682                                "spawned events daemon"
683                            );
684                            child = Some(spawned_child);
685                        }
686                        Err(error) => {
687                            tracing::warn!(error = %error, "failed to spawn events daemon");
688                        }
689                    }
690                }
691                Err(error) => {
692                    tracing::warn!(error = %error, "cannot resolve current executable for events daemon spawn");
693                }
694            }
695        }
696        let pause = if readiness.awaiting() {
697            FAST_PROBE_INTERVAL
698        } else if readiness.is_ready() {
699            PROBE_INTERVAL
700        } else {
701            UNANSWERED_PROBE_INTERVAL
702        };
703        tokio::select! {
704            _ = shutdown.cancelled() => break,
705            _ = tokio::time::sleep(pause) => {}
706        }
707    }
708    if let Some(mut c) = child.take() {
709        // Our child, our cleanup: the events daemon holds no volatile queue
710        // state (SQLite is the durability), and the next daemon host's
711        // supervisor respawns it.
712        let _ = c.kill();
713        let _ = c.wait();
714        tracing::info!("events daemon child stopped with supervisor shutdown");
715    }
716}
717
718#[cfg(unix)]
719pub(super) fn events_daemon_command(
720    executable: &Path,
721    db_path: &Path,
722    socket_path: &Path,
723    wal_ceiling: WalCeilingPolicy,
724    disk_guard: khive_db::EffectiveDiskGuardConfig,
725    volume_lock_dir: &Path,
726) -> std::process::Command {
727    let source = match wal_ceiling.source {
728        khive_db::WalCeilingSource::BackendField => "backend_field",
729        khive_db::WalCeilingSource::Environment => "environment",
730        khive_db::WalCeilingSource::Default => "default",
731    };
732    let mut command = std::process::Command::new(executable);
733    command
734        .arg("events-daemon")
735        .arg("--db")
736        .arg(db_path)
737        .arg("--socket")
738        .arg(socket_path)
739        .arg("--wal-ceiling-bytes")
740        .arg(wal_ceiling.bytes.to_string())
741        .arg("--wal-ceiling-source")
742        .arg(source)
743        .arg("--disk-reserve-bytes")
744        .arg(disk_guard.reserve_bytes.to_string())
745        .arg("--disk-guard-deadline-ms")
746        .arg(disk_guard.guard_deadline_ms.to_string())
747        .arg("--disk-reserve-source")
748        .arg(disk_guard.reserve_source.as_str())
749        .arg("--disk-deadline-source")
750        .arg(disk_guard.deadline_source.as_str())
751        .arg("--disk-legacy-environment-present")
752        .arg(disk_guard.legacy_environment_present.to_string())
753        .arg("--volume-lock-dir")
754        .arg(volume_lock_dir)
755        .stdin(std::process::Stdio::null())
756        .stdout(std::process::Stdio::null())
757        .stderr(std::process::Stdio::null());
758    command
759}
760
761/// Serve the events daemon loop on `socket_path`, owning `db_path`.
762///
763/// Binds the socket (removing a stale path first), then accepts connections
764/// for the process lifetime. Each connection is served sequentially:
765/// same-uid admission, then a read-frame → dispatch → write-frame loop until
766/// the peer disconnects. All storage goes through `SqlEventStore` on a
767/// backend opened read-write against `db_path`; the events schema is ensured
768/// once at boot.
769///
770/// Bind-path trust mirrors the main daemon socket: the socket directory must
771/// pass the same ownership/mode/swap-resistance validation, and the bound
772/// socket entry is chmod'd 0600 fail-closed. Pathname reachability is not
773/// identity — clients additionally verify the peer uid on every connect —
774/// but a hardened bind path is what keeps the *bind* itself out of another
775/// user's hands.
776///
777/// This standalone entry resolves its environment once before opening the
778/// daemon. Hosts with resolved policies use [`run_events_daemon_with_policies`].
779#[cfg(unix)]
780pub async fn run_events_daemon(db_path: &Path, socket_path: &Path) -> anyhow::Result<()> {
781    let (wal_ceiling, disk_guard, volume_lock_dir) = standalone_daemon_policies(db_path)?;
782    run_events_daemon_with_policies(
783        db_path,
784        socket_path,
785        wal_ceiling,
786        disk_guard,
787        volume_lock_dir,
788    )
789    .await
790}
791
792/// Serve using the exact policies inherited from the opened main backend.
793#[cfg(unix)]
794pub async fn run_events_daemon_with_policies(
795    db_path: &Path,
796    socket_path: &Path,
797    wal_ceiling: WalCeilingPolicy,
798    disk_guard: khive_db::EffectiveDiskGuardConfig,
799    volume_lock_dir: PathBuf,
800) -> anyhow::Result<()> {
801    disk_guard.validate()?;
802    wal_ceiling
803        .validate_static(true, true, false)
804        .map_err(crate::error::RuntimeError::from)?;
805    // The subcommand's `--db`/`--socket` arrive from argv and may be
806    // relative; anchor them before anything derives a parent from them.
807    let db_path = &absolutize(db_path);
808    let socket_path = &absolutize(socket_path);
809    validate_events_socket_path(socket_path)?;
810    // Directory trust comes FIRST: the lock guard below opens and chmods a
811    // path in this directory, and validating only before the later bind
812    // would let those operations run in a directory another user controls.
813    if let Some(parent) = socket_path.parent() {
814        std::fs::create_dir_all(parent)?;
815        crate::daemon::ensure_socket_dir_is_trusted(parent)?;
816    }
817    let _guard = match acquire_events_daemon_guard_outcome(socket_path) {
818        EventsDaemonGuardAcquisition::Held(guard) => guard,
819        refusal => {
820            tracing::info!(
821                socket = %socket_path.display(),
822                reason = ?refusal,
823                "events daemon lock unavailable; exiting"
824            );
825            return Ok(());
826        }
827    };
828    ensure_events_db_owner_only(db_path)?;
829    // Tighten a pre-existing database and any `-wal`/`-shm` an earlier
830    // process left behind BEFORE SQLite opens the database. The order is
831    // load-bearing: hardening opens and closes its own descriptor on each of
832    // these files, and POSIX advisory locks are per process and per inode,
833    // released by the close of ANY descriptor for the inode (fcntl(2)). Run
834    // after the open, it silently dropped the SHARED lock SQLite keeps on the
835    // database in WAL mode and the shared lock on the `-shm` DMS byte, so the
836    // next external connection to close (a backup tool, an inspection shell)
837    // took itself for the last connection, checkpointed, and unlinked the
838    // sidecars underneath this daemon, which kept writing to the unlinked
839    // inodes. Sidecars SQLite creates from here on inherit the database
840    // file's mode; the check after the open below opens nothing.
841    let before_open = harden_events_db_sidecars(db_path)?;
842    let backend = Arc::new(
843        StorageBackend::sqlite_with_max_readers_and_policies(
844            db_path,
845            None,
846            wal_ceiling,
847            disk_guard,
848            volume_lock_dir,
849        )
850        .map_err(crate::error::RuntimeError::from)?,
851    );
852    // Ensure the schema once, loudly, before accepting traffic.
853    backend.events()?;
854    verify_events_db_owner_only_unopened(db_path, &before_open)?;
855
856    // Integration tests widen the window between the main daemon accepting
857    // requests and this socket answering, which a slow host produces.
858    if let Some(delay) = harness_duration_ms("KHIVE_TEST_EVENTS_DAEMON_BIND_DELAY_MS") {
859        tracing::info!(
860            delay_ms = delay.as_millis() as u64,
861            "test harness: delaying events socket bind"
862        );
863        tokio::time::sleep(delay).await;
864    }
865    if socket_path.exists() {
866        std::fs::remove_file(socket_path)?;
867    }
868    let listener = UnixListener::bind(socket_path)?;
869    {
870        use std::os::unix::fs::PermissionsExt;
871        // Fail closed, same contract as the main daemon socket: if the entry
872        // cannot be made owner-only, drop the listener and remove it rather
873        // than serve a world-reachable socket the design never covered.
874        if let Err(e) =
875            std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))
876        {
877            drop(listener);
878            let _ = std::fs::remove_file(socket_path);
879            anyhow::bail!(
880                "refusing to serve events: cannot chmod 0600 {}: {e}. The events socket must \
881                 be owner-only.",
882                socket_path.display()
883            );
884        }
885    }
886    let daemon_euid = unsafe { libc::geteuid() };
887    // Per-namespace store cache: `events_for_namespace` takes a writer-lane
888    // checkout and re-runs the schema DDL on every call, so paying it once
889    // per namespace instead of once per request keeps the writer lane for
890    // actual writes. The namespace is client-supplied, so the cache is
891    // bounded (`MAX_CACHED_NAMESPACE_STORES`) rather than trusted small.
892    let stores: NamespaceStores = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
893    let connections = Arc::new(tokio::sync::Semaphore::new(MAX_EVENTS_CONNECTIONS));
894    let frame_budget = Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_REQUEST_BYTES));
895    tracing::info!(
896        socket = %socket_path.display(),
897        db = %db_path.display(),
898        wal_ceiling_configured_bytes = wal_ceiling.bytes,
899        wal_ceiling_effective_bytes = wal_ceiling.effective_bytes(backend.is_read_only()),
900        wal_ceiling_source = ?wal_ceiling.source,
901        "events daemon listening"
902    );
903
904    loop {
905        let (stream, _addr) = match listener.accept().await {
906            Ok(pair) => pair,
907            Err(error) => {
908                tracing::warn!(error = %error, "events daemon accept failed");
909                continue;
910            }
911        };
912        match crate::daemon::peer_uid(&stream) {
913            Ok(uid) if crate::daemon::uid_is_permitted(uid, daemon_euid) => {}
914            Ok(uid) => {
915                tracing::warn!(peer_uid = uid, "events daemon rejected foreign-uid peer");
916                continue;
917            }
918            Err(error) => {
919                tracing::warn!(error = %error, "events daemon could not read peer credentials");
920                continue;
921            }
922        }
923        let permit = match Arc::clone(&connections).try_acquire_owned() {
924            Ok(permit) => permit,
925            Err(_) => {
926                // At the cap: drop the stream (peer sees EOF and retries via
927                // its own reconnect path) instead of queueing unbounded work.
928                tracing::warn!(
929                    cap = MAX_EVENTS_CONNECTIONS,
930                    "events daemon at connection cap; dropping new connection"
931                );
932                continue;
933            }
934        };
935        let backend = Arc::clone(&backend);
936        let stores = Arc::clone(&stores);
937        let frame_budget = Arc::clone(&frame_budget);
938        crate::daemon::spawn_named_tracked_task("events_connection", async move {
939            serve_events_conn(stream, backend, stores, frame_budget).await;
940            drop(permit);
941        });
942    }
943}