1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3use std::time::Duration;
4
5use khive_db::{StorageBackend, WalCeilingPolicy};
6use tokio::net::UnixListener;
7
8use super::file_hardening::{
9 ensure_events_db_owner_only, harden_events_db_sidecars, verify_events_db_owner_only_unopened,
10};
11use super::forwarder::connect_verified;
12use super::paths::{absolutize, MAX_EVENTS_CONNECTIONS, MAX_INFLIGHT_REQUEST_BYTES};
13use super::policy_entries::standalone_daemon_policies;
14use super::server::{serve_events_conn, NamespaceStores};
15use super::socket_path::validate_events_socket_path;
16
17#[cfg(unix)]
21pub struct EventsDaemonGuard {
22 pub(super) _file: std::fs::File,
23}
24
25#[cfg(unix)]
29pub fn try_acquire_events_daemon_guard(socket_path: &Path) -> Option<EventsDaemonGuard> {
30 match acquire_events_daemon_guard_outcome(socket_path) {
31 EventsDaemonGuardAcquisition::Held(guard) => Some(guard),
32 _ => None,
33 }
34}
35
36#[cfg(unix)]
39pub(super) enum EventsDaemonGuardAcquisition {
40 Held(EventsDaemonGuard),
41 Contended,
42 OpenFailed(std::io::Error),
43 HardeningRefused(std::io::Error),
44}
45
46#[cfg(unix)]
47impl std::fmt::Debug for EventsDaemonGuardAcquisition {
48 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
49 match self {
50 Self::Held(_) => f.write_str("Held"),
51 Self::Contended => f.write_str("Contended"),
52 Self::OpenFailed(error) => f.debug_tuple("OpenFailed").field(error).finish(),
53 Self::HardeningRefused(error) => {
54 f.debug_tuple("HardeningRefused").field(error).finish()
55 }
56 }
57 }
58}
59
60#[cfg(unix)]
66pub(super) fn acquire_events_daemon_guard_outcome(
67 socket_path: &Path,
68) -> EventsDaemonGuardAcquisition {
69 use EventsDaemonGuardAcquisition::{Contended, HardeningRefused, Held, OpenFailed};
70
71 let lock_path = socket_path.with_extension("lock");
72 if let Some(parent) = lock_path.parent() {
73 if let Err(error) = std::fs::create_dir_all(parent) {
74 return OpenFailed(error);
75 }
76 }
77 let opened = khive_fs::opened_file::open_writable_file_nofollow(
81 &lock_path,
82 khive_fs::fd_relative::OpenFileOptions {
83 read_write: false,
84 create: khive_fs::fd_relative::Create::IfMissing,
85 nonblock: false,
86 mode: 0o600,
87 },
88 );
89 let file = match opened {
90 Ok(file) => file,
91 Err(error) if error.raw_os_error() == Some(libc::ELOOP) => {
92 return HardeningRefused(error);
93 }
94 Err(error) => return OpenFailed(error),
95 };
96 if let Err(error) = file.set_permissions(
102 <std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
103 ) {
104 return HardeningRefused(error);
105 }
106 use std::os::fd::AsRawFd;
107 let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
110 if rc == 0 {
111 Held(EventsDaemonGuard { _file: file })
112 } else {
113 let error = std::io::Error::last_os_error();
114 if error.raw_os_error() == Some(libc::EWOULDBLOCK)
115 || error.raw_os_error() == Some(libc::EAGAIN)
116 {
117 Contended
118 } else {
119 HardeningRefused(error)
120 }
121 }
122}
123
124#[cfg(unix)]
127pub const EVENTS_READINESS_WINDOW: Duration = Duration::from_secs(10);
128
129#[cfg(unix)]
145#[derive(Clone, Debug)]
146pub struct EventsReadiness {
147 pub(super) socket: PathBuf,
148 pub(super) deadline: tokio::time::Instant,
149 pub(super) ready: tokio::sync::watch::Receiver<bool>,
150}
151
152#[cfg(unix)]
154#[derive(Debug)]
155pub struct EventsReadinessSignal {
156 pub(super) deadline: tokio::time::Instant,
157 pub(super) ready: tokio::sync::watch::Sender<bool>,
158}
159
160#[cfg(unix)]
162#[derive(Clone, Debug, PartialEq, Eq)]
163pub enum EventsNotReadyReason {
164 WindowElapsed,
166 SupervisorStopped,
168}
169
170#[cfg(unix)]
173#[derive(Clone, Debug, PartialEq, Eq)]
174pub struct EventsNotReady {
175 pub socket: PathBuf,
176 pub reason: EventsNotReadyReason,
177}
178
179#[cfg(unix)]
180impl std::fmt::Display for EventsNotReady {
181 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
182 match self.reason {
183 EventsNotReadyReason::WindowElapsed => write!(
184 f,
185 "events daemon at {} has not answered since this daemon started; \
186 the request was refused before dispatch",
187 self.socket.display()
188 ),
189 EventsNotReadyReason::SupervisorStopped => write!(
190 f,
191 "events daemon supervision for {} stopped before the socket answered; \
192 the request was refused before dispatch",
193 self.socket.display()
194 ),
195 }
196 }
197}
198
199#[cfg(unix)]
202pub fn events_readiness(
203 socket: PathBuf,
204 window: Duration,
205) -> (EventsReadinessSignal, EventsReadiness) {
206 let deadline = tokio::time::Instant::now() + window;
207 let (sender, receiver) = tokio::sync::watch::channel(false);
208 (
209 EventsReadinessSignal {
210 deadline,
211 ready: sender,
212 },
213 EventsReadiness {
214 socket,
215 deadline,
216 ready: receiver,
217 },
218 )
219}
220
221#[cfg(unix)]
222impl EventsReadinessSignal {
223 pub fn mark_ready(&self) {
225 self.ready
226 .send_if_modified(|ready| !std::mem::replace(ready, true));
227 }
228
229 pub fn awaiting(&self) -> bool {
232 !*self.ready.borrow() && tokio::time::Instant::now() < self.deadline
233 }
234
235 pub fn is_ready(&self) -> bool {
237 *self.ready.borrow()
238 }
239}
240
241#[cfg(unix)]
242impl EventsReadiness {
243 pub fn socket(&self) -> &Path {
245 &self.socket
246 }
247
248 pub fn is_ready(&self) -> bool {
250 *self.ready.borrow()
251 }
252
253 pub async fn wait(&self) -> Result<(), EventsNotReady> {
256 let mut ready = self.ready.clone();
257 let outcome = tokio::time::timeout_at(self.deadline, ready.wait_for(|ready| *ready))
258 .await
259 .map(|answered| answered.map(|_| ()));
260 let refused = |reason| EventsNotReady {
261 socket: self.socket.clone(),
262 reason,
263 };
264 match outcome {
265 Ok(Ok(())) => Ok(()),
266 Ok(Err(_)) => Err(refused(EventsNotReadyReason::SupervisorStopped)),
267 Err(_) if self.is_ready() => Ok(()),
268 Err(_) => Err(refused(EventsNotReadyReason::WindowElapsed)),
269 }
270 }
271}
272
273#[cfg(unix)]
277pub fn events_readiness_window() -> Duration {
278 harness_duration_ms("KHIVE_TEST_EVENTS_READY_WINDOW_MS").unwrap_or(EVENTS_READINESS_WINDOW)
279}
280
281#[cfg(unix)]
285pub(crate) fn harness_duration_ms(name: &str) -> Option<Duration> {
286 if std::env::var("KHIVE_TEST_HARNESS").as_deref() != Ok("1") {
287 return None;
288 }
289 std::env::var(name)
290 .ok()?
291 .trim()
292 .parse::<u64>()
293 .ok()
294 .map(Duration::from_millis)
295}
296
297#[cfg(unix)]
324pub async fn supervise_events_daemon(db_path: PathBuf, socket_path: PathBuf) {
325 match standalone_daemon_policies(&db_path) {
326 Ok((wal_ceiling, disk_guard, volume_lock_dir)) => {
327 supervise_events_daemon_with_policies(
328 db_path,
329 socket_path,
330 wal_ceiling,
331 disk_guard,
332 volume_lock_dir,
333 )
334 .await;
335 }
336 Err(error) => {
337 tracing::warn!(%error, "invalid events daemon policy; events supervisor not started");
338 }
339 }
340}
341
342#[cfg(unix)]
344pub async fn supervise_events_daemon_with_policies(
345 db_path: PathBuf,
346 socket_path: PathBuf,
347 wal_ceiling: WalCeilingPolicy,
348 disk_guard: khive_db::EffectiveDiskGuardConfig,
349 volume_lock_dir: PathBuf,
350) {
351 let (readiness, _unobserved) = events_readiness(socket_path.clone(), EVENTS_READINESS_WINDOW);
352 supervise_events_daemon_with_readiness(
353 db_path,
354 socket_path,
355 wal_ceiling,
356 disk_guard,
357 volume_lock_dir,
358 readiness,
359 )
360 .await;
361}
362
363#[cfg(unix)]
376pub async fn supervise_events_daemon_with_readiness(
377 db_path: PathBuf,
378 socket_path: PathBuf,
379 wal_ceiling: WalCeilingPolicy,
380 disk_guard: khive_db::EffectiveDiskGuardConfig,
381 volume_lock_dir: PathBuf,
382 readiness: EventsReadinessSignal,
383) {
384 if let Err(error) = disk_guard.validate() {
385 tracing::warn!(%error, "invalid disk policy; events supervisor not started");
386 return;
387 }
388 if let Err(error) = wal_ceiling.validate_static(true, true, false) {
389 tracing::warn!(error = %error, "invalid WAL ceiling; events supervisor not started");
390 return;
391 }
392 const PROBE_INTERVAL: Duration = Duration::from_secs(15);
393 const FAST_PROBE_INTERVAL: Duration = Duration::from_millis(20);
394 const UNANSWERED_PROBE_INTERVAL: Duration = Duration::from_secs(1);
395 let shutdown = crate::daemon::daemon_shutdown_token();
396 let mut child: Option<std::process::Child> = None;
397 let mut respawns: u64 = 0;
398 let mut last_spawn: Option<std::time::Instant> = None;
399 loop {
400 poll_events_daemon_child(&mut child, std::process::Child::try_wait);
401
402 let reachable = match connect_verified(&socket_path).await {
403 Ok(_stream) => true,
404 Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
405 tracing::warn!(
406 socket = %socket_path.display(),
407 error = %error,
408 "events socket answered by a foreign uid; treating as unreachable"
409 );
410 false
411 }
412 Err(_) => false,
413 };
414 if reachable {
415 readiness.mark_ready();
416 }
417 let spawn_due = last_spawn.is_none_or(|at| at.elapsed() >= PROBE_INTERVAL);
418 if !reachable && child.is_none() && spawn_due {
419 last_spawn = Some(std::time::Instant::now());
420 match std::env::current_exe() {
421 Ok(exe) => {
422 let spawned = events_daemon_command(
423 &exe,
424 &db_path,
425 &socket_path,
426 wal_ceiling,
427 disk_guard,
428 &volume_lock_dir,
429 )
430 .spawn();
431 match spawned {
432 Ok(spawned_child) => {
433 respawns += 1;
434 tracing::info!(
435 pid = spawned_child.id(),
436 respawns,
437 socket = %socket_path.display(),
438 "spawned events daemon"
439 );
440 child = Some(spawned_child);
441 }
442 Err(error) => {
443 tracing::warn!(error = %error, "failed to spawn events daemon");
444 }
445 }
446 }
447 Err(error) => {
448 tracing::warn!(error = %error, "cannot resolve current executable for events daemon spawn");
449 }
450 }
451 }
452 let pause = if readiness.awaiting() {
453 FAST_PROBE_INTERVAL
454 } else if readiness.is_ready() {
455 PROBE_INTERVAL
456 } else {
457 UNANSWERED_PROBE_INTERVAL
458 };
459 tokio::select! {
460 _ = shutdown.cancelled() => break,
461 _ = tokio::time::sleep(pause) => {}
462 }
463 }
464 if let Some(mut c) = child.take() {
465 let _ = c.kill();
469 let _ = c.wait();
470 tracing::info!("events daemon child stopped with supervisor shutdown");
471 }
472}
473
474#[cfg(unix)]
475fn poll_events_daemon_child(
476 child: &mut Option<std::process::Child>,
477 poll: impl FnOnce(&mut std::process::Child) -> std::io::Result<Option<std::process::ExitStatus>>,
478) {
479 let Some(c) = child.as_mut() else {
480 return;
481 };
482 match poll(c) {
483 Ok(Some(status)) => {
484 tracing::info!(%status, "events daemon child exited");
485 *child = None;
486 }
487 Ok(None) => {}
488 Err(error) if error.raw_os_error() == Some(libc::ECHILD) => {
489 tracing::warn!(%error, "events daemon child is no longer waitable; clearing handle");
491 *child = None;
492 }
493 Err(error) => {
494 tracing::warn!(%error, "cannot poll events daemon child; retaining handle for retry");
495 }
496 }
497}
498
499#[cfg(all(test, unix))]
500#[path = "daemon_reaping_tests.rs"]
501mod reaping_tests;
502
503#[cfg(unix)]
504pub(super) fn events_daemon_command(
505 executable: &Path,
506 db_path: &Path,
507 socket_path: &Path,
508 wal_ceiling: WalCeilingPolicy,
509 disk_guard: khive_db::EffectiveDiskGuardConfig,
510 volume_lock_dir: &Path,
511) -> std::process::Command {
512 let source = match wal_ceiling.source {
513 khive_db::WalCeilingSource::BackendField => "backend_field",
514 khive_db::WalCeilingSource::Environment => "environment",
515 khive_db::WalCeilingSource::Default => "default",
516 };
517 let mut command = std::process::Command::new(executable);
518 command
519 .arg("events-daemon")
520 .arg("--db")
521 .arg(db_path)
522 .arg("--socket")
523 .arg(socket_path)
524 .arg("--wal-ceiling-bytes")
525 .arg(wal_ceiling.bytes.to_string())
526 .arg("--wal-ceiling-source")
527 .arg(source)
528 .arg("--disk-reserve-bytes")
529 .arg(disk_guard.reserve_bytes.to_string())
530 .arg("--disk-guard-deadline-ms")
531 .arg(disk_guard.guard_deadline_ms.to_string())
532 .arg("--disk-reserve-source")
533 .arg(disk_guard.reserve_source.as_str())
534 .arg("--disk-deadline-source")
535 .arg(disk_guard.deadline_source.as_str())
536 .arg("--disk-legacy-environment-present")
537 .arg(disk_guard.legacy_environment_present.to_string())
538 .arg("--volume-lock-dir")
539 .arg(volume_lock_dir)
540 .stdin(std::process::Stdio::null())
541 .stdout(std::process::Stdio::null())
542 .stderr(std::process::Stdio::null());
543 command
544}
545
546#[cfg(unix)]
565pub async fn run_events_daemon(db_path: &Path, socket_path: &Path) -> anyhow::Result<()> {
566 let (wal_ceiling, disk_guard, volume_lock_dir) = standalone_daemon_policies(db_path)?;
567 run_events_daemon_with_policies(
568 db_path,
569 socket_path,
570 wal_ceiling,
571 disk_guard,
572 volume_lock_dir,
573 )
574 .await
575}
576
577#[cfg(unix)]
579pub async fn run_events_daemon_with_policies(
580 db_path: &Path,
581 socket_path: &Path,
582 wal_ceiling: WalCeilingPolicy,
583 disk_guard: khive_db::EffectiveDiskGuardConfig,
584 volume_lock_dir: PathBuf,
585) -> anyhow::Result<()> {
586 disk_guard.validate()?;
587 wal_ceiling
588 .validate_static(true, true, false)
589 .map_err(crate::error::RuntimeError::from)?;
590 let db_path = &absolutize(db_path);
593 let socket_path = &absolutize(socket_path);
594 validate_events_socket_path(socket_path)?;
595 if let Some(parent) = socket_path.parent() {
599 std::fs::create_dir_all(parent)?;
600 crate::daemon::ensure_socket_dir_is_trusted(parent)?;
601 }
602 let _guard = match acquire_events_daemon_guard_outcome(socket_path) {
603 EventsDaemonGuardAcquisition::Held(guard) => guard,
604 refusal => {
605 tracing::info!(
606 socket = %socket_path.display(),
607 reason = ?refusal,
608 "events daemon lock unavailable; exiting"
609 );
610 return Ok(());
611 }
612 };
613 ensure_events_db_owner_only(db_path)?;
614 let before_open = harden_events_db_sidecars(db_path)?;
627 let backend = Arc::new(
628 StorageBackend::sqlite_with_max_readers_and_policies(
629 db_path,
630 None,
631 wal_ceiling,
632 disk_guard,
633 volume_lock_dir,
634 )
635 .map_err(crate::error::RuntimeError::from)?,
636 );
637 backend.events()?;
639 verify_events_db_owner_only_unopened(db_path, &before_open)?;
640
641 if let Some(delay) = harness_duration_ms("KHIVE_TEST_EVENTS_DAEMON_BIND_DELAY_MS") {
644 tracing::info!(
645 delay_ms = delay.as_millis() as u64,
646 "test harness: delaying events socket bind"
647 );
648 tokio::time::sleep(delay).await;
649 }
650 if socket_path.exists() {
651 std::fs::remove_file(socket_path)?;
652 }
653 let listener = UnixListener::bind(socket_path)?;
654 {
655 use std::os::unix::fs::PermissionsExt;
656 if let Err(e) =
660 std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))
661 {
662 drop(listener);
663 let _ = std::fs::remove_file(socket_path);
664 anyhow::bail!(
665 "refusing to serve events: cannot chmod 0600 {}: {e}. The events socket must \
666 be owner-only.",
667 socket_path.display()
668 );
669 }
670 }
671 let daemon_euid = unsafe { libc::geteuid() };
672 let stores: NamespaceStores = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
678 let connections = Arc::new(tokio::sync::Semaphore::new(MAX_EVENTS_CONNECTIONS));
679 let frame_budget = Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_REQUEST_BYTES));
680 tracing::info!(
681 socket = %socket_path.display(),
682 db = %db_path.display(),
683 wal_ceiling_configured_bytes = wal_ceiling.bytes,
684 wal_ceiling_effective_bytes = wal_ceiling.effective_bytes(backend.is_read_only()),
685 wal_ceiling_source = ?wal_ceiling.source,
686 "events daemon listening"
687 );
688
689 loop {
690 let (stream, _addr) = match listener.accept().await {
691 Ok(pair) => pair,
692 Err(error) => {
693 tracing::warn!(error = %error, "events daemon accept failed");
694 continue;
695 }
696 };
697 match crate::daemon::peer_uid(&stream) {
698 Ok(uid) if crate::daemon::uid_is_permitted(uid, daemon_euid) => {}
699 Ok(uid) => {
700 tracing::warn!(peer_uid = uid, "events daemon rejected foreign-uid peer");
701 continue;
702 }
703 Err(error) => {
704 tracing::warn!(error = %error, "events daemon could not read peer credentials");
705 continue;
706 }
707 }
708 let permit = match Arc::clone(&connections).try_acquire_owned() {
709 Ok(permit) => permit,
710 Err(_) => {
711 tracing::warn!(
714 cap = MAX_EVENTS_CONNECTIONS,
715 "events daemon at connection cap; dropping new connection"
716 );
717 continue;
718 }
719 };
720 let backend = Arc::clone(&backend);
721 let stores = Arc::clone(&stores);
722 let frame_budget = Arc::clone(&frame_budget);
723 crate::daemon::spawn_named_tracked_task("events_connection", async move {
724 serve_events_conn(stream, backend, stores, frame_budget).await;
725 drop(permit);
726 });
727 }
728}