use super::*;
#[cfg(unix)]
pub struct EventsDaemonGuard {
pub(super) _file: std::fs::File,
}
#[cfg(unix)]
pub fn try_acquire_events_daemon_guard(socket_path: &Path) -> Option<EventsDaemonGuard> {
match acquire_events_daemon_guard_outcome(socket_path) {
EventsDaemonGuardAcquisition::Held(guard) => Some(guard),
_ => None,
}
}
#[cfg(unix)]
pub(super) enum EventsDaemonGuardAcquisition {
Held(EventsDaemonGuard),
Contended,
OpenFailed(std::io::Error),
HardeningRefused(std::io::Error),
}
#[cfg(unix)]
impl std::fmt::Debug for EventsDaemonGuardAcquisition {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Held(_) => f.write_str("Held"),
Self::Contended => f.write_str("Contended"),
Self::OpenFailed(error) => f.debug_tuple("OpenFailed").field(error).finish(),
Self::HardeningRefused(error) => {
f.debug_tuple("HardeningRefused").field(error).finish()
}
}
}
}
#[cfg(unix)]
pub(super) fn acquire_events_daemon_guard_outcome(
socket_path: &Path,
) -> EventsDaemonGuardAcquisition {
use EventsDaemonGuardAcquisition::{Contended, HardeningRefused, Held, OpenFailed};
let lock_path = socket_path.with_extension("lock");
if let Some(parent) = lock_path.parent() {
if let Err(error) = std::fs::create_dir_all(parent) {
return OpenFailed(error);
}
}
let opened = khive_fs::opened_file::open_writable_file_nofollow(
&lock_path,
khive_fs::fd_relative::OpenFileOptions {
read_write: false,
create: khive_fs::fd_relative::Create::IfMissing,
nonblock: false,
mode: 0o600,
},
);
let file = match opened {
Ok(file) => file,
Err(error) if error.raw_os_error() == Some(libc::ELOOP) => {
return HardeningRefused(error);
}
Err(error) => return OpenFailed(error),
};
if let Err(error) = file.set_permissions(
<std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
) {
return HardeningRefused(error);
}
use std::os::fd::AsRawFd;
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
if rc == 0 {
Held(EventsDaemonGuard { _file: file })
} else {
let error = std::io::Error::last_os_error();
if error.raw_os_error() == Some(libc::EWOULDBLOCK)
|| error.raw_os_error() == Some(libc::EAGAIN)
{
Contended
} else {
HardeningRefused(error)
}
}
}
pub(super) fn events_db_targets(db_path: &Path) -> [PathBuf; 3] {
["", "-wal", "-shm"].map(|suffix| {
let mut name = db_path.as_os_str().to_os_string();
name.push(suffix);
PathBuf::from(name)
})
}
pub(super) fn refuse_events_db_symlinks(db_path: &Path) -> anyhow::Result<()> {
for path in events_db_targets(db_path) {
match std::fs::symlink_metadata(&path) {
Ok(meta) if meta.file_type().is_symlink() => anyhow::bail!(
"refusing to serve events: {} is a symlink; the events database and its \
sidecars must be regular files",
path.display()
),
Ok(_) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => anyhow::bail!(
"refusing to serve events: cannot inspect {}: {e}",
path.display()
),
}
}
Ok(())
}
#[cfg(unix)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) struct EventsFileIdentity {
pub(super) device: u64,
pub(super) inode: u64,
}
#[cfg(unix)]
impl EventsFileIdentity {
pub(super) fn from_metadata(metadata: &std::fs::Metadata) -> Self {
use std::os::unix::fs::MetadataExt;
Self {
device: metadata.dev(),
inode: metadata.ino(),
}
}
}
#[cfg(unix)]
pub(super) type EventsDbIdentities = [Option<EventsFileIdentity>; 3];
#[cfg(unix)]
pub(super) fn ensure_events_db_parent_trusted(db_path: &Path) -> anyhow::Result<()> {
let db_path = absolutize(db_path);
let parent = db_path.parent().filter(|p| !p.as_os_str().is_empty());
match parent {
Some(dir) => crate::daemon::ensure_store_dir_is_trusted(dir, &db_path).map_err(|e| {
anyhow::anyhow!("refusing to serve events from an untrusted directory: {e}")
}),
None => Ok(()),
}
}
#[cfg(unix)]
pub(super) fn ensure_events_db_owner_only(db_path: &Path) -> anyhow::Result<()> {
use std::os::unix::fs::OpenOptionsExt;
refuse_events_db_symlinks(db_path)?;
if let Some(parent) = db_path.parent() {
std::fs::create_dir_all(parent)?;
}
ensure_events_db_parent_trusted(db_path)?;
match std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(db_path)
{
Ok(_created) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()),
Err(e) => Err(anyhow::anyhow!(
"refusing to serve events: cannot create {} owner-only: {e}",
db_path.display()
)),
}
}
#[cfg(unix)]
pub(super) fn harden_events_db_sidecars(db_path: &Path) -> anyhow::Result<EventsDbIdentities> {
use khive_fs::opened_file::ContainedOpenError;
use std::os::unix::fs::PermissionsExt;
let mut identities = [None; 3];
for (index, path) in events_db_targets(db_path).into_iter().enumerate() {
let file = match khive_fs::opened_file::open_regular_file_nofollow(&path) {
Ok(file) => file,
Err(ContainedOpenError::Open(e)) if e.kind() == std::io::ErrorKind::NotFound => {
continue;
}
Err(ContainedOpenError::Open(e)) => anyhow::bail!(
"refusing to serve events: cannot open {} without following symlinks: \
{e}. The events database and its sidecars must be regular files.",
path.display()
),
Err(ContainedOpenError::Metadata(e)) => return Err(e.into()),
Err(_) => anyhow::bail!(
"refusing to serve events: {} is not a regular file. The events \
database and its sidecars must be regular files.",
path.display()
),
};
let metadata = file.metadata()?;
if !metadata.file_type().is_file() {
anyhow::bail!(
"refusing to serve events: {} is not a regular file. The events \
database and its sidecars must be regular files.",
path.display()
);
}
file.set_permissions(std::fs::Permissions::from_mode(0o600))
.map_err(|e| {
anyhow::anyhow!(
"refusing to serve events: cannot chmod 0600 {}: {e}. The events \
database and its sidecars must be owner-only.",
path.display()
)
})?;
identities[index] = Some(EventsFileIdentity::from_metadata(&metadata));
}
Ok(identities)
}
#[cfg(unix)]
pub(super) fn verify_events_db_owner_only_unopened(
db_path: &Path,
before: &EventsDbIdentities,
) -> anyhow::Result<()> {
use std::os::unix::fs::PermissionsExt;
for (index, path) in events_db_targets(db_path).into_iter().enumerate() {
let metadata = match std::fs::symlink_metadata(&path) {
Ok(metadata) => metadata,
Err(e) if e.kind() == std::io::ErrorKind::NotFound && index != 0 => continue,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => anyhow::bail!(
"refusing to serve events: main database {} disappeared after SQLite open; \
refusing to serve an unlinked database",
path.display()
),
Err(e) => {
anyhow::bail!(
"refusing to serve events: cannot stat {}: {e}",
path.display()
)
}
};
if !metadata.file_type().is_file() {
anyhow::bail!(
"refusing to serve events: {} is not a regular file. The events database \
and its sidecars must be regular files.",
path.display()
);
}
let mode = metadata.permissions().mode() & 0o777;
if mode & 0o077 != 0 {
anyhow::bail!(
"refusing to serve events: {} is mode {mode:03o}, not owner-only. The events \
database and its sidecars must be owner-only.",
path.display()
);
}
if before[index]
.is_some_and(|identity| identity != EventsFileIdentity::from_metadata(&metadata))
{
anyhow::bail!(
"refusing to serve events: {} changed identity between pre-open hardening \
and post-open verification",
path.display()
);
}
}
Ok(())
}
#[cfg(unix)]
pub const EVENTS_READINESS_WINDOW: Duration = Duration::from_secs(10);
#[cfg(unix)]
#[derive(Clone, Debug)]
pub struct EventsReadiness {
pub(super) socket: PathBuf,
pub(super) deadline: tokio::time::Instant,
pub(super) ready: tokio::sync::watch::Receiver<bool>,
}
#[cfg(unix)]
#[derive(Debug)]
pub struct EventsReadinessSignal {
pub(super) deadline: tokio::time::Instant,
pub(super) ready: tokio::sync::watch::Sender<bool>,
}
#[cfg(unix)]
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum EventsNotReadyReason {
WindowElapsed,
SupervisorStopped,
}
#[cfg(unix)]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EventsNotReady {
pub socket: PathBuf,
pub reason: EventsNotReadyReason,
}
#[cfg(unix)]
impl std::fmt::Display for EventsNotReady {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self.reason {
EventsNotReadyReason::WindowElapsed => write!(
f,
"events daemon at {} has not answered since this daemon started; \
the request was refused before dispatch",
self.socket.display()
),
EventsNotReadyReason::SupervisorStopped => write!(
f,
"events daemon supervision for {} stopped before the socket answered; \
the request was refused before dispatch",
self.socket.display()
),
}
}
}
#[cfg(unix)]
pub fn events_readiness(
socket: PathBuf,
window: Duration,
) -> (EventsReadinessSignal, EventsReadiness) {
let deadline = tokio::time::Instant::now() + window;
let (sender, receiver) = tokio::sync::watch::channel(false);
(
EventsReadinessSignal {
deadline,
ready: sender,
},
EventsReadiness {
socket,
deadline,
ready: receiver,
},
)
}
#[cfg(unix)]
impl EventsReadinessSignal {
pub fn mark_ready(&self) {
self.ready
.send_if_modified(|ready| !std::mem::replace(ready, true));
}
pub fn awaiting(&self) -> bool {
!*self.ready.borrow() && tokio::time::Instant::now() < self.deadline
}
pub fn is_ready(&self) -> bool {
*self.ready.borrow()
}
}
#[cfg(unix)]
impl EventsReadiness {
pub fn socket(&self) -> &Path {
&self.socket
}
pub fn is_ready(&self) -> bool {
*self.ready.borrow()
}
pub async fn wait(&self) -> Result<(), EventsNotReady> {
let mut ready = self.ready.clone();
let outcome = tokio::time::timeout_at(self.deadline, ready.wait_for(|ready| *ready))
.await
.map(|answered| answered.map(|_| ()));
let refused = |reason| EventsNotReady {
socket: self.socket.clone(),
reason,
};
match outcome {
Ok(Ok(())) => Ok(()),
Ok(Err(_)) => Err(refused(EventsNotReadyReason::SupervisorStopped)),
Err(_) if self.is_ready() => Ok(()),
Err(_) => Err(refused(EventsNotReadyReason::WindowElapsed)),
}
}
}
#[cfg(unix)]
pub fn events_readiness_window() -> Duration {
harness_duration_ms("KHIVE_TEST_EVENTS_READY_WINDOW_MS").unwrap_or(EVENTS_READINESS_WINDOW)
}
#[cfg(unix)]
pub(crate) fn harness_duration_ms(name: &str) -> Option<Duration> {
if std::env::var("KHIVE_TEST_HARNESS").as_deref() != Ok("1") {
return None;
}
std::env::var(name)
.ok()?
.trim()
.parse::<u64>()
.ok()
.map(Duration::from_millis)
}
#[cfg(unix)]
pub async fn supervise_events_daemon(db_path: PathBuf, socket_path: PathBuf) {
match standalone_daemon_policies(&db_path) {
Ok((wal_ceiling, disk_guard, volume_lock_dir)) => {
supervise_events_daemon_with_policies(
db_path,
socket_path,
wal_ceiling,
disk_guard,
volume_lock_dir,
)
.await;
}
Err(error) => {
tracing::warn!(%error, "invalid events daemon policy; events supervisor not started");
}
}
}
#[cfg(unix)]
pub async fn supervise_events_daemon_with_policies(
db_path: PathBuf,
socket_path: PathBuf,
wal_ceiling: WalCeilingPolicy,
disk_guard: khive_db::EffectiveDiskGuardConfig,
volume_lock_dir: PathBuf,
) {
let (readiness, _unobserved) = events_readiness(socket_path.clone(), EVENTS_READINESS_WINDOW);
supervise_events_daemon_with_readiness(
db_path,
socket_path,
wal_ceiling,
disk_guard,
volume_lock_dir,
readiness,
)
.await;
}
#[cfg(unix)]
pub async fn supervise_events_daemon_with_readiness(
db_path: PathBuf,
socket_path: PathBuf,
wal_ceiling: WalCeilingPolicy,
disk_guard: khive_db::EffectiveDiskGuardConfig,
volume_lock_dir: PathBuf,
readiness: EventsReadinessSignal,
) {
if let Err(error) = disk_guard.validate() {
tracing::warn!(%error, "invalid disk policy; events supervisor not started");
return;
}
if let Err(error) = wal_ceiling.validate_static(true, true, false) {
tracing::warn!(error = %error, "invalid WAL ceiling; events supervisor not started");
return;
}
const PROBE_INTERVAL: Duration = Duration::from_secs(15);
const FAST_PROBE_INTERVAL: Duration = Duration::from_millis(20);
const UNANSWERED_PROBE_INTERVAL: Duration = Duration::from_secs(1);
let shutdown = crate::daemon::daemon_shutdown_token();
let mut child: Option<std::process::Child> = None;
let mut respawns: u64 = 0;
let mut last_spawn: Option<std::time::Instant> = None;
loop {
if let Some(c) = child.as_mut() {
match c.try_wait() {
Ok(Some(status)) => {
tracing::info!(%status, "events daemon child exited");
child = None;
}
Ok(None) => {}
Err(error) => {
tracing::warn!(error = %error, "cannot poll events daemon child; dropping handle");
child = None;
}
}
}
let reachable = match connect_verified(&socket_path).await {
Ok(_stream) => true,
Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
tracing::warn!(
socket = %socket_path.display(),
error = %error,
"events socket answered by a foreign uid; treating as unreachable"
);
false
}
Err(_) => false,
};
if reachable {
readiness.mark_ready();
}
let spawn_due = last_spawn.is_none_or(|at| at.elapsed() >= PROBE_INTERVAL);
if !reachable && child.is_none() && spawn_due {
last_spawn = Some(std::time::Instant::now());
match std::env::current_exe() {
Ok(exe) => {
let spawned = events_daemon_command(
&exe,
&db_path,
&socket_path,
wal_ceiling,
disk_guard,
&volume_lock_dir,
)
.spawn();
match spawned {
Ok(spawned_child) => {
respawns += 1;
tracing::info!(
pid = spawned_child.id(),
respawns,
socket = %socket_path.display(),
"spawned events daemon"
);
child = Some(spawned_child);
}
Err(error) => {
tracing::warn!(error = %error, "failed to spawn events daemon");
}
}
}
Err(error) => {
tracing::warn!(error = %error, "cannot resolve current executable for events daemon spawn");
}
}
}
let pause = if readiness.awaiting() {
FAST_PROBE_INTERVAL
} else if readiness.is_ready() {
PROBE_INTERVAL
} else {
UNANSWERED_PROBE_INTERVAL
};
tokio::select! {
_ = shutdown.cancelled() => break,
_ = tokio::time::sleep(pause) => {}
}
}
if let Some(mut c) = child.take() {
let _ = c.kill();
let _ = c.wait();
tracing::info!("events daemon child stopped with supervisor shutdown");
}
}
#[cfg(unix)]
pub(super) fn events_daemon_command(
executable: &Path,
db_path: &Path,
socket_path: &Path,
wal_ceiling: WalCeilingPolicy,
disk_guard: khive_db::EffectiveDiskGuardConfig,
volume_lock_dir: &Path,
) -> std::process::Command {
let source = match wal_ceiling.source {
khive_db::WalCeilingSource::BackendField => "backend_field",
khive_db::WalCeilingSource::Environment => "environment",
khive_db::WalCeilingSource::Default => "default",
};
let mut command = std::process::Command::new(executable);
command
.arg("events-daemon")
.arg("--db")
.arg(db_path)
.arg("--socket")
.arg(socket_path)
.arg("--wal-ceiling-bytes")
.arg(wal_ceiling.bytes.to_string())
.arg("--wal-ceiling-source")
.arg(source)
.arg("--disk-reserve-bytes")
.arg(disk_guard.reserve_bytes.to_string())
.arg("--disk-guard-deadline-ms")
.arg(disk_guard.guard_deadline_ms.to_string())
.arg("--disk-reserve-source")
.arg(disk_guard.reserve_source.as_str())
.arg("--disk-deadline-source")
.arg(disk_guard.deadline_source.as_str())
.arg("--disk-legacy-environment-present")
.arg(disk_guard.legacy_environment_present.to_string())
.arg("--volume-lock-dir")
.arg(volume_lock_dir)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null());
command
}
#[cfg(unix)]
pub async fn run_events_daemon(db_path: &Path, socket_path: &Path) -> anyhow::Result<()> {
let (wal_ceiling, disk_guard, volume_lock_dir) = standalone_daemon_policies(db_path)?;
run_events_daemon_with_policies(
db_path,
socket_path,
wal_ceiling,
disk_guard,
volume_lock_dir,
)
.await
}
#[cfg(unix)]
pub async fn run_events_daemon_with_policies(
db_path: &Path,
socket_path: &Path,
wal_ceiling: WalCeilingPolicy,
disk_guard: khive_db::EffectiveDiskGuardConfig,
volume_lock_dir: PathBuf,
) -> anyhow::Result<()> {
disk_guard.validate()?;
wal_ceiling
.validate_static(true, true, false)
.map_err(crate::error::RuntimeError::from)?;
let db_path = &absolutize(db_path);
let socket_path = &absolutize(socket_path);
validate_events_socket_path(socket_path)?;
if let Some(parent) = socket_path.parent() {
std::fs::create_dir_all(parent)?;
crate::daemon::ensure_socket_dir_is_trusted(parent)?;
}
let _guard = match acquire_events_daemon_guard_outcome(socket_path) {
EventsDaemonGuardAcquisition::Held(guard) => guard,
refusal => {
tracing::info!(
socket = %socket_path.display(),
reason = ?refusal,
"events daemon lock unavailable; exiting"
);
return Ok(());
}
};
ensure_events_db_owner_only(db_path)?;
let before_open = harden_events_db_sidecars(db_path)?;
let backend = Arc::new(
StorageBackend::sqlite_with_max_readers_and_policies(
db_path,
None,
wal_ceiling,
disk_guard,
volume_lock_dir,
)
.map_err(crate::error::RuntimeError::from)?,
);
backend.events()?;
verify_events_db_owner_only_unopened(db_path, &before_open)?;
if let Some(delay) = harness_duration_ms("KHIVE_TEST_EVENTS_DAEMON_BIND_DELAY_MS") {
tracing::info!(
delay_ms = delay.as_millis() as u64,
"test harness: delaying events socket bind"
);
tokio::time::sleep(delay).await;
}
if socket_path.exists() {
std::fs::remove_file(socket_path)?;
}
let listener = UnixListener::bind(socket_path)?;
{
use std::os::unix::fs::PermissionsExt;
if let Err(e) =
std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))
{
drop(listener);
let _ = std::fs::remove_file(socket_path);
anyhow::bail!(
"refusing to serve events: cannot chmod 0600 {}: {e}. The events socket must \
be owner-only.",
socket_path.display()
);
}
}
let daemon_euid = unsafe { libc::geteuid() };
let stores: NamespaceStores = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
let connections = Arc::new(tokio::sync::Semaphore::new(MAX_EVENTS_CONNECTIONS));
let frame_budget = Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_REQUEST_BYTES));
tracing::info!(
socket = %socket_path.display(),
db = %db_path.display(),
wal_ceiling_configured_bytes = wal_ceiling.bytes,
wal_ceiling_effective_bytes = wal_ceiling.effective_bytes(backend.is_read_only()),
wal_ceiling_source = ?wal_ceiling.source,
"events daemon listening"
);
loop {
let (stream, _addr) = match listener.accept().await {
Ok(pair) => pair,
Err(error) => {
tracing::warn!(error = %error, "events daemon accept failed");
continue;
}
};
match crate::daemon::peer_uid(&stream) {
Ok(uid) if crate::daemon::uid_is_permitted(uid, daemon_euid) => {}
Ok(uid) => {
tracing::warn!(peer_uid = uid, "events daemon rejected foreign-uid peer");
continue;
}
Err(error) => {
tracing::warn!(error = %error, "events daemon could not read peer credentials");
continue;
}
}
let permit = match Arc::clone(&connections).try_acquire_owned() {
Ok(permit) => permit,
Err(_) => {
tracing::warn!(
cap = MAX_EVENTS_CONNECTIONS,
"events daemon at connection cap; dropping new connection"
);
continue;
}
};
let backend = Arc::clone(&backend);
let stores = Arc::clone(&stores);
let frame_budget = Arc::clone(&frame_budget);
crate::daemon::spawn_named_tracked_task("events_connection", async move {
serve_events_conn(stream, backend, stores, frame_budget).await;
drop(permit);
});
}
}