1use std::collections::BTreeMap;
41use std::path::{Path, PathBuf};
42#[cfg(unix)]
43use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
44use std::sync::Arc;
45#[cfg(unix)]
46use std::time::Duration;
47
48use async_trait::async_trait;
49use khive_db::{StorageBackend, WalCeilingPolicy};
50use khive_storage::event::{
51 EventGroupBy, EventPageQuery, EventPageWindow, IdempotentEventBatchResult,
52};
53use khive_storage::{
54 BatchWriteSummary, Event, EventFilter, EventStore, Page, PageRequest, StorageError,
55 StorageResult,
56};
57use serde::{Deserialize, Serialize};
58
59#[cfg(unix)]
60use tokio::net::{UnixListener, UnixStream};
61use uuid::Uuid;
62
63#[cfg(unix)]
64use crate::daemon::{read_frame, write_frame};
65
66pub const EVENTS_PROTOCOL_VERSION: u32 = 5;
83
84#[cfg(unix)]
88pub const DEFAULT_APPEND_QUEUE_BATCHES: usize = 4096;
89
90#[cfg(unix)]
94pub const DEFAULT_APPEND_QUEUE_BYTES: usize = 32 * 1024 * 1024;
95
96#[cfg(unix)]
101const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
102
103#[cfg(unix)]
105const FORWARDER_BACKOFF: Duration = Duration::from_secs(2);
106
107#[cfg(unix)]
114const MAX_EVENTS_CONNECTIONS: usize = 128;
115
116#[cfg(unix)]
124const CONN_IO_TIMEOUT: Duration = Duration::from_secs(60);
125
126#[cfg(unix)]
136const MAX_INFLIGHT_REQUEST_BYTES: usize = 64 * 1024 * 1024;
137
138#[cfg(unix)]
143const MAX_CACHED_NAMESPACE_STORES: usize = 1024;
144
145const EVENTS_SYMLINK_HOP_BOUND: u32 = 40;
150
151pub const MAX_QUERY_EVENTS_PAGE_ROWS: u32 = 4096;
169
170pub fn events_db_path_beside(main_db: &Path) -> PathBuf {
185 let resolved = std::fs::canonicalize(main_db)
192 .unwrap_or_else(|_| resolve_dangling_final_component(main_db));
193 let mut name = resolved
194 .file_name()
195 .map(std::ffi::OsStr::to_os_string)
196 .unwrap_or_else(|| std::ffi::OsString::from("khive.db"));
197 name.push(".events.db");
198 let path = match resolved.parent().filter(|dir| !dir.as_os_str().is_empty()) {
199 Some(dir) => std::fs::canonicalize(dir)
200 .unwrap_or_else(|_| dir.to_path_buf())
201 .join(&name),
202 None => PathBuf::from(name),
203 };
204 absolutize(&path)
205}
206
207fn resolve_dangling_final_component(path: &Path) -> PathBuf {
214 let mut current = path.to_path_buf();
215 for _ in 0..EVENTS_SYMLINK_HOP_BOUND {
220 match std::fs::read_link(¤t) {
221 Ok(target) => {
222 current = if target.is_absolute() {
223 target
224 } else {
225 match current.parent().filter(|dir| !dir.as_os_str().is_empty()) {
226 Some(dir) => dir.join(target),
227 None => target,
228 }
229 };
230 }
231 Err(_) => break,
232 }
233 }
234 current
235}
236
237fn absolutize(path: &Path) -> PathBuf {
243 if path.is_absolute() {
244 return path.to_path_buf();
245 }
246 std::env::current_dir()
247 .map(|cwd| cwd.join(path))
248 .unwrap_or_else(|_| path.to_path_buf())
249}
250
251pub fn events_socket_path_beside(events_db: &Path) -> PathBuf {
261 events_db.with_extension("sock")
262}
263
264#[cfg(unix)]
265#[path = "events_split_socket_path.rs"]
266pub(crate) mod socket_path;
267#[cfg(unix)]
268pub use socket_path::validate_events_socket_path;
269
270#[derive(Debug, Clone)]
272pub struct EventsSplitConfig {
273 pub db_path: PathBuf,
276 pub socket_path: Option<PathBuf>,
280}
281
282#[cfg(unix)]
292type ClientMap = std::collections::HashMap<PathBuf, Arc<EventsSplitClient>>;
293type BackendMap = std::collections::HashMap<PathBuf, (bool, Arc<StorageBackend>)>;
297
298#[cfg(unix)]
299fn client_registry() -> &'static std::sync::Mutex<ClientMap> {
300 static REGISTRY: std::sync::OnceLock<std::sync::Mutex<ClientMap>> = std::sync::OnceLock::new();
301 REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
302}
303
304fn direct_backend_registry() -> &'static std::sync::Mutex<BackendMap> {
305 static REGISTRY: std::sync::OnceLock<std::sync::Mutex<BackendMap>> = std::sync::OnceLock::new();
306 REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
307}
308
309#[cfg(any(test, feature = "test-internals"))]
316#[doc(hidden)]
317pub struct TestRegistryGuard<'a> {
318 root: &'a Path,
319 canonical_root: PathBuf,
320}
321
322#[cfg(any(test, feature = "test-internals"))]
323impl<'a> TestRegistryGuard<'a> {
324 pub fn new(root: &'a Path) -> Self {
326 Self {
327 root,
328 canonical_root: std::fs::canonicalize(root).expect("existing unique fixture root"),
329 }
330 }
331
332 fn remove_entries<T>(
333 &self,
334 registry: &std::sync::Mutex<std::collections::HashMap<PathBuf, T>>,
335 ) -> Vec<T> {
336 let mut entries = registry
337 .lock()
338 .unwrap_or_else(std::sync::PoisonError::into_inner);
339 let keys: Vec<_> = entries
340 .keys()
341 .filter(|path| path.starts_with(self.root) || path.starts_with(&self.canonical_root))
342 .cloned()
343 .collect();
344 keys.into_iter()
345 .filter_map(|key| entries.remove(&key))
346 .collect()
347 }
348}
349
350#[cfg(any(test, feature = "test-internals"))]
351impl Drop for TestRegistryGuard<'_> {
352 fn drop(&mut self) {
353 let backends = self.remove_entries(direct_backend_registry());
356 #[cfg(unix)]
357 let clients = self.remove_entries(client_registry());
358 drop(backends);
359 #[cfg(unix)]
360 drop(clients);
361 }
362}
363
364#[cfg(unix)]
367pub fn client_for(socket_path: &Path) -> crate::error::RuntimeResult<Arc<EventsSplitClient>> {
368 let mut registry = client_registry()
369 .lock()
370 .unwrap_or_else(std::sync::PoisonError::into_inner);
371 if let Some(existing) = registry.get(socket_path) {
372 return Ok(Arc::clone(existing));
373 }
374 let client = EventsSplitClient::new(socket_path.to_path_buf())?;
375 registry.insert(socket_path.to_path_buf(), Arc::clone(&client));
376 Ok(client)
377}
378
379pub fn direct_backend_for(db_path: &Path) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
382 direct_backend_with_max_readers(db_path, false, None)
383}
384
385pub fn direct_backend_read_only_for(
390 db_path: &Path,
391) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
392 direct_backend_with_max_readers(db_path, true, None)
393}
394
395pub(crate) fn direct_backend_with_max_readers(
397 db_path: &Path,
398 read_only: bool,
399 max_readers: Option<usize>,
400) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
401 let mut config = crate::RuntimeConfig {
402 db_path: Some(db_path.to_path_buf()),
403 ..crate::RuntimeConfig::no_embeddings()
404 };
405 let wal_ceiling = config.resolve_wal_ceiling_policy(read_only)?;
406 let disk_guard = config.resolve_disk_guard_policy(read_only)?;
407 direct_backend_with_policies(
408 db_path,
409 read_only,
410 max_readers,
411 wal_ceiling,
412 disk_guard,
413 config.volume_lock_dir,
414 )
415}
416
417pub(crate) fn direct_backend_with_policies(
421 db_path: &Path,
422 read_only: bool,
423 max_readers: Option<usize>,
424 wal_ceiling: WalCeilingPolicy,
425 disk_guard: Option<khive_db::EffectiveDiskGuardConfig>,
426 volume_lock_dir: Option<PathBuf>,
427) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
428 wal_ceiling.validate_static(true, true, read_only)?;
429 if !read_only {
430 disk_guard
431 .ok_or_else(|| {
432 crate::error::RuntimeError::Internal("missing events disk policy".into())
433 })?
434 .validate()?;
435 khive_db::require_volume_lock_dir(volume_lock_dir.clone())?;
436 }
437 let mut registry = direct_backend_registry()
438 .lock()
439 .unwrap_or_else(std::sync::PoisonError::into_inner);
440 let absolute_path = absolutize(db_path);
441 let db_path = absolute_path.as_path();
442 refuse_events_db_symlinks(db_path)
443 .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
444 #[cfg(unix)]
445 if !read_only {
446 if let Some(parent) = db_path.parent().filter(|p| !p.as_os_str().is_empty()) {
447 let _ = std::fs::create_dir_all(parent);
448 }
449 }
450 #[cfg(unix)]
456 ensure_events_db_parent_trusted(db_path)
457 .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
458 let key = std::fs::canonicalize(db_path)
459 .or_else(|error| {
460 if error.kind() != std::io::ErrorKind::NotFound {
461 return Err(error);
462 }
463 let parent = db_path.parent().ok_or(error)?;
464 let name = db_path.file_name().ok_or_else(|| {
465 std::io::Error::new(
466 std::io::ErrorKind::InvalidInput,
467 "events path has no file name",
468 )
469 })?;
470 std::fs::canonicalize(parent).map(|parent| parent.join(name))
471 })
472 .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
473 if let Some((existing_read_only, existing)) = registry.get(&key) {
474 if *existing_read_only != read_only {
475 let existing_mode = if *existing_read_only {
476 "read-only"
477 } else {
478 "writable"
479 };
480 let requested_mode = if read_only { "read-only" } else { "writable" };
481 return Err(crate::error::RuntimeError::InvalidInput(format!(
482 "events database {} is already open {existing_mode} in this process; cannot \
483 open it {requested_mode}; read-only events access requires a separate frozen snapshot",
484 key.display()
485 )));
486 }
487 let numbers = |p: Option<khive_db::EffectiveDiskGuardConfig>| {
488 p.map(|p| (p.reserve_bytes, p.guard_deadline_ms))
489 };
490 if numbers(existing.pool().effective_disk_guard_config()) != numbers(disk_guard) {
491 return Err(crate::error::RuntimeError::Internal(
492 "events database is already open with a different disk reserve/deadline policy"
493 .into(),
494 ));
495 }
496 if !read_only && existing.pool().config().volume_lock_dir != volume_lock_dir {
497 return Err(crate::error::RuntimeError::Internal(
498 "events database is already open with a different volume-lock directory".into(),
499 ));
500 }
501 let existing_bytes = existing.pool().config().wal_ceiling.bytes;
502 if existing_bytes != wal_ceiling.bytes {
503 return Err(khive_db::SqliteError::InvalidConfig(format!(
504 "events database {} is already open with wal_ceiling_bytes={existing_bytes}; \
505 requested {}; drain and restart before changing the WAL ceiling",
506 key.display(),
507 wal_ceiling.bytes
508 ))
509 .into());
510 }
511 return Ok(Arc::clone(existing));
512 }
513 let db_path = key.as_path();
514 #[cfg(unix)]
520 if !read_only {
521 use std::os::unix::fs::OpenOptionsExt;
522 let _ = std::fs::OpenOptions::new()
523 .write(true)
524 .create_new(true)
525 .mode(0o600)
526 .open(db_path);
527 let _before_open = harden_events_db_sidecars(db_path)
530 .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
531 }
532 let backend = Arc::new(if read_only {
533 StorageBackend::sqlite_read_only_with_max_readers_and_wal_ceiling(
534 db_path,
535 max_readers,
536 wal_ceiling,
537 )?
538 } else {
539 StorageBackend::sqlite_with_max_readers_and_policies(
540 db_path,
541 max_readers,
542 wal_ceiling,
543 disk_guard.ok_or_else(|| {
544 crate::error::RuntimeError::Internal("missing events disk policy".into())
545 })?,
546 khive_db::require_volume_lock_dir(volume_lock_dir)?,
547 )?
548 });
549 registry.insert(key, (read_only, Arc::clone(&backend)));
550 Ok(backend)
551}
552
553#[cfg(unix)]
556pub fn forwarding_metrics(socket_path: &Path) -> Option<EventsForwardingMetrics> {
557 let registry = client_registry()
558 .lock()
559 .unwrap_or_else(std::sync::PoisonError::into_inner);
560 registry.get(socket_path).map(|client| client.metrics())
561}
562
563#[derive(Debug, Serialize, Deserialize)]
569#[serde(tag = "op", rename_all = "snake_case")]
570pub enum EventsRequest {
571 AppendEvents {
575 protocol_version: u32,
576 namespace: String,
577 events: Vec<Event>,
578 },
579 AppendEventsIdempotent {
581 protocol_version: u32,
582 namespace: String,
583 events: Vec<Event>,
584 },
585 GetEvent {
586 protocol_version: u32,
587 namespace: String,
588 id: Uuid,
589 },
590 QueryEvents {
591 protocol_version: u32,
592 namespace: String,
593 filter: EventFilter,
594 page: PageRequest,
595 },
596 QueryEventPage {
597 protocol_version: u32,
598 namespace: String,
599 query: EventPageQuery,
600 },
601 CountEvents {
602 protocol_version: u32,
603 namespace: String,
604 filter: EventFilter,
605 },
606 CountEventsGrouped {
607 protocol_version: u32,
608 namespace: String,
609 filter: EventFilter,
610 group_by: EventGroupBy,
611 },
612}
613
614impl EventsRequest {
615 fn protocol_version(&self) -> u32 {
616 match self {
617 Self::AppendEvents {
618 protocol_version, ..
619 }
620 | Self::AppendEventsIdempotent {
621 protocol_version, ..
622 }
623 | Self::GetEvent {
624 protocol_version, ..
625 }
626 | Self::QueryEvents {
627 protocol_version, ..
628 }
629 | Self::QueryEventPage {
630 protocol_version, ..
631 }
632 | Self::CountEvents {
633 protocol_version, ..
634 }
635 | Self::CountEventsGrouped {
636 protocol_version, ..
637 } => *protocol_version,
638 }
639 }
640
641 fn namespace(&self) -> &str {
642 match self {
643 Self::AppendEvents { namespace, .. }
644 | Self::AppendEventsIdempotent { namespace, .. }
645 | Self::GetEvent { namespace, .. }
646 | Self::QueryEvents { namespace, .. }
647 | Self::QueryEventPage { namespace, .. }
648 | Self::CountEvents { namespace, .. }
649 | Self::CountEventsGrouped { namespace, .. } => namespace,
650 }
651 }
652}
653
654#[derive(Debug, Serialize, Deserialize)]
656#[serde(tag = "kind", rename_all = "snake_case")]
657pub enum EventsResponse {
658 Appended {
659 summary: BatchWriteSummary,
660 },
661 Idempotent {
662 result: IdempotentEventBatchResult,
663 },
664 Event {
665 event: Option<Event>,
666 },
667 Pageful {
668 page: Page<Event>,
669 },
670 EventPageWindow {
671 window: EventPageWindow,
672 },
673 Count {
674 count: u64,
675 },
676 GroupedCount {
677 counts: BTreeMap<String, u64>,
678 },
679 Error {
689 message: String,
690 retryable: bool,
691 #[serde(default)]
692 writer_task_failure: Option<WireWriterTaskFailure>,
693 #[serde(default, skip_serializing_if = "Option::is_none")]
695 sqlite_write_failure: Option<khive_storage::error::SqliteWriteFailure>,
696 },
697}
698
699#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
702#[serde(tag = "kind", rename_all = "snake_case")]
703pub enum WireWriterTaskFailure {
704 RequestFailed {
705 request_state: WireWriterTaskState,
706 },
707 TaskTerminated {
708 request_state: WireWriterTaskState,
709 #[serde(default, skip_serializing_if = "Option::is_none")]
710 sqlite_full_codes: Option<(i32, i32)>,
711 },
712}
713
714#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
718#[serde(rename_all = "snake_case")]
719pub enum WireWriterTaskState {
720 NotStarted,
721 TransactionRolledBack,
722 SideEffectsUnknown,
723}
724
725impl From<khive_storage::WriterTaskRequestState> for WireWriterTaskState {
726 fn from(state: khive_storage::WriterTaskRequestState) -> Self {
727 use khive_storage::WriterTaskRequestState as S;
728 match state {
729 S::NotStarted => Self::NotStarted,
730 S::TransactionRolledBack => Self::TransactionRolledBack,
731 S::SideEffectsUnknown => Self::SideEffectsUnknown,
732 }
733 }
734}
735
736impl From<WireWriterTaskState> for khive_storage::WriterTaskRequestState {
737 fn from(state: WireWriterTaskState) -> Self {
738 use khive_storage::WriterTaskRequestState as S;
739 match state {
740 WireWriterTaskState::NotStarted => S::NotStarted,
741 WireWriterTaskState::TransactionRolledBack => S::TransactionRolledBack,
742 WireWriterTaskState::SideEffectsUnknown => S::SideEffectsUnknown,
743 }
744 }
745}
746
747#[cfg(unix)]
755pub struct EventsDaemonGuard {
756 _file: std::fs::File,
757}
758
759#[cfg(unix)]
763pub fn try_acquire_events_daemon_guard(socket_path: &Path) -> Option<EventsDaemonGuard> {
764 match acquire_events_daemon_guard_outcome(socket_path) {
765 EventsDaemonGuardAcquisition::Held(guard) => Some(guard),
766 _ => None,
767 }
768}
769
770#[cfg(unix)]
773enum EventsDaemonGuardAcquisition {
774 Held(EventsDaemonGuard),
775 Contended,
776 OpenFailed(std::io::Error),
777 HardeningRefused(std::io::Error),
778}
779
780#[cfg(unix)]
781impl std::fmt::Debug for EventsDaemonGuardAcquisition {
782 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
783 match self {
784 Self::Held(_) => f.write_str("Held"),
785 Self::Contended => f.write_str("Contended"),
786 Self::OpenFailed(error) => f.debug_tuple("OpenFailed").field(error).finish(),
787 Self::HardeningRefused(error) => {
788 f.debug_tuple("HardeningRefused").field(error).finish()
789 }
790 }
791 }
792}
793
794#[cfg(unix)]
800fn acquire_events_daemon_guard_outcome(socket_path: &Path) -> EventsDaemonGuardAcquisition {
801 use EventsDaemonGuardAcquisition::{Contended, HardeningRefused, Held, OpenFailed};
802
803 let lock_path = socket_path.with_extension("lock");
804 if let Some(parent) = lock_path.parent() {
805 if let Err(error) = std::fs::create_dir_all(parent) {
806 return OpenFailed(error);
807 }
808 }
809 use std::os::unix::fs::OpenOptionsExt;
810 let file = match std::fs::OpenOptions::new()
814 .create(true)
815 .truncate(false)
816 .write(true)
817 .mode(0o600)
818 .custom_flags(libc::O_NOFOLLOW)
819 .open(&lock_path)
820 {
821 Ok(file) => file,
822 Err(error) if error.raw_os_error() == Some(libc::ELOOP) => {
823 return HardeningRefused(error);
824 }
825 Err(error) => return OpenFailed(error),
826 };
827 if let Err(error) = file.set_permissions(
833 <std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
834 ) {
835 return HardeningRefused(error);
836 }
837 use std::os::fd::AsRawFd;
838 let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
841 if rc == 0 {
842 Held(EventsDaemonGuard { _file: file })
843 } else {
844 let error = std::io::Error::last_os_error();
845 if error.raw_os_error() == Some(libc::EWOULDBLOCK)
846 || error.raw_os_error() == Some(libc::EAGAIN)
847 {
848 Contended
849 } else {
850 HardeningRefused(error)
851 }
852 }
853}
854
855fn events_db_targets(db_path: &Path) -> [PathBuf; 3] {
856 ["", "-wal", "-shm"].map(|suffix| {
857 let mut name = db_path.as_os_str().to_os_string();
858 name.push(suffix);
859 PathBuf::from(name)
860 })
861}
862
863fn refuse_events_db_symlinks(db_path: &Path) -> anyhow::Result<()> {
873 for path in events_db_targets(db_path) {
874 match std::fs::symlink_metadata(&path) {
875 Ok(meta) if meta.file_type().is_symlink() => anyhow::bail!(
876 "refusing to serve events: {} is a symlink; the events database and its \
877 sidecars must be regular files",
878 path.display()
879 ),
880 Ok(_) => {}
881 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
882 Err(e) => anyhow::bail!(
883 "refusing to serve events: cannot inspect {}: {e}",
884 path.display()
885 ),
886 }
887 }
888 Ok(())
889}
890
891#[cfg(unix)]
892#[derive(Clone, Copy, Debug, PartialEq, Eq)]
893struct EventsFileIdentity {
894 device: u64,
895 inode: u64,
896}
897
898#[cfg(unix)]
899impl EventsFileIdentity {
900 fn from_metadata(metadata: &std::fs::Metadata) -> Self {
901 use std::os::unix::fs::MetadataExt;
902 Self {
903 device: metadata.dev(),
904 inode: metadata.ino(),
905 }
906 }
907}
908
909#[cfg(unix)]
912type EventsDbIdentities = [Option<EventsFileIdentity>; 3];
913
914#[cfg(unix)]
923fn ensure_events_db_parent_trusted(db_path: &Path) -> anyhow::Result<()> {
924 let parent = absolutize(db_path);
925 let parent = parent.parent().filter(|p| !p.as_os_str().is_empty());
926 match parent {
927 Some(dir) => crate::daemon::ensure_socket_dir_is_trusted(dir).map_err(|e| {
928 anyhow::anyhow!("refusing to serve events from an untrusted directory: {e}")
929 }),
930 None => Ok(()),
931 }
932}
933
934#[cfg(unix)]
940fn ensure_events_db_owner_only(db_path: &Path) -> anyhow::Result<()> {
941 use std::os::unix::fs::OpenOptionsExt;
942 refuse_events_db_symlinks(db_path)?;
943 if let Some(parent) = db_path.parent() {
944 std::fs::create_dir_all(parent)?;
945 }
946 ensure_events_db_parent_trusted(db_path)?;
949 match std::fs::OpenOptions::new()
950 .write(true)
951 .create_new(true)
952 .mode(0o600)
953 .open(db_path)
954 {
955 Ok(_created) => Ok(()),
957 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()),
958 Err(e) => Err(anyhow::anyhow!(
959 "refusing to serve events: cannot create {} owner-only: {e}",
960 db_path.display()
961 )),
962 }
963}
964
965#[cfg(unix)]
978fn harden_events_db_sidecars(db_path: &Path) -> anyhow::Result<EventsDbIdentities> {
979 use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
980 let mut identities = [None; 3];
981 for (index, path) in events_db_targets(db_path).into_iter().enumerate() {
982 let file = match std::fs::OpenOptions::new()
989 .read(true)
990 .custom_flags(libc::O_NOFOLLOW)
991 .open(&path)
992 {
993 Ok(file) => file,
994 Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
995 Err(e) => {
996 anyhow::bail!(
997 "refusing to serve events: cannot open {} without following symlinks: \
998 {e}. The events database and its sidecars must be regular files.",
999 path.display()
1000 )
1001 }
1002 };
1003 let metadata = file.metadata()?;
1006 if !metadata.file_type().is_file() {
1007 anyhow::bail!(
1008 "refusing to serve events: {} is not a regular file. The events \
1009 database and its sidecars must be regular files.",
1010 path.display()
1011 );
1012 }
1013 file.set_permissions(std::fs::Permissions::from_mode(0o600))
1014 .map_err(|e| {
1015 anyhow::anyhow!(
1016 "refusing to serve events: cannot chmod 0600 {}: {e}. The events \
1017 database and its sidecars must be owner-only.",
1018 path.display()
1019 )
1020 })?;
1021 identities[index] = Some(EventsFileIdentity::from_metadata(&metadata));
1024 }
1025 Ok(identities)
1026}
1027
1028#[cfg(unix)]
1046fn verify_events_db_owner_only_unopened(
1047 db_path: &Path,
1048 before: &EventsDbIdentities,
1049) -> anyhow::Result<()> {
1050 use std::os::unix::fs::PermissionsExt;
1051 for (index, path) in events_db_targets(db_path).into_iter().enumerate() {
1052 let metadata = match std::fs::symlink_metadata(&path) {
1053 Ok(metadata) => metadata,
1054 Err(e) if e.kind() == std::io::ErrorKind::NotFound && index != 0 => continue,
1055 Err(e) if e.kind() == std::io::ErrorKind::NotFound => anyhow::bail!(
1056 "refusing to serve events: main database {} disappeared after SQLite open; \
1057 refusing to serve an unlinked database",
1058 path.display()
1059 ),
1060 Err(e) => {
1061 anyhow::bail!(
1062 "refusing to serve events: cannot stat {}: {e}",
1063 path.display()
1064 )
1065 }
1066 };
1067 if !metadata.file_type().is_file() {
1068 anyhow::bail!(
1069 "refusing to serve events: {} is not a regular file. The events database \
1070 and its sidecars must be regular files.",
1071 path.display()
1072 );
1073 }
1074 let mode = metadata.permissions().mode() & 0o777;
1075 if mode & 0o077 != 0 {
1076 anyhow::bail!(
1077 "refusing to serve events: {} is mode {mode:03o}, not owner-only. The events \
1078 database and its sidecars must be owner-only.",
1079 path.display()
1080 );
1081 }
1082 if before[index]
1083 .is_some_and(|identity| identity != EventsFileIdentity::from_metadata(&metadata))
1084 {
1085 anyhow::bail!(
1086 "refusing to serve events: {} changed identity between pre-open hardening \
1087 and post-open verification",
1088 path.display()
1089 );
1090 }
1091 }
1092 Ok(())
1093}
1094
1095#[cfg(unix)]
1121pub async fn supervise_events_daemon(db_path: PathBuf, socket_path: PathBuf) {
1122 match standalone_daemon_policies(&db_path) {
1123 Ok((wal_ceiling, disk_guard, volume_lock_dir)) => {
1124 supervise_events_daemon_with_policies(
1125 db_path,
1126 socket_path,
1127 wal_ceiling,
1128 disk_guard,
1129 volume_lock_dir,
1130 )
1131 .await;
1132 }
1133 Err(error) => {
1134 tracing::warn!(%error, "invalid events daemon policy; events supervisor not started");
1135 }
1136 }
1137}
1138
1139#[cfg(unix)]
1141pub async fn supervise_events_daemon_with_policies(
1142 db_path: PathBuf,
1143 socket_path: PathBuf,
1144 wal_ceiling: WalCeilingPolicy,
1145 disk_guard: khive_db::EffectiveDiskGuardConfig,
1146 volume_lock_dir: PathBuf,
1147) {
1148 if let Err(error) = disk_guard.validate() {
1149 tracing::warn!(%error, "invalid disk policy; events supervisor not started");
1150 return;
1151 }
1152 if let Err(error) = wal_ceiling.validate_static(true, true, false) {
1153 tracing::warn!(error = %error, "invalid WAL ceiling; events supervisor not started");
1154 return;
1155 }
1156 const PROBE_INTERVAL: Duration = Duration::from_secs(15);
1157 let shutdown = crate::daemon::daemon_shutdown_token();
1158 let mut child: Option<std::process::Child> = None;
1159 let mut respawns: u64 = 0;
1160 loop {
1161 if let Some(c) = child.as_mut() {
1165 match c.try_wait() {
1166 Ok(Some(status)) => {
1167 tracing::info!(%status, "events daemon child exited");
1168 child = None;
1169 }
1170 Ok(None) => {}
1171 Err(error) => {
1172 tracing::warn!(error = %error, "cannot poll events daemon child; dropping handle");
1173 child = None;
1174 }
1175 }
1176 }
1177
1178 let reachable = match connect_verified(&socket_path).await {
1179 Ok(_stream) => true,
1180 Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
1181 tracing::warn!(
1182 socket = %socket_path.display(),
1183 error = %error,
1184 "events socket answered by a foreign uid; treating as unreachable"
1185 );
1186 false
1187 }
1188 Err(_) => false,
1189 };
1190 if !reachable && child.is_none() {
1191 match std::env::current_exe() {
1192 Ok(exe) => {
1193 let spawned = events_daemon_command(
1194 &exe,
1195 &db_path,
1196 &socket_path,
1197 wal_ceiling,
1198 disk_guard,
1199 &volume_lock_dir,
1200 )
1201 .spawn();
1202 match spawned {
1203 Ok(spawned_child) => {
1204 respawns += 1;
1205 tracing::info!(
1206 pid = spawned_child.id(),
1207 respawns,
1208 socket = %socket_path.display(),
1209 "spawned events daemon"
1210 );
1211 child = Some(spawned_child);
1212 }
1213 Err(error) => {
1214 tracing::warn!(error = %error, "failed to spawn events daemon");
1215 }
1216 }
1217 }
1218 Err(error) => {
1219 tracing::warn!(error = %error, "cannot resolve current executable for events daemon spawn");
1220 }
1221 }
1222 }
1223 tokio::select! {
1224 _ = shutdown.cancelled() => break,
1225 _ = tokio::time::sleep(PROBE_INTERVAL) => {}
1226 }
1227 }
1228 if let Some(mut c) = child.take() {
1229 let _ = c.kill();
1233 let _ = c.wait();
1234 tracing::info!("events daemon child stopped with supervisor shutdown");
1235 }
1236}
1237
1238#[cfg(unix)]
1239fn events_daemon_command(
1240 executable: &Path,
1241 db_path: &Path,
1242 socket_path: &Path,
1243 wal_ceiling: WalCeilingPolicy,
1244 disk_guard: khive_db::EffectiveDiskGuardConfig,
1245 volume_lock_dir: &Path,
1246) -> std::process::Command {
1247 let source = match wal_ceiling.source {
1248 khive_db::WalCeilingSource::BackendField => "backend_field",
1249 khive_db::WalCeilingSource::Environment => "environment",
1250 khive_db::WalCeilingSource::Default => "default",
1251 };
1252 let mut command = std::process::Command::new(executable);
1253 command
1254 .arg("events-daemon")
1255 .arg("--db")
1256 .arg(db_path)
1257 .arg("--socket")
1258 .arg(socket_path)
1259 .arg("--wal-ceiling-bytes")
1260 .arg(wal_ceiling.bytes.to_string())
1261 .arg("--wal-ceiling-source")
1262 .arg(source)
1263 .arg("--disk-reserve-bytes")
1264 .arg(disk_guard.reserve_bytes.to_string())
1265 .arg("--disk-guard-deadline-ms")
1266 .arg(disk_guard.guard_deadline_ms.to_string())
1267 .arg("--disk-reserve-source")
1268 .arg(disk_guard.reserve_source.as_str())
1269 .arg("--disk-deadline-source")
1270 .arg(disk_guard.deadline_source.as_str())
1271 .arg("--disk-legacy-environment-present")
1272 .arg(disk_guard.legacy_environment_present.to_string())
1273 .arg("--volume-lock-dir")
1274 .arg(volume_lock_dir)
1275 .stdin(std::process::Stdio::null())
1276 .stdout(std::process::Stdio::null())
1277 .stderr(std::process::Stdio::null());
1278 command
1279}
1280
1281#[cfg(unix)]
1300pub async fn run_events_daemon(db_path: &Path, socket_path: &Path) -> anyhow::Result<()> {
1301 let (wal_ceiling, disk_guard, volume_lock_dir) = standalone_daemon_policies(db_path)?;
1302 run_events_daemon_with_policies(
1303 db_path,
1304 socket_path,
1305 wal_ceiling,
1306 disk_guard,
1307 volume_lock_dir,
1308 )
1309 .await
1310}
1311
1312#[cfg(unix)]
1314pub async fn run_events_daemon_with_policies(
1315 db_path: &Path,
1316 socket_path: &Path,
1317 wal_ceiling: WalCeilingPolicy,
1318 disk_guard: khive_db::EffectiveDiskGuardConfig,
1319 volume_lock_dir: PathBuf,
1320) -> anyhow::Result<()> {
1321 disk_guard.validate()?;
1322 wal_ceiling
1323 .validate_static(true, true, false)
1324 .map_err(crate::error::RuntimeError::from)?;
1325 let db_path = &absolutize(db_path);
1328 let socket_path = &absolutize(socket_path);
1329 validate_events_socket_path(socket_path)?;
1330 if let Some(parent) = socket_path.parent() {
1334 std::fs::create_dir_all(parent)?;
1335 crate::daemon::ensure_socket_dir_is_trusted(parent)?;
1336 }
1337 let _guard = match acquire_events_daemon_guard_outcome(socket_path) {
1338 EventsDaemonGuardAcquisition::Held(guard) => guard,
1339 refusal => {
1340 tracing::info!(
1341 socket = %socket_path.display(),
1342 reason = ?refusal,
1343 "events daemon lock unavailable; exiting"
1344 );
1345 return Ok(());
1346 }
1347 };
1348 ensure_events_db_owner_only(db_path)?;
1349 let before_open = harden_events_db_sidecars(db_path)?;
1362 let backend = Arc::new(
1363 StorageBackend::sqlite_with_max_readers_and_policies(
1364 db_path,
1365 None,
1366 wal_ceiling,
1367 disk_guard,
1368 volume_lock_dir,
1369 )
1370 .map_err(crate::error::RuntimeError::from)?,
1371 );
1372 backend.events()?;
1374 verify_events_db_owner_only_unopened(db_path, &before_open)?;
1375
1376 if socket_path.exists() {
1377 std::fs::remove_file(socket_path)?;
1378 }
1379 let listener = UnixListener::bind(socket_path)?;
1380 {
1381 use std::os::unix::fs::PermissionsExt;
1382 if let Err(e) =
1386 std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))
1387 {
1388 drop(listener);
1389 let _ = std::fs::remove_file(socket_path);
1390 anyhow::bail!(
1391 "refusing to serve events: cannot chmod 0600 {}: {e}. The events socket must \
1392 be owner-only.",
1393 socket_path.display()
1394 );
1395 }
1396 }
1397 let daemon_euid = unsafe { libc::geteuid() };
1398 let stores: NamespaceStores = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
1404 let connections = Arc::new(tokio::sync::Semaphore::new(MAX_EVENTS_CONNECTIONS));
1405 let frame_budget = Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_REQUEST_BYTES));
1406 tracing::info!(
1407 socket = %socket_path.display(),
1408 db = %db_path.display(),
1409 wal_ceiling_configured_bytes = wal_ceiling.bytes,
1410 wal_ceiling_effective_bytes = wal_ceiling.effective_bytes(backend.is_read_only()),
1411 wal_ceiling_source = ?wal_ceiling.source,
1412 "events daemon listening"
1413 );
1414
1415 loop {
1416 let (stream, _addr) = match listener.accept().await {
1417 Ok(pair) => pair,
1418 Err(error) => {
1419 tracing::warn!(error = %error, "events daemon accept failed");
1420 continue;
1421 }
1422 };
1423 match crate::daemon::peer_uid(&stream) {
1424 Ok(uid) if crate::daemon::uid_is_permitted(uid, daemon_euid) => {}
1425 Ok(uid) => {
1426 tracing::warn!(peer_uid = uid, "events daemon rejected foreign-uid peer");
1427 continue;
1428 }
1429 Err(error) => {
1430 tracing::warn!(error = %error, "events daemon could not read peer credentials");
1431 continue;
1432 }
1433 }
1434 let permit = match Arc::clone(&connections).try_acquire_owned() {
1435 Ok(permit) => permit,
1436 Err(_) => {
1437 tracing::warn!(
1440 cap = MAX_EVENTS_CONNECTIONS,
1441 "events daemon at connection cap; dropping new connection"
1442 );
1443 continue;
1444 }
1445 };
1446 let backend = Arc::clone(&backend);
1447 let stores = Arc::clone(&stores);
1448 let frame_budget = Arc::clone(&frame_budget);
1449 crate::daemon::spawn_named_tracked_task("events_connection", async move {
1450 serve_events_conn(stream, backend, stores, frame_budget).await;
1451 drop(permit);
1452 });
1453 }
1454}
1455
1456#[cfg(unix)]
1457#[path = "events_split_policy_entries.rs"]
1458mod policy_entries;
1459#[cfg(unix)]
1460use policy_entries::standalone_daemon_policies;
1461#[cfg(unix)]
1462pub use policy_entries::{
1463 run_events_daemon_with_wal_ceiling, supervise_events_daemon_with_wal_ceiling,
1464};
1465
1466#[cfg(all(test, unix))]
1467#[path = "events_wal_policy_tests.rs"]
1468mod wal_policy_tests;
1469
1470#[cfg(unix)]
1478async fn read_frame_budgeted(
1479 stream: &mut UnixStream,
1480 budget: &Arc<tokio::sync::Semaphore>,
1481) -> std::io::Result<(Vec<u8>, tokio::sync::OwnedSemaphorePermit)> {
1482 use tokio::io::AsyncReadExt;
1483 let mut len_buf = [0u8; 4];
1484 stream.read_exact(&mut len_buf).await?;
1485 let len = u32::from_be_bytes(len_buf) as usize;
1486 if len > crate::daemon::MAX_FRAME_BYTES {
1487 return Err(std::io::Error::new(
1488 std::io::ErrorKind::InvalidData,
1489 format!(
1490 "daemon frame of {len} bytes exceeds {} cap",
1491 crate::daemon::MAX_FRAME_BYTES
1492 ),
1493 ));
1494 }
1495 let permit = Arc::clone(budget)
1496 .acquire_many_owned(len as u32)
1497 .await
1498 .map_err(|_| std::io::Error::other("frame budget closed"))?;
1499 let mut buf = vec![0u8; len];
1500 stream.read_exact(&mut buf).await?;
1501 Ok((buf, permit))
1502}
1503
1504#[cfg(unix)]
1505type NamespaceStores =
1506 Arc<std::sync::Mutex<std::collections::HashMap<String, Arc<dyn EventStore>>>>;
1507
1508#[cfg(unix)]
1520fn namespace_store(
1521 backend: &StorageBackend,
1522 stores: &NamespaceStores,
1523 namespace: &str,
1524) -> Result<Arc<dyn EventStore>, khive_db::SqliteError> {
1525 namespace_store_with_cap(backend, stores, namespace, MAX_CACHED_NAMESPACE_STORES)
1526}
1527
1528#[cfg(unix)]
1529fn namespace_store_with_cap(
1530 backend: &StorageBackend,
1531 stores: &NamespaceStores,
1532 namespace: &str,
1533 cap: usize,
1534) -> Result<Arc<dyn EventStore>, khive_db::SqliteError> {
1535 let key = namespace.trim();
1536 if let Some(store) = stores
1537 .lock()
1538 .unwrap_or_else(std::sync::PoisonError::into_inner)
1539 .get(key)
1540 {
1541 return Ok(Arc::clone(store));
1542 }
1543 let store = backend.events_for_namespace(namespace)?;
1544 let mut map = stores
1545 .lock()
1546 .unwrap_or_else(std::sync::PoisonError::into_inner);
1547 if map.len() >= cap && !map.contains_key(key) {
1548 if let Some(victim) = map.keys().next().cloned() {
1556 map.remove(&victim);
1557 }
1558 }
1559 map.insert(key.to_string(), Arc::clone(&store));
1560 Ok(store)
1561}
1562
1563#[cfg(unix)]
1564async fn serve_events_conn(
1565 mut stream: UnixStream,
1566 backend: Arc<StorageBackend>,
1567 stores: NamespaceStores,
1568 frame_budget: Arc<tokio::sync::Semaphore>,
1569) {
1570 loop {
1571 let (payload, budget_permit) = match tokio::time::timeout(
1577 CONN_IO_TIMEOUT,
1578 read_frame_budgeted(&mut stream, &frame_budget),
1579 )
1580 .await
1581 {
1582 Ok(Ok(pair)) => pair,
1583 Ok(Err(_)) | Err(_) => return,
1585 };
1586 let response = match serde_json::from_slice::<EventsRequest>(&payload) {
1587 Ok(request) => dispatch_events_request(request, &backend, &stores).await,
1588 Err(error) => EventsResponse::Error {
1589 message: format!("events daemon could not parse request frame: {error}"),
1590 retryable: false,
1591 writer_task_failure: None,
1592 sqlite_write_failure: None,
1593 },
1594 };
1595 let bytes = match serde_json::to_vec(&response) {
1596 Ok(bytes) => bytes,
1597 Err(error) => {
1598 tracing::error!(error = %error, "events daemon response serialization failed");
1599 return;
1600 }
1601 };
1602 let bytes = if bytes.len() > crate::daemon::MAX_FRAME_BYTES {
1603 let refusal = EventsResponse::Error {
1607 message: format!(
1608 "response_frame_size_limit: {} response bytes exceed the {}-byte events IPC frame cap; request a narrower page",
1609 bytes.len(),
1610 crate::daemon::MAX_FRAME_BYTES
1611 ),
1612 retryable: false,
1613 writer_task_failure: None,
1614 sqlite_write_failure: None,
1615 };
1616 match serde_json::to_vec(&refusal) {
1617 Ok(bytes) => bytes,
1618 Err(error) => {
1619 tracing::error!(error = %error, "events daemon frame-size refusal serialization failed");
1620 return;
1621 }
1622 }
1623 } else {
1624 bytes
1625 };
1626 match tokio::time::timeout(CONN_IO_TIMEOUT, write_frame(&mut stream, &bytes)).await {
1629 Ok(Ok(())) => {}
1630 Ok(Err(_)) | Err(_) => return,
1631 }
1632 drop(budget_permit);
1636 }
1637}
1638
1639#[cfg(unix)]
1640async fn dispatch_events_request(
1641 request: EventsRequest,
1642 backend: &StorageBackend,
1643 stores: &NamespaceStores,
1644) -> EventsResponse {
1645 if request.protocol_version() != EVENTS_PROTOCOL_VERSION {
1646 return EventsResponse::Error {
1647 message: format!(
1648 "events protocol version mismatch: daemon speaks {}, client sent {}",
1649 EVENTS_PROTOCOL_VERSION,
1650 request.protocol_version()
1651 ),
1652 retryable: false,
1653 writer_task_failure: None,
1654 sqlite_write_failure: None,
1655 };
1656 }
1657 if let Err(error) = khive_types::Namespace::parse(request.namespace().trim()) {
1663 return EventsResponse::Error {
1664 message: format!("events request namespace rejected: {error}"),
1665 retryable: false,
1666 writer_task_failure: None,
1667 sqlite_write_failure: None,
1668 };
1669 }
1670 let store = match namespace_store(backend, stores, request.namespace()) {
1671 Ok(store) => store,
1672 Err(error) => {
1673 return EventsResponse::Error {
1674 message: format!("events store unavailable: {error}"),
1675 retryable: true,
1676 writer_task_failure: None,
1677 sqlite_write_failure: None,
1678 };
1679 }
1680 };
1681 match request {
1682 EventsRequest::AppendEvents { events, .. } => match store.append_events(events).await {
1683 Ok(summary) => EventsResponse::Appended { summary },
1684 Err(error) => storage_error_response(&error),
1685 },
1686 EventsRequest::AppendEventsIdempotent { events, .. } => {
1687 match store.append_events_idempotent(events).await {
1688 Ok(result) => EventsResponse::Idempotent { result },
1689 Err(error) => storage_error_response(&error),
1690 }
1691 }
1692 EventsRequest::GetEvent { id, .. } => match store.get_event(id).await {
1693 Ok(event) => EventsResponse::Event { event },
1694 Err(error) => storage_error_response(&error),
1695 },
1696 EventsRequest::QueryEvents { filter, page, .. } => {
1697 if page.limit > MAX_QUERY_EVENTS_PAGE_ROWS {
1698 return EventsResponse::Error {
1699 message: format!(
1700 "events query page limit {} exceeds the daemon cap of {} rows; \
1701 request narrower pages",
1702 page.limit, MAX_QUERY_EVENTS_PAGE_ROWS
1703 ),
1704 retryable: false,
1705 writer_task_failure: None,
1706 sqlite_write_failure: None,
1707 };
1708 }
1709 match store.query_events(filter, page).await {
1710 Ok(page) => EventsResponse::Pageful { page },
1711 Err(error) => storage_error_response(&error),
1712 }
1713 }
1714 EventsRequest::QueryEventPage { query, .. } => {
1715 if let Err(error) = crate::event_page::validate_page_query(&query) {
1716 return storage_error_response(&error);
1717 }
1718 match store.query_event_page(query).await {
1719 Ok(window) => EventsResponse::EventPageWindow { window },
1720 Err(error) => storage_error_response(&error),
1721 }
1722 }
1723 EventsRequest::CountEvents { filter, .. } => match store.count_events(filter).await {
1724 Ok(count) => EventsResponse::Count { count },
1725 Err(error) => storage_error_response(&error),
1726 },
1727 EventsRequest::CountEventsGrouped {
1728 filter, group_by, ..
1729 } => match store.count_events_grouped(filter, group_by).await {
1730 Ok(counts) => EventsResponse::GroupedCount { counts },
1731 Err(error) => storage_error_response(&error),
1732 },
1733 }
1734}
1735
1736#[cfg(unix)]
1737fn storage_error_response(error: &StorageError) -> EventsResponse {
1738 let (message, writer_task_failure) = match error.without_sqlite_write_stage() {
1739 StorageError::WriterTaskRequestFailed {
1740 request_state,
1741 source,
1742 } => (
1743 source.to_string(),
1744 Some(WireWriterTaskFailure::RequestFailed {
1745 request_state: (*request_state).into(),
1746 }),
1747 ),
1748 StorageError::WriterTaskTerminated {
1749 request_state,
1750 sqlite_full_codes,
1751 } => (
1752 error.to_string(),
1753 Some(WireWriterTaskFailure::TaskTerminated {
1754 request_state: (*request_state).into(),
1755 sqlite_full_codes: *sqlite_full_codes,
1756 }),
1757 ),
1758 _ => (error.to_string(), None),
1759 };
1760 EventsResponse::Error {
1761 message,
1762 retryable: error.is_retryable(),
1767 writer_task_failure,
1768 sqlite_write_failure: error.sqlite_write_failure(),
1769 }
1770}
1771
1772#[cfg(unix)]
1783async fn connect_verified(socket_path: &Path) -> std::io::Result<UnixStream> {
1784 let stream = UnixStream::connect(socket_path).await?;
1785 let own_euid = unsafe { libc::geteuid() } as u32;
1787 let peer = crate::daemon::peer_uid(&stream)?;
1788 if !crate::daemon::uid_is_permitted(peer, own_euid) {
1789 return Err(std::io::Error::new(
1790 std::io::ErrorKind::PermissionDenied,
1791 format!(
1792 "events socket {} answered by uid {peer}, not this process's uid {own_euid}; \
1793 refusing to exchange event frames with an unowned daemon",
1794 socket_path.display()
1795 ),
1796 ));
1797 }
1798 Ok(stream)
1799}
1800
1801#[cfg(unix)]
1805#[derive(Debug, Clone, Copy, Serialize)]
1806pub struct EventsForwardingMetrics {
1807 pub forwarded_batches: u64,
1808 pub forwarded_events: u64,
1809 pub dropped_batches: u64,
1810 pub dropped_events: u64,
1811 pub queued_bytes: usize,
1813}
1814
1815#[cfg(unix)]
1816#[derive(Debug, Default)]
1817struct ForwardingCounters {
1818 forwarded_batches: AtomicU64,
1819 forwarded_events: AtomicU64,
1820 dropped_batches: AtomicU64,
1821 dropped_events: AtomicU64,
1822 queued_bytes: AtomicUsize,
1823}
1824
1825#[cfg(unix)]
1828#[derive(Debug)]
1829struct AppendByteReservation {
1830 counters: Arc<ForwardingCounters>,
1831 bytes: usize,
1832}
1833
1834#[cfg(unix)]
1835impl Drop for AppendByteReservation {
1836 fn drop(&mut self) {
1837 self.counters
1838 .queued_bytes
1839 .fetch_sub(self.bytes, Ordering::AcqRel);
1840 }
1841}
1842
1843#[cfg(unix)]
1844#[derive(Debug)]
1845struct QueuedAppend {
1846 request: EventsRequest,
1847 count: u64,
1848 _reservation: AppendByteReservation,
1849}
1850
1851#[cfg(all(test, unix))]
1852impl QueuedAppend {
1853 fn unmetered(namespace: &str, events: Vec<Event>, counters: Arc<ForwardingCounters>) -> Self {
1854 let count = events.len() as u64;
1855 Self {
1856 request: EventsRequest::AppendEvents {
1857 protocol_version: EVENTS_PROTOCOL_VERSION,
1858 namespace: namespace.to_string(),
1859 events,
1860 },
1861 count,
1862 _reservation: AppendByteReservation { counters, bytes: 0 },
1863 }
1864 }
1865}
1866
1867#[cfg(unix)]
1870#[derive(Default)]
1871struct BoundedFrameCounter {
1872 bytes: usize,
1873 exceeded: bool,
1874}
1875
1876#[cfg(unix)]
1877impl std::io::Write for BoundedFrameCounter {
1878 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
1879 let next = self.bytes.saturating_add(buf.len());
1880 if next > crate::daemon::MAX_FRAME_BYTES {
1881 self.exceeded = true;
1882 return Err(std::io::Error::new(
1883 std::io::ErrorKind::InvalidData,
1884 "events append request exceeds IPC frame cap",
1885 ));
1886 }
1887 self.bytes = next;
1888 Ok(buf.len())
1889 }
1890
1891 fn flush(&mut self) -> std::io::Result<()> {
1892 Ok(())
1893 }
1894}
1895
1896#[cfg(unix)]
1897fn reserve_append_bytes(
1898 counters: &Arc<ForwardingCounters>,
1899 bytes: usize,
1900 budget: usize,
1901) -> Option<AppendByteReservation> {
1902 let mut used = counters.queued_bytes.load(Ordering::Acquire);
1903 loop {
1904 let next = used.checked_add(bytes)?;
1905 if next > budget {
1906 return None;
1907 }
1908 match counters.queued_bytes.compare_exchange_weak(
1909 used,
1910 next,
1911 Ordering::AcqRel,
1912 Ordering::Acquire,
1913 ) {
1914 Ok(_) => {
1915 return Some(AppendByteReservation {
1916 counters: Arc::clone(counters),
1917 bytes,
1918 });
1919 }
1920 Err(observed) => used = observed,
1921 }
1922 }
1923}
1924
1925#[cfg(unix)]
1933pub struct EventsSplitClient {
1934 socket_path: PathBuf,
1935 append_tx: tokio::sync::mpsc::Sender<QueuedAppend>,
1936 append_queue_byte_budget: usize,
1937 counters: Arc<ForwardingCounters>,
1938 outage_logged: Arc<AtomicBool>,
1941 preflight_store: Arc<dyn EventStore>,
1943}
1944
1945#[cfg(unix)]
1946impl std::fmt::Debug for EventsSplitClient {
1947 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1948 f.debug_struct("EventsSplitClient")
1949 .field("socket_path", &self.socket_path)
1950 .finish_non_exhaustive()
1951 }
1952}
1953
1954#[cfg(unix)]
1955impl EventsSplitClient {
1956 pub fn new(socket_path: PathBuf) -> crate::error::RuntimeResult<Arc<Self>> {
1958 Self::new_with_queue_depth(socket_path, DEFAULT_APPEND_QUEUE_BATCHES)
1959 }
1960
1961 pub fn new_with_queue_depth(
1964 socket_path: PathBuf,
1965 queue_depth: usize,
1966 ) -> crate::error::RuntimeResult<Arc<Self>> {
1967 Self::new_with_queue_depth_and_delivery_timeout(socket_path, queue_depth, REQUEST_TIMEOUT)
1968 }
1969
1970 fn new_with_queue_depth_and_delivery_timeout(
1974 socket_path: PathBuf,
1975 queue_depth: usize,
1976 delivery_timeout: Duration,
1977 ) -> crate::error::RuntimeResult<Arc<Self>> {
1978 Self::new_with_limits_and_delivery_timeout(
1979 socket_path,
1980 queue_depth,
1981 DEFAULT_APPEND_QUEUE_BYTES,
1982 delivery_timeout,
1983 )
1984 }
1985
1986 fn new_with_limits_and_delivery_timeout(
1987 socket_path: PathBuf,
1988 queue_depth: usize,
1989 byte_budget: usize,
1990 delivery_timeout: Duration,
1991 ) -> crate::error::RuntimeResult<Arc<Self>> {
1992 validate_events_socket_path(&socket_path)?;
1993 let preflight_backend = StorageBackend::memory()?;
1994 let preflight_store = preflight_backend.events()?;
1995 let (append_tx, append_rx) = tokio::sync::mpsc::channel::<QueuedAppend>(queue_depth.max(1));
1999 let counters = Arc::new(ForwardingCounters::default());
2000 let outage_logged = Arc::new(AtomicBool::new(false));
2001
2002 let client = Arc::new(Self {
2003 socket_path: socket_path.clone(),
2004 append_tx,
2005 append_queue_byte_budget: byte_budget,
2006 counters: Arc::clone(&counters),
2007 outage_logged: Arc::clone(&outage_logged),
2008 preflight_store,
2009 });
2010
2011 crate::daemon::spawn_named_tracked_task(
2012 "events_forwarder",
2013 run_forwarder(
2014 socket_path,
2015 append_rx,
2016 counters,
2017 outage_logged,
2018 delivery_timeout,
2019 crate::daemon::daemon_shutdown_token(),
2020 ),
2021 );
2022 Ok(client)
2023 }
2024
2025 pub fn metrics(&self) -> EventsForwardingMetrics {
2027 EventsForwardingMetrics {
2028 forwarded_batches: self.counters.forwarded_batches.load(Ordering::Relaxed),
2029 forwarded_events: self.counters.forwarded_events.load(Ordering::Relaxed),
2030 dropped_batches: self.counters.dropped_batches.load(Ordering::Relaxed),
2031 dropped_events: self.counters.dropped_events.load(Ordering::Relaxed),
2032 queued_bytes: self.counters.queued_bytes.load(Ordering::Acquire),
2033 }
2034 }
2035
2036 fn enqueue(&self, namespace: &str, events: Vec<Event>) {
2039 let count = events.len() as u64;
2040 let request = EventsRequest::AppendEvents {
2041 protocol_version: EVENTS_PROTOCOL_VERSION,
2042 namespace: namespace.to_string(),
2043 events,
2044 };
2045 let mut frame_size = BoundedFrameCounter::default();
2046 if let Err(error) = serde_json::to_writer(&mut frame_size, &request) {
2047 self.count_append_drop(
2048 count,
2049 if frame_size.exceeded {
2050 "frame cap"
2051 } else {
2052 "serialization"
2053 },
2054 );
2055 tracing::warn!(error = %error, "events append batch could not fit a wire frame");
2056 return;
2057 }
2058 let Some(reservation) = reserve_append_bytes(
2059 &self.counters,
2060 frame_size.bytes,
2061 self.append_queue_byte_budget,
2062 ) else {
2063 self.count_append_drop(count, "queue byte budget");
2064 return;
2065 };
2066 let batch = QueuedAppend {
2067 request,
2068 count,
2069 _reservation: reservation,
2070 };
2071 match self.append_tx.try_send(batch) {
2072 Ok(()) => {}
2073 Err(_) => {
2074 self.count_append_drop(count, "queue batch limit");
2076 }
2077 }
2078 }
2079
2080 fn count_append_drop(&self, count: u64, reason: &'static str) {
2081 self.counters
2082 .dropped_batches
2083 .fetch_add(1, Ordering::Relaxed);
2084 self.counters
2085 .dropped_events
2086 .fetch_add(count, Ordering::Relaxed);
2087 if !self.outage_logged.swap(true, Ordering::Relaxed) {
2088 tracing::warn!(
2089 dropped_events = count,
2090 reason,
2091 "events append queue rejected loss-tolerant batch"
2092 );
2093 }
2094 }
2095
2096 async fn round_trip(&self, request: &EventsRequest) -> StorageResult<EventsResponse> {
2101 let op = "events daemon round-trip";
2102 let payload = serde_json::to_vec(request).map_err(|error| StorageError::Serialization {
2103 capability: khive_storage::StorageCapability::Events,
2104 message: format!("events request serialization failed: {error}"),
2105 })?;
2106
2107 let attempt = async {
2108 let mut stream = connect_verified(&self.socket_path)
2109 .await
2110 .map_err(|error| ("connect", error))?;
2111 write_frame(&mut stream, &payload)
2112 .await
2113 .map_err(|error| ("write", error))?;
2114 read_frame(&mut stream)
2115 .await
2116 .map_err(|error| ("read", error))
2117 };
2118 let bytes = match tokio::time::timeout(REQUEST_TIMEOUT, attempt).await {
2119 Ok(Ok(bytes)) => bytes,
2120 Ok(Err(("read", error))) => {
2121 return Err(StorageError::Serialization {
2122 capability: khive_storage::StorageCapability::Events,
2123 message: format!(
2124 "events daemon closed or broke the response after connection at {}: {error}",
2125 self.socket_path.display()
2126 ),
2127 });
2128 }
2129 Ok(Err((stage, error))) => {
2130 return Err(StorageError::Pool {
2131 operation: op.into(),
2132 message: format!(
2133 "events daemon {stage} failed at {}: {error}",
2134 self.socket_path.display()
2135 ),
2136 });
2137 }
2138 Err(_elapsed) => {
2139 return Err(StorageError::Timeout {
2140 operation: op.into(),
2141 });
2142 }
2143 };
2144
2145 serde_json::from_slice::<EventsResponse>(&bytes).map_err(|error| {
2146 StorageError::Serialization {
2147 capability: khive_storage::StorageCapability::Events,
2148 message: format!("events response deserialization failed: {error}"),
2149 }
2150 })
2151 }
2152}
2153
2154#[cfg(unix)]
2161fn drain_dropped_queue(
2162 rx: &mut tokio::sync::mpsc::Receiver<QueuedAppend>,
2163 counters: &ForwardingCounters,
2164) {
2165 let mut dropped_batches = 0u64;
2166 let mut dropped_events = 0u64;
2167 while let Ok(batch) = rx.try_recv() {
2168 dropped_batches += 1;
2169 dropped_events += batch.count;
2170 }
2172 if dropped_batches > 0 {
2173 counters
2174 .dropped_batches
2175 .fetch_add(dropped_batches, Ordering::Relaxed);
2176 counters
2177 .dropped_events
2178 .fetch_add(dropped_events, Ordering::Relaxed);
2179 tracing::warn!(
2180 dropped_batches,
2181 dropped_events,
2182 "events forwarder shutting down; dropping queued loss-tolerant batches"
2183 );
2184 }
2185}
2186
2187#[cfg(unix)]
2192async fn run_forwarder(
2193 socket_path: PathBuf,
2194 mut rx: tokio::sync::mpsc::Receiver<QueuedAppend>,
2195 counters: Arc<ForwardingCounters>,
2196 outage_logged: Arc<AtomicBool>,
2197 delivery_timeout: Duration,
2198 shutdown: tokio_util::sync::CancellationToken,
2199) {
2200 let mut conn: Option<UnixStream> = None;
2208 loop {
2209 let batch = tokio::select! {
2210 _ = shutdown.cancelled() => {
2211 drain_dropped_queue(&mut rx, &counters);
2212 break;
2213 },
2214 received = rx.recv() => match received {
2215 Some(batch) => batch,
2216 None => break,
2217 },
2218 };
2219 let count = batch.count;
2220 let payload = match serde_json::to_vec(&batch.request) {
2221 Ok(payload) => payload,
2222 Err(error) => {
2223 counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
2224 counters.dropped_events.fetch_add(count, Ordering::Relaxed);
2225 tracing::error!(error = %error, "events forwarder serialization failed; batch dropped");
2226 continue;
2227 }
2228 };
2229
2230 let delivered = tokio::select! {
2238 _ = shutdown.cancelled() => {
2239 counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
2240 counters.dropped_events.fetch_add(count, Ordering::Relaxed);
2241 tracing::warn!(
2242 dropped_events = count,
2243 "events forwarder shutting down mid-delivery; in-flight batch dropped"
2244 );
2245 drain_dropped_queue(&mut rx, &counters);
2246 break;
2247 },
2248 outcome = tokio::time::timeout(
2249 delivery_timeout,
2250 deliver_batch(&socket_path, &mut conn, &payload),
2251 ) => match outcome {
2252 Ok(delivered) => delivered,
2253 Err(_elapsed) => {
2254 conn = None;
2255 false
2256 }
2257 },
2258 };
2259 if delivered {
2260 counters.forwarded_batches.fetch_add(1, Ordering::Relaxed);
2261 counters
2262 .forwarded_events
2263 .fetch_add(count, Ordering::Relaxed);
2264 if outage_logged.swap(false, Ordering::Relaxed) {
2265 tracing::info!("events daemon reachable again; forwarding resumed");
2266 }
2267 } else {
2268 counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
2269 counters.dropped_events.fetch_add(count, Ordering::Relaxed);
2270 if !outage_logged.swap(true, Ordering::Relaxed) {
2271 tracing::warn!(
2272 socket = %socket_path.display(),
2273 "events daemon unreachable; dropping loss-tolerant events until it returns"
2274 );
2275 }
2276 tokio::select! {
2277 _ = shutdown.cancelled() => {
2278 drain_dropped_queue(&mut rx, &counters);
2279 break;
2280 },
2281 _ = tokio::time::sleep(FORWARDER_BACKOFF) => {}
2282 }
2283 }
2284 }
2285}
2286
2287#[cfg(unix)]
2292async fn deliver_batch(socket_path: &Path, conn: &mut Option<UnixStream>, payload: &[u8]) -> bool {
2293 for _attempt in 0..2u8 {
2294 if conn.is_none() {
2295 match connect_verified(socket_path).await {
2296 Ok(stream) => *conn = Some(stream),
2297 Err(_) => return false,
2298 }
2299 }
2300 let stream = conn.as_mut().expect("connection populated above");
2301 let ok = async {
2302 write_frame(stream, payload).await?;
2303 let bytes = read_frame(stream).await?;
2304 std::io::Result::Ok(bytes)
2305 }
2306 .await;
2307 match ok {
2308 Ok(bytes) => {
2309 return !matches!(
2310 serde_json::from_slice::<EventsResponse>(&bytes),
2311 Ok(EventsResponse::Error { .. }) | Err(_)
2312 );
2313 }
2314 Err(_) => {
2315 *conn = None;
2318 }
2319 }
2320 }
2321 false
2322}
2323
2324#[cfg(unix)]
2333#[derive(Debug)]
2334pub struct ForwardingEventStore {
2335 namespace: String,
2336 client: Arc<EventsSplitClient>,
2337}
2338
2339#[cfg(unix)]
2340impl ForwardingEventStore {
2341 pub fn new(namespace: impl Into<String>, client: Arc<EventsSplitClient>) -> Self {
2342 Self {
2343 namespace: namespace.into(),
2344 client,
2345 }
2346 }
2347
2348 fn unexpected(&self, op: &'static str, response: EventsResponse) -> StorageError {
2349 match response {
2350 EventsResponse::Error {
2351 message,
2352 retryable,
2353 writer_task_failure,
2354 sqlite_write_failure,
2355 } => {
2356 let source = if retryable {
2359 StorageError::Pool {
2360 operation: op.into(),
2361 message,
2362 }
2363 } else {
2364 StorageError::InvalidInput {
2365 capability: khive_storage::StorageCapability::Events,
2366 operation: op.into(),
2367 message,
2368 }
2369 };
2370 let source = match sqlite_write_failure {
2371 Some(failure) => source.with_sqlite_write_failure(failure),
2372 None => source,
2373 };
2374 match writer_task_failure {
2375 Some(WireWriterTaskFailure::RequestFailed { request_state }) => {
2376 StorageError::WriterTaskRequestFailed {
2377 request_state: request_state.into(),
2378 source: Box::new(source),
2379 }
2380 }
2381 Some(WireWriterTaskFailure::TaskTerminated {
2382 request_state,
2383 sqlite_full_codes,
2384 }) => {
2385 let terminal = StorageError::WriterTaskTerminated {
2386 request_state: request_state.into(),
2387 sqlite_full_codes,
2388 };
2389 match sqlite_write_failure {
2390 Some(failure) => terminal.with_sqlite_write_failure(failure),
2391 None => terminal,
2392 }
2393 }
2394 None => source,
2395 }
2396 }
2397 other => StorageError::Serialization {
2398 capability: khive_storage::StorageCapability::Events,
2399 message: format!("events daemon returned mismatched response for {op}: {other:?}"),
2400 },
2401 }
2402 }
2403}
2404
2405#[cfg(unix)]
2406#[async_trait]
2407impl EventStore for ForwardingEventStore {
2408 async fn append_event(&self, event: Event) -> StorageResult<()> {
2409 self.client.enqueue(&self.namespace, vec![event]);
2410 Ok(())
2411 }
2412
2413 async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
2414 let attempted = events.len() as u64;
2415 self.client.enqueue(&self.namespace, events);
2416 Ok(BatchWriteSummary {
2419 attempted,
2420 affected: attempted,
2421 ..BatchWriteSummary::default()
2422 })
2423 }
2424
2425 async fn get_event(&self, id: Uuid) -> StorageResult<Option<Event>> {
2426 let request = EventsRequest::GetEvent {
2427 protocol_version: EVENTS_PROTOCOL_VERSION,
2428 namespace: self.namespace.clone(),
2429 id,
2430 };
2431 match self.client.round_trip(&request).await? {
2432 EventsResponse::Event { event } => Ok(event),
2433 other => Err(self.unexpected("get_event", other)),
2434 }
2435 }
2436
2437 async fn query_events(
2438 &self,
2439 filter: EventFilter,
2440 page: PageRequest,
2441 ) -> StorageResult<Page<Event>> {
2442 let request = EventsRequest::QueryEvents {
2443 protocol_version: EVENTS_PROTOCOL_VERSION,
2444 namespace: self.namespace.clone(),
2445 filter,
2446 page,
2447 };
2448 match self.client.round_trip(&request).await? {
2449 EventsResponse::Pageful { page } => Ok(page),
2450 other => Err(self.unexpected("query_events", other)),
2451 }
2452 }
2453
2454 async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
2455 let request = EventsRequest::CountEvents {
2456 protocol_version: EVENTS_PROTOCOL_VERSION,
2457 namespace: self.namespace.clone(),
2458 filter,
2459 };
2460 match self.client.round_trip(&request).await? {
2461 EventsResponse::Count { count } => Ok(count),
2462 other => Err(self.unexpected("count_events", other)),
2463 }
2464 }
2465
2466 async fn count_events_grouped(
2467 &self,
2468 filter: EventFilter,
2469 group_by: EventGroupBy,
2470 ) -> StorageResult<BTreeMap<String, u64>> {
2471 let request = EventsRequest::CountEventsGrouped {
2472 protocol_version: EVENTS_PROTOCOL_VERSION,
2473 namespace: self.namespace.clone(),
2474 filter,
2475 group_by,
2476 };
2477 match self.client.round_trip(&request).await? {
2478 EventsResponse::GroupedCount { counts } => Ok(counts),
2479 other => Err(self.unexpected("count_events_grouped", other)),
2480 }
2481 }
2482
2483 async fn query_event_page(&self, query: EventPageQuery) -> StorageResult<EventPageWindow> {
2484 crate::event_page::validate_page_query(&query)?;
2485 let request = EventsRequest::QueryEventPage {
2486 protocol_version: EVENTS_PROTOCOL_VERSION,
2487 namespace: self.namespace.clone(),
2488 query: query.clone(),
2489 };
2490 match self.client.round_trip(&request).await? {
2491 EventsResponse::EventPageWindow { window } => {
2492 crate::event_page::validate_window(&query, Some(&self.namespace), &window)?;
2493 Ok(window)
2494 }
2495 error @ EventsResponse::Error { .. } => Err(self.unexpected("query_event_page", error)),
2496 _ => Err(crate::event_page::page_error(
2497 "events page response kind mismatch",
2498 )),
2499 }
2500 }
2501
2502 fn preflight_event(&self, event: &Event) -> StorageResult<()> {
2503 self.client.preflight_store.preflight_event(event)
2505 }
2506
2507 async fn append_events_idempotent(
2508 &self,
2509 events: Vec<Event>,
2510 ) -> StorageResult<IdempotentEventBatchResult> {
2511 let request = EventsRequest::AppendEventsIdempotent {
2512 protocol_version: EVENTS_PROTOCOL_VERSION,
2513 namespace: self.namespace.clone(),
2514 events,
2515 };
2516 match self.client.round_trip(&request).await? {
2517 EventsResponse::Idempotent { result } => Ok(result),
2518 other => Err(self.unexpected("append_events_idempotent", other)),
2519 }
2520 }
2521
2522 fn supports_idempotent_audit_batch(&self) -> bool {
2523 true
2524 }
2525}
2526
2527pub struct SplitEventStore {
2544 legacy: Arc<dyn EventStore>,
2545 lane: Arc<dyn EventStore>,
2546}
2547
2548impl std::fmt::Debug for SplitEventStore {
2549 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2550 f.debug_struct("SplitEventStore").finish_non_exhaustive()
2551 }
2552}
2553
2554impl SplitEventStore {
2555 pub const MAX_MERGED_WINDOW_ROWS: u64 = 100_000;
2563
2564 pub fn new(legacy: Arc<dyn EventStore>, lane: Arc<dyn EventStore>) -> Self {
2565 Self { legacy, lane }
2566 }
2567}
2568
2569#[async_trait]
2570impl EventStore for SplitEventStore {
2571 async fn append_event(&self, event: Event) -> StorageResult<()> {
2572 self.legacy.append_event(event).await
2573 }
2574
2575 async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
2576 self.legacy.append_events(events).await
2577 }
2578
2579 async fn get_event(&self, id: Uuid) -> StorageResult<Option<Event>> {
2580 if let Some(event) = self.legacy.get_event(id).await? {
2583 return Ok(Some(event));
2584 }
2585 self.lane.get_event(id).await
2586 }
2587
2588 async fn query_events(
2589 &self,
2590 filter: EventFilter,
2591 page: PageRequest,
2592 ) -> StorageResult<Page<Event>> {
2593 let window = page.offset.saturating_add(u64::from(page.limit));
2607 if window > Self::MAX_MERGED_WINDOW_ROWS {
2608 return Err(StorageError::InvalidInput {
2609 capability: khive_storage::StorageCapability::Events,
2610 operation: "query_events".into(),
2611 message: format!(
2612 "offset+limit ({window}) exceeds the merged event plane's window bound \
2613 of {}; page deep windows with a `before` cursor at offset 0, or narrow \
2614 the filter",
2615 Self::MAX_MERGED_WINDOW_ROWS
2616 ),
2617 });
2618 }
2619 let prefix = PageRequest {
2620 offset: 0,
2621 limit: window.min(u64::from(u32::MAX)) as u32,
2622 };
2623 let legacy = self
2624 .legacy
2625 .query_events(filter.clone(), prefix.clone())
2626 .await?;
2627 let lane = self.lane.query_events(filter, prefix).await?;
2628 let total = match (legacy.total, lane.total) {
2629 (Some(a), Some(b)) => Some(a + b),
2630 _ => None,
2631 };
2632 let mut items = legacy.items;
2633 items.extend(lane.items);
2634 items.sort_by(|a, b| {
2635 b.created_at
2636 .cmp(&a.created_at)
2637 .then_with(|| b.id.cmp(&a.id))
2638 });
2639 let items = items
2640 .into_iter()
2641 .skip(page.offset as usize)
2642 .take(page.limit as usize)
2643 .collect();
2644 Ok(Page { items, total })
2645 }
2646
2647 async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
2648 let legacy = self.legacy.count_events(filter.clone()).await?;
2649 let lane = self.lane.count_events(filter).await?;
2650 Ok(legacy + lane)
2651 }
2652
2653 async fn count_events_grouped(
2654 &self,
2655 filter: EventFilter,
2656 group_by: EventGroupBy,
2657 ) -> StorageResult<BTreeMap<String, u64>> {
2658 let mut counts = self
2659 .legacy
2660 .count_events_grouped(filter.clone(), group_by)
2661 .await?;
2662 for (key, count) in self.lane.count_events_grouped(filter, group_by).await? {
2663 let total = counts.entry(key).or_default();
2664 *total = total
2665 .checked_add(count)
2666 .ok_or_else(|| StorageError::InvalidInput {
2667 capability: khive_storage::StorageCapability::Events,
2668 operation: "count_events_grouped".into(),
2669 message: "grouped event count overflow".into(),
2670 })?;
2671 }
2672 Ok(counts)
2673 }
2674
2675 async fn query_event_page(&self, query: EventPageQuery) -> StorageResult<EventPageWindow> {
2676 crate::event_page::split_page(self.legacy.as_ref(), self.lane.as_ref(), query).await
2677 }
2678
2679 fn preflight_event(&self, event: &Event) -> StorageResult<()> {
2680 self.lane.preflight_event(event)
2683 }
2684
2685 async fn append_events_idempotent(
2686 &self,
2687 events: Vec<Event>,
2688 ) -> StorageResult<IdempotentEventBatchResult> {
2689 if events.is_empty() {
2699 return self.lane.append_events_idempotent(events).await;
2700 }
2701 let ids: Vec<Uuid> = events.iter().map(|event| event.id).collect();
2702 let probe_limit = u32::try_from(ids.len()).map_err(|_| StorageError::InvalidInput {
2703 capability: khive_storage::StorageCapability::Events,
2704 operation: "append_events_idempotent".into(),
2705 message: format!(
2706 "batch of {} rows exceeds the legacy probe window",
2707 ids.len()
2708 ),
2709 })?;
2710 let existing = self
2711 .legacy
2712 .query_events(
2713 EventFilter {
2714 ids,
2715 ..EventFilter::default()
2716 },
2717 PageRequest {
2718 offset: 0,
2719 limit: probe_limit,
2720 },
2721 )
2722 .await?;
2723 let legacy_resident: std::collections::HashSet<Uuid> =
2724 existing.items.iter().map(|event| event.id).collect();
2725 if legacy_resident.is_empty() {
2726 return self.lane.append_events_idempotent(events).await;
2727 }
2728 let mut legacy_rows = Vec::new();
2729 let mut lane_rows = Vec::new();
2730 let mut routed_to_legacy = Vec::with_capacity(events.len());
2731 for event in events {
2732 if legacy_resident.contains(&event.id) {
2733 routed_to_legacy.push(true);
2734 legacy_rows.push(event);
2735 } else {
2736 routed_to_legacy.push(false);
2737 lane_rows.push(event);
2738 }
2739 }
2740 let legacy_expected = legacy_rows.len();
2741 let lane_expected = lane_rows.len();
2742 let legacy_result = self.legacy.append_events_idempotent(legacy_rows).await?;
2743 let lane_result = if lane_expected == 0 {
2744 IdempotentEventBatchResult { rows: Vec::new() }
2745 } else {
2746 self.lane.append_events_idempotent(lane_rows).await?
2747 };
2748 if legacy_result.rows.len() != legacy_expected || lane_result.rows.len() != lane_expected {
2749 return Err(StorageError::Driver {
2750 capability: khive_storage::StorageCapability::Events,
2751 operation: "append_events_idempotent".into(),
2752 source: format!(
2753 "idempotent sub-batch result length mismatch: legacy {}/{}, lane {}/{}",
2754 legacy_result.rows.len(),
2755 legacy_expected,
2756 lane_result.rows.len(),
2757 lane_expected
2758 )
2759 .into(),
2760 });
2761 }
2762 let mut legacy_iter = legacy_result.rows.into_iter();
2763 let mut lane_iter = lane_result.rows.into_iter();
2764 let rows = routed_to_legacy
2765 .into_iter()
2766 .map(|to_legacy| {
2767 if to_legacy {
2768 legacy_iter.next().expect("length checked above")
2769 } else {
2770 lane_iter.next().expect("length checked above")
2771 }
2772 })
2773 .collect();
2774 Ok(IdempotentEventBatchResult { rows })
2775 }
2776
2777 fn supports_idempotent_audit_batch(&self) -> bool {
2778 self.lane.supports_idempotent_audit_batch() && self.legacy.supports_idempotent_audit_batch()
2782 }
2783}
2784
2785#[cfg(test)]
2786#[path = "events_split_sqlite_full_tests.rs"]
2787mod sqlite_full_tests;
2788
2789#[cfg(test)]
2790mod tests {
2791 use super::*;
2792
2793 #[test]
2794 fn fixture_registry_cleanup_preserves_other_roots() {
2795 let live_dir = tempfile::tempdir().unwrap();
2796 let _live_guard = TestRegistryGuard::new(live_dir.path());
2797 let live_path = live_dir.path().join("events.db");
2798 let live_backend = direct_backend_for(&live_path).unwrap();
2799
2800 let finished_dir = tempfile::tempdir().unwrap();
2801 let finished_backend = {
2802 let _finished_guard = TestRegistryGuard::new(finished_dir.path());
2803 let backend = direct_backend_for(&finished_dir.path().join("events.db")).unwrap();
2804 let weak = Arc::downgrade(&backend);
2805 drop(backend);
2806 assert!(
2807 weak.upgrade().is_some(),
2808 "registry retains the live fixture"
2809 );
2810 weak
2811 };
2812
2813 assert!(
2814 finished_backend.upgrade().is_none(),
2815 "FD_FIXTURE_REGISTRY_RELEASED"
2816 );
2817 let same_live_backend = direct_backend_for(&live_path).unwrap();
2818 assert!(
2819 Arc::ptr_eq(&live_backend, &same_live_backend),
2820 "FD_FIXTURE_OTHER_ROOT_RETAINED"
2821 );
2822 }
2823
2824 #[cfg(unix)]
2825 mod target_filter_tests {
2826 include!("events_split_target_tests.rs");
2827 }
2828
2829 include!("events_sidecar_path_tests.rs");
2834
2835 #[cfg(unix)]
2836 #[test]
2837 fn deep_symlink_chains_within_the_kernel_bound_derive_the_target_sidecar() {
2838 let dir = tempfile::tempdir().unwrap();
2843 let real = dir.path().join("real.db");
2844 let mut prev = real.clone();
2845 for i in 0..35 {
2846 let link = dir.path().join(format!("hop{i}.db"));
2847 std::os::unix::fs::symlink(&prev, &link).unwrap();
2848 prev = link;
2849 }
2850 assert_eq!(
2851 events_db_path_beside(&prev),
2852 events_db_path_beside(&real),
2853 "a 35-hop dangling chain must derive the target's sidecar"
2854 );
2855 assert_ne!(
2857 events_db_path_beside(&dir.path().join("unrelated.db")),
2858 events_db_path_beside(&real)
2859 );
2860 }
2861
2862 include!("events_sidecar_hardening_tests.rs");
2863
2864 #[cfg(unix)]
2865 fn write_owner_only_test_file(path: &Path) {
2866 use std::os::unix::fs::PermissionsExt;
2867 std::fs::write(path, b"").unwrap();
2868 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).unwrap();
2869 }
2870
2871 #[cfg(unix)]
2872 #[test]
2873 fn unopened_check_refuses_same_mode_regular_file_replacement() {
2874 for target_index in 0..3 {
2878 let dir = tempfile::tempdir().unwrap();
2879 let db = dir.path().join("events.db");
2880 let targets = events_db_targets(&db);
2881 for path in &targets {
2882 write_owner_only_test_file(path);
2883 }
2884 let before_open = harden_events_db_sidecars(&db).unwrap();
2885 let replacement = dir.path().join("replacement");
2886 write_owner_only_test_file(&replacement);
2887 let replacement_id = EventsFileIdentity::from_metadata(
2888 &std::fs::symlink_metadata(&replacement).unwrap(),
2889 );
2890 assert_ne!(before_open[target_index], Some(replacement_id));
2891 let _connection = rusqlite::Connection::open(&db).unwrap();
2894 verify_events_db_owner_only_unopened(&db, &before_open).unwrap();
2895 std::fs::rename(&replacement, &targets[target_index]).unwrap();
2896 let error = verify_events_db_owner_only_unopened(&db, &before_open)
2897 .unwrap_err()
2898 .to_string();
2899 assert!(error.contains("changed identity"), "{error}");
2900 assert!(
2901 error.contains(&targets[target_index].display().to_string()),
2902 "{error}"
2903 );
2904 }
2905 }
2906
2907 #[cfg(unix)]
2908 #[test]
2909 fn unopened_check_accepts_sidecars_missing_at_check_time() {
2910 let dir = tempfile::tempdir().unwrap();
2911 let db = dir.path().join("events.db");
2912 let targets = events_db_targets(&db);
2913 for path in &targets {
2914 write_owner_only_test_file(path);
2915 }
2916 let before_open = harden_events_db_sidecars(&db).unwrap();
2917 let _connection = rusqlite::Connection::open(&db).unwrap();
2918 for path in targets.iter().skip(1) {
2919 std::fs::remove_file(path).unwrap();
2920 verify_events_db_owner_only_unopened(&db, &before_open)
2921 .expect("SQLite sidecars may disappear without losing the main database");
2922 }
2923 }
2924
2925 #[cfg(unix)]
2926 #[test]
2927 fn unopened_check_refuses_main_database_removed_after_sqlite_open() {
2928 let dir = tempfile::tempdir().unwrap();
2929 let db = dir.path().join("events.db");
2930 ensure_events_db_owner_only(&db).unwrap();
2931 let before_open = harden_events_db_sidecars(&db).unwrap();
2932 let backend = StorageBackend::sqlite_for_test(&db).unwrap();
2933 backend.events().unwrap();
2934 verify_events_db_owner_only_unopened(&db, &before_open).unwrap();
2935 std::fs::remove_file(&db).unwrap();
2936 let error = verify_events_db_owner_only_unopened(&db, &before_open)
2939 .unwrap_err()
2940 .to_string();
2941 assert!(
2942 error.contains("main database") && error.contains("disappeared"),
2943 "{error}"
2944 );
2945 }
2946
2947 #[cfg(unix)]
2948 #[test]
2949 fn unopened_check_accepts_fresh_sqlite_sidecars_and_unchanged_main() {
2950 let dir = tempfile::tempdir().unwrap();
2951 let db = dir.path().join("new.events.db");
2952 assert!(!db.exists());
2953 ensure_events_db_owner_only(&db).unwrap();
2954 let before_open = harden_events_db_sidecars(&db).unwrap();
2955 assert!(before_open[0].is_some());
2956 assert_eq!(&before_open[1..], &[None, None]);
2957 let backend = StorageBackend::sqlite_for_test(&db).unwrap();
2958 backend.events().unwrap();
2959 for path in events_db_targets(&db).iter().skip(1) {
2960 assert!(path.exists(), "SQLite creates {}", path.display());
2961 }
2962 verify_events_db_owner_only_unopened(&db, &before_open)
2963 .expect("new SQLite sidecars have no pre-open identity to contradict");
2964 }
2965
2966 #[test]
2967 fn socket_derives_beside_the_sidecar() {
2968 let db = events_db_path_beside(Path::new("/data/khive.db"));
2969 let socket = events_socket_path_beside(&db);
2970 assert!(socket.ends_with("khive.db.events.sock"), "got {socket:?}");
2971 }
2972
2973 #[test]
2974 fn bare_relative_main_db_yields_an_absolute_sidecar() {
2975 let path = events_db_path_beside(Path::new("khive.db"));
2979 assert!(path.is_absolute(), "got relative {path:?}");
2980 assert!(path.ends_with("khive.db.events.db"), "got {path:?}");
2981 }
2982
2983 #[cfg(unix)]
2984 #[tokio::test]
2985 async fn over_cap_query_page_limit_is_refused_before_materialization() {
2986 let backend = Arc::new(StorageBackend::memory().unwrap());
2991 let stores: NamespaceStores =
2992 Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
2993 let request = |limit: u32| EventsRequest::QueryEvents {
2994 protocol_version: EVENTS_PROTOCOL_VERSION,
2995 namespace: "local".to_string(),
2996 filter: EventFilter::default(),
2997 page: PageRequest { offset: 0, limit },
2998 };
2999 let refused =
3000 dispatch_events_request(request(MAX_QUERY_EVENTS_PAGE_ROWS + 1), &backend, &stores)
3001 .await;
3002 match refused {
3003 EventsResponse::Error {
3004 message,
3005 retryable,
3006 writer_task_failure,
3007 ..
3008 } => {
3009 assert!(
3010 message.contains(&MAX_QUERY_EVENTS_PAGE_ROWS.to_string()),
3011 "refusal must name the cap: {message}"
3012 );
3013 assert!(!retryable, "an over-cap page is not transient");
3014 assert!(writer_task_failure.is_none());
3015 }
3016 other => panic!("over-cap query must be refused, got {other:?}"),
3017 }
3018 let at_cap =
3021 dispatch_events_request(request(MAX_QUERY_EVENTS_PAGE_ROWS), &backend, &stores).await;
3022 assert!(
3023 matches!(at_cap, EventsResponse::Pageful { .. }),
3024 "at-cap query must reach the store, got {at_cap:?}"
3025 );
3026 }
3027
3028 #[cfg(unix)]
3029 #[test]
3030 fn lease_contention_crosses_events_wire_as_retryable() {
3031 assert_eq!(EVENTS_PROTOCOL_VERSION, 5);
3032 for (phase, expected_retryable) in [
3033 (khive_storage::CapacityUnavailablePhase::LockContended, true),
3034 (khive_storage::CapacityUnavailablePhase::Lock, false),
3035 ] {
3036 let response = storage_error_response(&StorageError::CapacityUnavailable {
3037 capability: khive_storage::StorageCapability::Sql,
3038 phase,
3039 message: "lease admission failed".into(),
3040 });
3041 let encoded = serde_json::to_vec(&response).expect("serialize server error");
3042 let decoded: EventsResponse =
3043 serde_json::from_slice(&encoded).expect("decode unchanged protocol");
3044 let EventsResponse::Error { retryable, .. } = decoded else {
3045 panic!("capacity failure must be an error frame");
3046 };
3047 assert_eq!(retryable, expected_retryable, "{phase:?}");
3048 }
3049 }
3050
3051 #[cfg(unix)]
3052 #[test]
3053 fn side_effects_unknown_state_crosses_the_wire_as_terminal_failure() {
3054 let response = storage_error_response(&StorageError::writer_task_terminated(
3055 khive_storage::WriterTaskRequestState::SideEffectsUnknown,
3056 ));
3057 assert!(
3058 matches!(
3059 response,
3060 EventsResponse::Error {
3061 writer_task_failure: Some(WireWriterTaskFailure::TaskTerminated {
3062 request_state: WireWriterTaskState::SideEffectsUnknown,
3063 sqlite_full_codes: None,
3064 }),
3065 ..
3066 }
3067 ),
3068 "an unknown-commit-state termination must carry its state on the wire"
3069 );
3070 let busy = storage_error_response(&StorageError::WriterTaskBusy { timeout_ms: 5 });
3072 assert!(matches!(
3073 busy,
3074 EventsResponse::Error {
3075 writer_task_failure: None,
3076 sqlite_write_failure: None,
3077 ..
3078 }
3079 ));
3080 }
3081
3082 #[cfg(unix)]
3083 #[test]
3084 fn proven_rollback_crosses_the_wire_without_claiming_task_termination() {
3085 let response = storage_error_response(&StorageError::WriterTaskRequestFailed {
3086 request_state: khive_storage::WriterTaskRequestState::TransactionRolledBack,
3087 source: Box::new(StorageError::Pool {
3088 operation: "writer_task_commit".into(),
3089 message: "commit refused".into(),
3090 }),
3091 });
3092 assert!(matches!(
3093 response,
3094 EventsResponse::Error {
3095 writer_task_failure: Some(WireWriterTaskFailure::RequestFailed {
3096 request_state: WireWriterTaskState::TransactionRolledBack,
3097 }),
3098 ..
3099 }
3100 ));
3101 }
3102
3103 #[test]
3104 fn error_frames_without_writer_failure_still_parse() {
3105 let bytes = br#"{"kind":"error","message":"boom","retryable":true}"#;
3109 let parsed: EventsResponse = serde_json::from_slice(bytes).expect("stateless frame parses");
3110 assert!(matches!(
3111 parsed,
3112 EventsResponse::Error {
3113 retryable: true,
3114 writer_task_failure: None,
3115 sqlite_write_failure: None,
3116 ..
3117 }
3118 ));
3119 }
3120
3121 fn split_retry_event(verb: &str) -> Event {
3122 Event::new(
3123 "test",
3124 verb,
3125 khive_types::EventKind::RecallExecuted,
3126 khive_types::SubstrateKind::Note,
3127 "agent:test",
3128 )
3129 }
3130
3131 fn store_pair(dir: &Path) -> (Arc<dyn EventStore>, Arc<dyn EventStore>) {
3132 let legacy = direct_backend_for(&dir.join("legacy.db"))
3133 .expect("legacy backend")
3134 .events_for_namespace("test")
3135 .expect("legacy store");
3136 let lane = direct_backend_for(&dir.join("lane.db"))
3137 .expect("lane backend")
3138 .events_for_namespace("test")
3139 .expect("lane store");
3140 (legacy, lane)
3141 }
3142
3143 #[tokio::test]
3148 async fn idempotent_retry_of_legacy_resident_rows_does_not_duplicate() {
3149 use khive_storage::event::EventAppendDisposition;
3150
3151 let dir = tempfile::tempdir().unwrap();
3152 let _registry_guard = TestRegistryGuard::new(dir.path());
3153 let (legacy, lane) = store_pair(dir.path());
3154
3155 let resident = split_retry_event("recall");
3156 legacy
3157 .append_events_idempotent(vec![resident.clone()])
3158 .await
3159 .expect("pre-cutover landing");
3160
3161 let split = SplitEventStore::new(Arc::clone(&legacy), Arc::clone(&lane));
3162 let fresh = split_retry_event("search");
3163 let result = split
3164 .append_events_idempotent(vec![resident.clone(), fresh.clone()])
3165 .await
3166 .expect("mixed retry batch");
3167 assert_eq!(
3168 result.rows,
3169 vec![
3170 EventAppendDisposition::AlreadyPresentIdentical,
3171 EventAppendDisposition::Inserted,
3172 ],
3173 "input order must be preserved across the two sub-batches"
3174 );
3175 assert_eq!(
3176 lane.count_events(EventFilter::default()).await.unwrap(),
3177 1,
3178 "the legacy-resident row must not reach the lane"
3179 );
3180 assert_eq!(
3181 split.count_events(EventFilter::default()).await.unwrap(),
3182 2,
3183 "merged count must not double-count the retried id"
3184 );
3185
3186 let mut mutated = resident.clone();
3189 mutated.verb = "other".to_string();
3190 let conflict = split
3191 .append_events_idempotent(vec![mutated])
3192 .await
3193 .expect("conflicting retry");
3194 assert_eq!(
3195 conflict.rows,
3196 vec![EventAppendDisposition::IdentityConflict]
3197 );
3198 assert_eq!(split.count_events(EventFilter::default()).await.unwrap(), 2);
3199 }
3200
3201 #[tokio::test]
3204 async fn merged_offset_window_is_bounded() {
3205 let dir = tempfile::tempdir().unwrap();
3206 let _registry_guard = TestRegistryGuard::new(dir.path());
3207 let (legacy, lane) = store_pair(dir.path());
3208 let split = SplitEventStore::new(legacy, lane);
3209
3210 let err = split
3211 .query_events(
3212 EventFilter::default(),
3213 PageRequest {
3214 offset: SplitEventStore::MAX_MERGED_WINDOW_ROWS,
3215 limit: 1,
3216 },
3217 )
3218 .await
3219 .expect_err("a window past the bound must be refused");
3220 assert!(
3221 matches!(err, StorageError::InvalidInput { .. }),
3222 "got {err:?}"
3223 );
3224 assert!(
3225 err.to_string().contains("before"),
3226 "the refusal must name the cursor remedy: {err}"
3227 );
3228
3229 let at_bound = split
3230 .query_events(
3231 EventFilter::default(),
3232 PageRequest {
3233 offset: SplitEventStore::MAX_MERGED_WINDOW_ROWS - 1,
3234 limit: 1,
3235 },
3236 )
3237 .await
3238 .expect("a window at the bound is admitted");
3239 assert!(at_bound.items.is_empty());
3240 }
3241
3242 #[cfg(unix)]
3243 #[tokio::test]
3244 async fn stateless_non_retryable_error_maps_terminal_not_writer_terminated() {
3245 let client = EventsSplitClient::new(std::path::PathBuf::from("/tmp/never-bound.sock"))
3252 .expect("client builds");
3253 let store = ForwardingEventStore::new("test", client);
3254 let bytes = br#"{"kind":"error","message":"refused","retryable":false}"#;
3255 let parsed: EventsResponse = serde_json::from_slice(bytes).expect("frame parses");
3256 assert!(matches!(
3257 store.unexpected("append", parsed),
3258 StorageError::InvalidInput { .. }
3259 ));
3260 let with_state = br#"{"kind":"error","message":"died","retryable":false,"writer_task_failure":{"kind":"task_terminated","request_state":"side_effects_unknown"}}"#;
3264 let parsed: EventsResponse =
3265 serde_json::from_slice(with_state).expect("stateful frame parses");
3266 assert!(matches!(
3267 store.unexpected("append", parsed),
3268 StorageError::WriterTaskTerminated { .. }
3269 ));
3270 }
3271
3272 #[cfg(unix)]
3273 #[tokio::test]
3274 async fn writer_task_states_cross_the_wire_verbatim() {
3275 use khive_storage::WriterTaskRequestState as S;
3281 let dir = tempfile::tempdir().unwrap();
3282 let client =
3283 EventsSplitClient::new(dir.path().join("never-bound.sock")).expect("client builds");
3284 let store = ForwardingEventStore::new("test", client);
3285 for state in [
3286 S::NotStarted,
3287 S::TransactionRolledBack,
3288 S::SideEffectsUnknown,
3289 ] {
3290 let response = storage_error_response(&StorageError::writer_task_terminated(state));
3291 let bytes = serde_json::to_vec(&response).unwrap();
3293 let parsed: EventsResponse = serde_json::from_slice(&bytes).unwrap();
3294 let err = store.unexpected("append_events_idempotent", parsed);
3295 assert!(
3296 matches!(
3297 err,
3298 StorageError::WriterTaskTerminated { request_state, .. } if request_state == state
3299 ),
3300 "state {state:?} did not survive the socket: got {err:?}"
3301 );
3302 }
3303 let busy = storage_error_response(&StorageError::WriterTaskBusy { timeout_ms: 5 });
3305 assert!(matches!(
3306 busy,
3307 EventsResponse::Error {
3308 writer_task_failure: None,
3309 sqlite_write_failure: None,
3310 ..
3311 }
3312 ));
3313 }
3314
3315 #[cfg(unix)]
3316 #[tokio::test]
3317 async fn native_write_stages_survive_direct_and_request_error_frames() {
3318 use khive_storage::error::{SqliteWriteFailure, SqliteWriteStage};
3319 use khive_storage::WriterTaskRequestState;
3320 let dir = tempfile::tempdir().unwrap();
3321 let client = EventsSplitClient::new(dir.path().join("never-bound.sock")).unwrap();
3322 let store = ForwardingEventStore::new("test", client);
3323 for (stage, extended) in [
3324 (SqliteWriteStage::Begin, rusqlite::ffi::SQLITE_BUSY_RECOVERY),
3325 (
3326 SqliteWriteStage::Statement,
3327 rusqlite::ffi::SQLITE_CONSTRAINT_UNIQUE,
3328 ),
3329 (SqliteWriteStage::Commit, rusqlite::ffi::SQLITE_IOERR_FSYNC),
3330 (SqliteWriteStage::Statement, rusqlite::ffi::SQLITE_FULL),
3331 ] {
3332 for request in [false, true] {
3333 for retryable in [false, true] {
3334 let failure = SqliteWriteFailure {
3335 stage,
3336 primary_code: extended & 0xff,
3337 extended_code: extended,
3338 settlement_unknown: false,
3339 };
3340 let source = if retryable {
3341 StorageError::Pool {
3342 operation: "append".into(),
3343 message: "no code in this message".into(),
3344 }
3345 } else {
3346 StorageError::InvalidInput {
3347 capability: khive_storage::StorageCapability::Events,
3348 operation: "append".into(),
3349 message: "no code in this message".into(),
3350 }
3351 }
3352 .with_sqlite_write_failure(failure);
3353 let source = if request {
3354 StorageError::WriterTaskRequestFailed {
3355 request_state: WriterTaskRequestState::TransactionRolledBack,
3356 source: Box::new(source),
3357 }
3358 } else {
3359 source
3360 };
3361 let bytes = serde_json::to_vec(&storage_error_response(&source)).unwrap();
3362 let error = store.unexpected("append", serde_json::from_slice(&bytes).unwrap());
3363 assert_eq!(error.sqlite_write_failure(), Some(failure));
3364 assert_eq!(error.is_retryable(), retryable);
3365 assert_eq!(
3366 matches!(
3367 &error,
3368 StorageError::WriterTaskRequestFailed {
3369 request_state: WriterTaskRequestState::TransactionRolledBack,
3370 ..
3371 }
3372 ),
3373 request
3374 );
3375 let wire =
3376 crate::runtime_error_value(error.into(), crate::DomainDisposition::Unknown);
3377 let expected = if extended == rusqlite::ffi::SQLITE_FULL {
3378 "sqlite_disk_full"
3379 } else {
3380 stage.as_str()
3381 };
3382 assert_eq!(wire["stage"], expected);
3383 assert_eq!(wire["sqlite_primary_code"], failure.primary_code);
3384 assert_eq!(wire["sqlite_extended_code"], extended);
3385 if request {
3386 assert_eq!(wire["task_terminated"], false);
3387 }
3388 }
3389 }
3390 }
3391 }
3392
3393 #[cfg(unix)]
3394 #[tokio::test]
3395 async fn terminal_native_evidence_keeps_unknown_outcome_precedence() {
3396 use khive_storage::error::{SqliteWriteFailure, SqliteWriteStage};
3397 use khive_storage::WriterTaskRequestState;
3398 let dir = tempfile::tempdir().unwrap();
3399 let client = EventsSplitClient::new(dir.path().join("never-bound.sock")).unwrap();
3400 let store = ForwardingEventStore::new("test", client);
3401 for terminal in [false, true] {
3402 for extended in [
3403 rusqlite::ffi::SQLITE_IOERR_FSYNC,
3404 rusqlite::ffi::SQLITE_FULL,
3405 ] {
3406 let failure = SqliteWriteFailure {
3407 stage: SqliteWriteStage::Commit,
3408 primary_code: extended & 0xff,
3409 extended_code: extended,
3410 settlement_unknown: false,
3411 };
3412 let source = if terminal {
3414 StorageError::WriterTaskTerminated {
3415 request_state: WriterTaskRequestState::SideEffectsUnknown,
3416 sqlite_full_codes: (extended == rusqlite::ffi::SQLITE_FULL)
3417 .then_some((extended, extended)),
3418 }
3419 } else {
3420 StorageError::WriterTaskRequestFailed {
3421 request_state: WriterTaskRequestState::SideEffectsUnknown,
3422 source: Box::new(StorageError::Internal("original cause".into())),
3423 }
3424 }
3425 .with_sqlite_write_failure(failure);
3426 let bytes = serde_json::to_vec(&storage_error_response(&source)).unwrap();
3427 let error = store.unexpected("append", serde_json::from_slice(&bytes).unwrap());
3428 assert_eq!(error.sqlite_write_failure(), Some(failure));
3429 if terminal {
3430 assert!(
3431 std::error::Error::source(error.without_sqlite_write_stage()).is_none()
3432 );
3433 }
3434 let wire =
3435 crate::runtime_error_value(error.into(), crate::DomainDisposition::Unknown);
3436 let expected = if extended == rusqlite::ffi::SQLITE_FULL {
3437 "sqlite_disk_full"
3438 } else if terminal {
3439 "writer_task_terminated"
3440 } else {
3441 "writer_task_request_failed"
3442 };
3443 assert_eq!(wire["code"], expected);
3444 assert_eq!(wire["stage"], expected);
3445 assert_eq!(wire["sqlite_write_stage"], "sqlite_commit_failure");
3446 assert_eq!(wire["sqlite_extended_code"], extended);
3447 assert_eq!(wire["request_state"], "side_effects_unknown");
3448 assert_eq!(wire["task_terminated"], terminal);
3449 assert_eq!(wire["retryable"], false);
3450 }
3451 }
3452 }
3453
3454 #[cfg(unix)]
3455 #[tokio::test]
3456 async fn poisoned_commit_evidence_does_not_claim_definite_failure() {
3457 use khive_storage::error::{SqliteWriteFailure, SqliteWriteStage};
3458 let dir = tempfile::tempdir().unwrap();
3459 let client = EventsSplitClient::new(dir.path().join("never-bound.sock")).unwrap();
3460 let store = ForwardingEventStore::new("test", client);
3461 let failure = SqliteWriteFailure {
3462 stage: SqliteWriteStage::Commit,
3463 primary_code: rusqlite::ffi::SQLITE_AUTH,
3464 extended_code: rusqlite::ffi::SQLITE_AUTH,
3465 settlement_unknown: true,
3466 };
3467 let original = StorageError::driver(
3468 khive_storage::StorageCapability::Sql,
3469 "execute_batch",
3470 rusqlite::Error::SqliteFailure(
3471 rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_AUTH),
3472 None,
3473 ),
3474 );
3475 let source = original.with_sqlite_write_failure(failure);
3476 let bytes = serde_json::to_vec(&storage_error_response(&source)).unwrap();
3477 let error = store.unexpected("append", serde_json::from_slice(&bytes).unwrap());
3478 assert_eq!(error.sqlite_write_failure(), Some(failure));
3479 assert!(!error.is_retryable());
3480 let wire = crate::runtime_error_value(error.into(), crate::DomainDisposition::Unknown);
3481 assert!(wire.get("code").is_none());
3482 assert!(wire.get("stage").is_none());
3483 assert_eq!(wire["sqlite_write_stage"], "sqlite_commit_failure");
3484 assert_eq!(wire["sqlite_primary_code"], rusqlite::ffi::SQLITE_AUTH);
3485 assert_eq!(wire["domain_disposition"], "unknown");
3486 }
3487
3488 #[cfg(unix)]
3489 #[tokio::test]
3490 async fn proven_rollback_round_trips_as_non_terminal_request_failure() {
3491 use khive_storage::WriterTaskRequestState as S;
3492
3493 let dir = tempfile::tempdir().unwrap();
3494 let client =
3495 EventsSplitClient::new(dir.path().join("never-bound.sock")).expect("client builds");
3496 let store = ForwardingEventStore::new("test", client);
3497 let response = storage_error_response(&StorageError::WriterTaskRequestFailed {
3498 request_state: S::TransactionRolledBack,
3499 source: Box::new(StorageError::Pool {
3500 operation: "writer_task_commit".into(),
3501 message: "commit refused".into(),
3502 }),
3503 });
3504 let bytes = serde_json::to_vec(&response).unwrap();
3505 let parsed: EventsResponse = serde_json::from_slice(&bytes).unwrap();
3506 let err = store.unexpected("append_events_idempotent", parsed);
3507 assert!(
3508 matches!(
3509 err,
3510 StorageError::WriterTaskRequestFailed {
3511 request_state: S::TransactionRolledBack,
3512 ..
3513 }
3514 ),
3515 "proven rollback must not reconstruct as a terminal writer: {err:?}"
3516 );
3517 }
3518
3519 #[cfg(unix)]
3520 #[test]
3521 fn direct_backend_hardens_preexisting_db_and_sidecars() {
3522 use std::os::unix::fs::PermissionsExt;
3523 let dir = tempfile::tempdir().unwrap();
3524 let _registry_guard = TestRegistryGuard::new(dir.path());
3525 let db = dir.path().join("pre-existing.events.db");
3526 let wal = dir.path().join("pre-existing.events.db-wal");
3527 std::fs::write(&db, b"").unwrap();
3528 std::fs::write(&wal, b"").unwrap();
3529 for path in [&db, &wal] {
3530 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o644)).unwrap();
3531 }
3532 assert_eq!(
3534 std::fs::metadata(&db).unwrap().permissions().mode() & 0o777,
3535 0o644
3536 );
3537 direct_backend_for(&db).expect("writable open succeeds");
3538 for path in [&db, &wal] {
3539 assert_eq!(
3540 std::fs::metadata(path).unwrap().permissions().mode() & 0o777,
3541 0o600,
3542 "pre-existing {} must be tightened to owner-only",
3543 path.display()
3544 );
3545 }
3546 }
3547
3548 #[cfg(unix)]
3549 #[test]
3550 fn namespace_store_cache_is_bounded_and_trim_normalized() {
3551 let backend = StorageBackend::memory().unwrap();
3552 let stores: NamespaceStores =
3553 Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
3554 namespace_store_with_cap(&backend, &stores, "alpha", 1).expect("first store");
3557 namespace_store_with_cap(&backend, &stores, "beta", 1).expect("second store evicts");
3558 {
3559 let map = stores.lock().unwrap();
3560 assert_eq!(map.len(), 1, "cache must not grow past its cap");
3561 assert!(
3562 map.contains_key("beta"),
3563 "the newest namespace must be admitted at the cap"
3564 );
3565 }
3566 namespace_store_with_cap(&backend, &stores, "beta", 1).expect("cached beta");
3569 namespace_store_with_cap(&backend, &stores, " beta ", 1).expect("trimmed spelling");
3570 {
3571 let map = stores.lock().unwrap();
3572 assert_eq!(
3573 map.len(),
3574 1,
3575 "spellings of one namespace must share one entry"
3576 );
3577 assert!(map.contains_key("beta"), "trimmed key is the cache key");
3578 }
3579 }
3580
3581 #[cfg(unix)]
3582 #[tokio::test]
3583 async fn frame_budget_admits_before_allocating_and_releases_after() {
3584 let (mut client, mut server) = UnixStream::pair().expect("socketpair");
3585 let budget = Arc::new(tokio::sync::Semaphore::new(1024));
3586 let payload = vec![7u8; 100];
3587 write_frame(&mut client, &payload).await.expect("write");
3588 let (bytes, permit) = read_frame_budgeted(&mut server, &budget)
3589 .await
3590 .expect("budgeted read");
3591 assert_eq!(bytes, payload);
3592 assert_eq!(budget.available_permits(), 1024 - 100);
3594 drop(permit);
3596 assert_eq!(budget.available_permits(), 1024);
3597 let mut oversized = Vec::from(((crate::daemon::MAX_FRAME_BYTES + 1) as u32).to_be_bytes());
3600 oversized.extend_from_slice(&[0u8; 8]);
3601 use tokio::io::AsyncWriteExt;
3602 client.write_all(&oversized).await.expect("raw prefix");
3603 let err = read_frame_budgeted(&mut server, &budget)
3604 .await
3605 .expect_err("oversized declaration must refuse");
3606 assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
3607 assert_eq!(
3608 budget.available_permits(),
3609 1024,
3610 "refusal must not consume budget"
3611 );
3612 }
3613
3614 #[cfg(unix)]
3615 #[tokio::test]
3616 async fn dispatch_rejects_invalid_wire_namespace() {
3617 let backend = Arc::new(StorageBackend::memory().unwrap());
3618 let stores: NamespaceStores =
3619 Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
3620 let huge = "n".repeat(64 * 1024);
3624 let response = dispatch_events_request(
3625 EventsRequest::CountEvents {
3626 protocol_version: EVENTS_PROTOCOL_VERSION,
3627 namespace: huge,
3628 filter: Default::default(),
3629 },
3630 &backend,
3631 &stores,
3632 )
3633 .await;
3634 assert!(
3635 matches!(
3636 &response,
3637 EventsResponse::Error {
3638 retryable: false,
3639 writer_task_failure: None,
3640 sqlite_write_failure: None,
3641 ..
3642 }
3643 ),
3644 "oversized namespace must be a typed refusal, got {response:?}"
3645 );
3646 assert_eq!(
3647 stores.lock().unwrap().len(),
3648 0,
3649 "a rejected namespace must never enter the cache"
3650 );
3651 let ok = dispatch_events_request(
3653 EventsRequest::CountEvents {
3654 protocol_version: EVENTS_PROTOCOL_VERSION,
3655 namespace: "local".to_string(),
3656 filter: Default::default(),
3657 },
3658 &backend,
3659 &stores,
3660 )
3661 .await;
3662 assert!(
3663 matches!(ok, EventsResponse::Count { .. }),
3664 "valid namespace must dispatch, got {ok:?}"
3665 );
3666 }
3667
3668 include!("events_split_shutdown_tests.rs");
3669
3670 #[cfg(unix)]
3671 #[tokio::test]
3672 async fn forwarder_abandons_hung_delivery() {
3673 let dir = tempfile::tempdir().unwrap();
3677 let socket = dir.path().join("hung.sock");
3678 let listener = tokio::net::UnixListener::bind(&socket).unwrap();
3679 let _server = tokio::spawn(async move {
3681 let mut held = Vec::new();
3682 loop {
3683 if let Ok((stream, _)) = listener.accept().await {
3684 held.push(stream);
3685 }
3686 }
3687 });
3688 let client = EventsSplitClient::new_with_queue_depth_and_delivery_timeout(
3689 socket.clone(),
3690 4,
3691 Duration::from_millis(100),
3692 )
3693 .expect("client builds");
3694 let event = Event::new(
3695 "test",
3696 "noop",
3697 khive_types::EventKind::Audit,
3698 khive_types::SubstrateKind::Event,
3699 "tester",
3700 );
3701 client.enqueue("test", vec![event]);
3702 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3703 loop {
3704 if client.metrics().dropped_batches >= 1 {
3705 break;
3706 }
3707 assert!(
3708 tokio::time::Instant::now() < deadline,
3709 "forwarder never abandoned the hung delivery: {:?}",
3710 client.metrics()
3711 );
3712 tokio::time::sleep(Duration::from_millis(20)).await;
3713 }
3714 }
3715
3716 #[cfg(unix)]
3717 #[test]
3718 fn wire_retryability_follows_the_storage_classifier() {
3719 let busy = storage_error_response(&StorageError::WriterTaskBusy { timeout_ms: 5 });
3720 assert!(
3721 matches!(
3722 busy,
3723 EventsResponse::Error {
3724 retryable: true,
3725 ..
3726 }
3727 ),
3728 "transient writer contention must stay retryable across the socket"
3729 );
3730 let terminated = storage_error_response(&StorageError::writer_task_terminated(
3733 khive_storage::WriterTaskRequestState::SideEffectsUnknown,
3734 ));
3735 assert!(
3736 matches!(
3737 terminated,
3738 EventsResponse::Error {
3739 retryable: false,
3740 ..
3741 }
3742 ),
3743 "a terminated writer must not be reported transient"
3744 );
3745 }
3746
3747 use khive_storage::event::EventAppendDisposition;
3748 use khive_types::{EventKind, SubstrateKind};
3749
3750 fn test_event(namespace: &str) -> Event {
3751 Event::new(
3752 namespace,
3753 "test.verb",
3754 EventKind::Audit,
3755 SubstrateKind::Event,
3756 "actor:test",
3757 )
3758 }
3759
3760 #[cfg(unix)]
3762 async fn boot_daemon(dir: &tempfile::TempDir) -> (PathBuf, PathBuf) {
3763 let db = dir.path().join("events.db");
3764 let socket = dir.path().join("events.sock");
3765 let (db_clone, socket_clone) = (db.clone(), socket.clone());
3766 tokio::spawn(async move {
3767 let _ = run_events_daemon(&db_clone, &socket_clone).await;
3768 });
3769 for _ in 0..100 {
3770 if UnixStream::connect(&socket).await.is_ok() {
3771 return (db, socket);
3772 }
3773 tokio::time::sleep(Duration::from_millis(20)).await;
3774 }
3775 panic!("events daemon did not come up on {}", socket.display());
3776 }
3777
3778 #[tokio::test]
3783 async fn split_store_routes_plain_to_legacy_idempotent_to_lane_and_merges_reads() {
3784 let dir = tempfile::tempdir().expect("tempdir");
3785 let _registry_guard = TestRegistryGuard::new(dir.path());
3786 let legacy_backend =
3787 direct_backend_for(&dir.path().join("legacy.db")).expect("legacy backend");
3788 let lane_backend = direct_backend_for(&dir.path().join("lane.db")).expect("lane backend");
3789 let legacy = legacy_backend
3790 .events_for_namespace("local")
3791 .expect("legacy store");
3792 let lane = lane_backend
3793 .events_for_namespace("local")
3794 .expect("lane store");
3795 let split = SplitEventStore::new(Arc::clone(&legacy), Arc::clone(&lane));
3796
3797 let plain = test_event("local");
3798 let plain_id = plain.id;
3799 split.append_event(plain).await.expect("plain append");
3800
3801 let audit = test_event("local");
3802 let audit_id = audit.id;
3803 let result = split
3804 .append_events_idempotent(vec![audit])
3805 .await
3806 .expect("idempotent append");
3807 assert_eq!(result.rows, vec![EventAppendDisposition::Inserted]);
3808
3809 assert!(legacy
3811 .get_event(plain_id)
3812 .await
3813 .expect("legacy get")
3814 .is_some());
3815 assert!(lane.get_event(plain_id).await.expect("lane get").is_none());
3816 assert!(legacy
3817 .get_event(audit_id)
3818 .await
3819 .expect("legacy get")
3820 .is_none());
3821 assert!(lane.get_event(audit_id).await.expect("lane get").is_some());
3822
3823 assert!(split
3825 .get_event(plain_id)
3826 .await
3827 .expect("split get")
3828 .is_some());
3829 assert!(split
3830 .get_event(audit_id)
3831 .await
3832 .expect("split get")
3833 .is_some());
3834 assert_eq!(
3835 split
3836 .count_events(EventFilter::default())
3837 .await
3838 .expect("split count"),
3839 2
3840 );
3841 let page = split
3842 .query_events(
3843 EventFilter::default(),
3844 PageRequest {
3845 offset: 0,
3846 limit: 10,
3847 },
3848 )
3849 .await
3850 .expect("split query");
3851 let ids: Vec<Uuid> = page.items.iter().map(|e| e.id).collect();
3852 assert!(ids.contains(&plain_id) && ids.contains(&audit_id));
3853
3854 let mut seen = Vec::new();
3857 for offset in 0..2 {
3858 let page = split
3859 .query_events(EventFilter::default(), PageRequest { offset, limit: 1 })
3860 .await
3861 .expect("windowed query");
3862 assert_eq!(page.items.len(), 1);
3863 seen.push(page.items[0].id);
3864 }
3865 seen.sort();
3866 let mut expected = vec![plain_id, audit_id];
3867 expected.sort();
3868 assert_eq!(seen, expected);
3869 }
3870
3871 #[tokio::test]
3884 async fn raw_sql_consumers_of_the_legacy_events_table_still_see_plain_appends() {
3885 use khive_storage::types::{SqlStatement, SqlValue};
3886
3887 let dir = tempfile::tempdir().expect("tempdir");
3888 let _registry_guard = TestRegistryGuard::new(dir.path());
3889 let legacy_backend =
3890 direct_backend_for(&dir.path().join("legacy-guard.db")).expect("legacy backend");
3891 let lane_backend =
3892 direct_backend_for(&dir.path().join("lane-guard.db")).expect("lane backend");
3893 let split = SplitEventStore::new(
3894 legacy_backend
3895 .events_for_namespace("local")
3896 .expect("legacy store"),
3897 lane_backend
3898 .events_for_namespace("local")
3899 .expect("lane store"),
3900 );
3901
3902 let provenance = test_event("local");
3905 let provenance_id = provenance.id;
3906 split.append_event(provenance).await.expect("plain append");
3907 let audit = test_event("local");
3908 let audit_id = audit.id;
3909 split
3910 .append_events_idempotent(vec![audit])
3911 .await
3912 .expect("idempotent append");
3913
3914 let count_by_raw_sql = |backend: Arc<StorageBackend>, id: Uuid| async move {
3915 let mut reader = backend.sql().reader().await.expect("sql reader");
3916 let rows = reader
3917 .query_all(SqlStatement {
3918 sql: "SELECT actor FROM events WHERE id = ?1".to_string(),
3919 params: vec![SqlValue::Text(id.to_string())],
3920 label: None,
3921 })
3922 .await
3923 .expect("raw events query");
3924 rows.len()
3925 };
3926
3927 assert_eq!(
3930 count_by_raw_sql(Arc::clone(&legacy_backend), provenance_id).await,
3931 1,
3932 "plain appends must stay visible to raw-SQL consumers of the legacy events table"
3933 );
3934 assert_eq!(
3937 count_by_raw_sql(Arc::clone(&legacy_backend), audit_id).await,
3938 0,
3939 "audit-lane rows must not land in the legacy events table"
3940 );
3941 assert_eq!(
3945 count_by_raw_sql(Arc::clone(&lane_backend), audit_id).await,
3946 1,
3947 "the raw query must prove it can find rows where they actually live"
3948 );
3949 }
3950
3951 #[cfg(unix)]
3954 #[tokio::test]
3955 async fn idempotent_append_round_trips_through_the_daemon() {
3956 let dir = tempfile::tempdir().expect("tempdir");
3957 let (_db, socket) = boot_daemon(&dir).await;
3958 let client = EventsSplitClient::new(socket).expect("client");
3959 let store = ForwardingEventStore::new("local", client);
3960
3961 let event = test_event("local");
3962 let id = event.id;
3963 let result = store
3964 .append_events_idempotent(vec![event.clone()])
3965 .await
3966 .expect("idempotent append over socket");
3967 assert_eq!(result.rows, vec![EventAppendDisposition::Inserted]);
3968
3969 let retry = store
3972 .append_events_idempotent(vec![event])
3973 .await
3974 .expect("idempotent retry");
3975 assert_eq!(
3976 retry.rows,
3977 vec![EventAppendDisposition::AlreadyPresentIdentical]
3978 );
3979
3980 let fetched = store.get_event(id).await.expect("get over socket");
3981 assert_eq!(fetched.map(|e| e.id), Some(id));
3982 let count = store
3983 .count_events(EventFilter::default())
3984 .await
3985 .expect("count over socket");
3986 assert_eq!(count, 1);
3987 }
3988
3989 #[cfg(unix)]
3992 #[tokio::test]
3993 async fn fire_and_forget_append_lands_in_the_daemon_store() {
3994 let dir = tempfile::tempdir().expect("tempdir");
3995 let (_db, socket) = boot_daemon(&dir).await;
3996 let client = EventsSplitClient::new(socket).expect("client");
3997 let store = ForwardingEventStore::new("local", Arc::clone(&client));
3998
3999 store
4000 .append_event(test_event("local"))
4001 .await
4002 .expect("append_event is fire-and-forget");
4003
4004 let (count, metrics) = tokio::time::timeout(Duration::from_secs(30), async {
4008 loop {
4009 let count = store
4010 .count_events(EventFilter::default())
4011 .await
4012 .expect("count over socket");
4013 let metrics = client.metrics();
4014 if (count == 1 && metrics.forwarded_events >= 1) || metrics.dropped_events > 0 {
4015 break (count, metrics);
4016 }
4017 tokio::time::sleep(Duration::from_millis(20)).await;
4018 }
4019 })
4020 .await
4021 .expect("forwarder must deliver or report a drop within 30 seconds");
4022 assert_eq!(
4023 metrics.dropped_events, 0,
4024 "forwarder dropped an event before it reached the daemon store"
4025 );
4026 assert_eq!(count, 1, "forwarded event must land in the daemon store");
4027 assert!(metrics.forwarded_events >= 1, "delivery must be counted");
4028 }
4029
4030 #[cfg(unix)]
4034 #[tokio::test]
4035 async fn queue_overflow_drops_and_counts_but_never_errors() {
4036 let dir = tempfile::tempdir().expect("tempdir");
4037 let dead_socket = dir.path().join("nobody-home.sock");
4038 let client = EventsSplitClient::new_with_queue_depth(dead_socket, 2).expect("client");
4039 let store = ForwardingEventStore::new("local", Arc::clone(&client));
4040
4041 for _ in 0..20 {
4042 store
4043 .append_event(test_event("local"))
4044 .await
4045 .expect("append_event must not error under overflow");
4046 }
4047 let metrics = client.metrics();
4048 assert!(
4049 metrics.dropped_events > 0,
4050 "overflow must register in the drop counter, got {metrics:?}"
4051 );
4052 }
4053
4054 #[cfg(unix)]
4058 #[tokio::test]
4059 async fn dead_socket_reads_fail_typed_and_preflight_stays_local() {
4060 let dir = tempfile::tempdir().expect("tempdir");
4061 let dead_socket = dir.path().join("nobody-home.sock");
4062 let client = EventsSplitClient::new(dead_socket).expect("client");
4063 let store = ForwardingEventStore::new("local", client);
4064
4065 let error = store
4066 .get_event(Uuid::new_v4())
4067 .await
4068 .expect_err("read against a dead socket must fail");
4069 assert!(
4070 matches!(error, StorageError::Pool { .. }),
4071 "expected the typed unreachable error, got {error:?}"
4072 );
4073
4074 assert!(store.supports_idempotent_audit_batch());
4077 store
4078 .preflight_event(&test_event("local"))
4079 .expect("offline preflight validates a well-formed event");
4080 }
4081
4082 #[cfg(unix)]
4085 #[tokio::test]
4086 async fn connected_events_peer_closing_without_response_is_not_unreachable() {
4087 let dir = tempfile::tempdir().unwrap();
4088 let socket = dir.path().join("closes-after-request.sock");
4089 let listener = UnixListener::bind(&socket).unwrap();
4090 let server = tokio::spawn(async move {
4091 let (mut stream, _) = listener.accept().await.unwrap();
4092 read_frame(&mut stream)
4093 .await
4094 .expect("complete request arrives");
4095 });
4097 let client = EventsSplitClient::new(socket).unwrap();
4098 let store = ForwardingEventStore::new("local", client);
4099 let error = store.get_event(Uuid::new_v4()).await.unwrap_err();
4100 assert!(
4101 matches!(error, StorageError::Serialization { .. }),
4102 "a post-connect response failure must not be Pool/unreachable: {error:?}"
4103 );
4104 server.await.unwrap();
4105 }
4106
4107 #[cfg(unix)]
4111 #[tokio::test]
4112 async fn query_page_over_frame_cap_returns_typed_size_refusal() {
4113 let dir = tempfile::tempdir().unwrap();
4114 let socket = dir.path().join("large-query.sock");
4115 let backend = Arc::new(StorageBackend::memory().unwrap());
4116 let event_store = backend.events_for_namespace("local").unwrap();
4117 let events: Vec<_> = (0..MAX_QUERY_EVENTS_PAGE_ROWS)
4118 .map(|_| {
4119 test_event("local").with_payload(serde_json::json!({"data": "x".repeat(3 * 1024)}))
4120 })
4121 .collect();
4122 event_store
4123 .append_events(events)
4124 .await
4125 .expect("seed large page");
4126
4127 let listener = UnixListener::bind(&socket).unwrap();
4128 let stores: NamespaceStores =
4129 Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
4130 let server = tokio::spawn(async move {
4131 let (stream, _) = listener.accept().await.unwrap();
4132 serve_events_conn(
4133 stream,
4134 backend,
4135 stores,
4136 Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_REQUEST_BYTES)),
4137 )
4138 .await;
4139 });
4140 let client = EventsSplitClient::new(socket).unwrap();
4141 let store = ForwardingEventStore::new("local", client);
4142 let error = store
4143 .query_events(
4144 EventFilter::default(),
4145 PageRequest {
4146 offset: 0,
4147 limit: MAX_QUERY_EVENTS_PAGE_ROWS,
4148 },
4149 )
4150 .await
4151 .expect_err("the full page cannot fit one frame");
4152 assert!(
4153 matches!(error, StorageError::InvalidInput { .. }),
4154 "expected non-retryable frame-size refusal, got {error:?}"
4155 );
4156 assert!(
4157 error.to_string().contains("response_frame_size_limit"),
4158 "{error}"
4159 );
4160 assert!(
4161 error.to_string().contains("request a narrower page"),
4162 "{error}"
4163 );
4164 server.abort();
4165 }
4166
4167 #[cfg(unix)]
4171 #[tokio::test]
4172 async fn forwarding_queue_enforces_serialized_byte_and_frame_limits_with_drop_metrics() {
4173 let dir = tempfile::tempdir().unwrap();
4174 let socket = dir.path().join("stalled-forwarder.sock");
4175 let listener = UnixListener::bind(&socket).unwrap();
4176 let server = tokio::spawn(async move {
4177 let (stream, _) = listener.accept().await.unwrap();
4178 let _hold_open = stream;
4179 std::future::pending::<()>().await;
4180 });
4181 let event = test_event("local").with_payload(serde_json::json!({"data": "x".repeat(4096)}));
4182 let first = vec![event.clone(), event.clone()];
4183 let request_bytes = serde_json::to_vec(&EventsRequest::AppendEvents {
4184 protocol_version: EVENTS_PROTOCOL_VERSION,
4185 namespace: "local".into(),
4186 events: first.clone(),
4187 })
4188 .unwrap()
4189 .len();
4190 let byte_budget = request_bytes + 64;
4191 let client = EventsSplitClient::new_with_limits_and_delivery_timeout(
4192 socket,
4193 8,
4194 byte_budget,
4195 Duration::from_secs(30),
4196 )
4197 .unwrap();
4198 let store = ForwardingEventStore::new("local", Arc::clone(&client));
4199 store.append_events(first.clone()).await.unwrap();
4200 assert_eq!(client.metrics().queued_bytes, request_bytes);
4201
4202 store.append_events(first).await.unwrap();
4203 let metrics = client.metrics();
4204 assert_eq!(metrics.dropped_batches, 1);
4205 assert_eq!(metrics.dropped_events, 2);
4206 assert_eq!(metrics.queued_bytes, request_bytes);
4207 assert!(metrics.queued_bytes <= byte_budget);
4208
4209 let oversized = test_event("local")
4212 .with_payload(serde_json::json!({"data": "x".repeat(crate::daemon::MAX_FRAME_BYTES)}));
4213 store.append_event(oversized).await.unwrap();
4214 let metrics = client.metrics();
4215 assert_eq!(metrics.dropped_batches, 2);
4216 assert_eq!(metrics.dropped_events, 3);
4217 assert_eq!(metrics.queued_bytes, request_bytes);
4218 server.abort();
4219 }
4220
4221 #[cfg(unix)]
4224 #[tokio::test]
4225 async fn protocol_version_skew_is_a_typed_refusal() {
4226 let dir = tempfile::tempdir().expect("tempdir");
4227 let (_db, socket) = boot_daemon(&dir).await;
4228
4229 let mut stream = UnixStream::connect(&socket).await.expect("connect");
4230 let request = EventsRequest::CountEvents {
4231 protocol_version: EVENTS_PROTOCOL_VERSION + 1,
4232 namespace: "local".into(),
4233 filter: EventFilter::default(),
4234 };
4235 let payload = serde_json::to_vec(&request).expect("serialize");
4236 write_frame(&mut stream, &payload).await.expect("write");
4237 let bytes = read_frame(&mut stream).await.expect("read");
4238 let response: EventsResponse = serde_json::from_slice(&bytes).expect("parse");
4239 match response {
4240 EventsResponse::Error {
4241 message, retryable, ..
4242 } => {
4243 assert!(!retryable, "version skew is not retryable");
4244 assert!(message.contains("protocol version"), "message: {message}");
4245 }
4246 other => panic!("expected a typed refusal, got {other:?}"),
4247 }
4248 }
4249
4250 #[cfg(unix)]
4254 #[test]
4255 fn events_daemon_guard_is_exclusive_then_reusable() {
4256 let dir = tempfile::tempdir().expect("tempdir");
4257 let socket = dir.path().join("events.sock");
4258 let first = match acquire_events_daemon_guard_outcome(&socket) {
4259 EventsDaemonGuardAcquisition::Held(guard) => guard,
4260 other => panic!("first acquire must succeed: {other:?}"),
4261 };
4262 let second = acquire_events_daemon_guard_outcome(&socket);
4263 assert!(
4264 matches!(second, EventsDaemonGuardAcquisition::Contended),
4265 "second acquire must report contention while the first guard is held: {second:?}"
4266 );
4267 drop(first);
4268 const MAX_ATTEMPTS: usize = 100;
4272 for attempt in 1..=MAX_ATTEMPTS {
4273 match acquire_events_daemon_guard_outcome(&socket) {
4274 EventsDaemonGuardAcquisition::Held(_) => return,
4275 EventsDaemonGuardAcquisition::Contended if attempt < MAX_ATTEMPTS => {
4276 std::thread::sleep(Duration::from_millis(10));
4277 }
4278 other => panic!(
4279 "acquire must succeed after release within {MAX_ATTEMPTS} attempts; \
4280 attempt {attempt}: {other:?}"
4281 ),
4282 }
4283 }
4284 unreachable!("the final acquisition attempt returns or reports its refusal");
4285 }
4286
4287 #[cfg(unix)]
4288 #[test]
4289 fn events_daemon_guard_open_failure_is_not_contention() {
4290 let dir = tempfile::tempdir().expect("tempdir");
4291 let clean = dir.path().join("clean.sock");
4292 assert!(matches!(
4293 acquire_events_daemon_guard_outcome(&clean),
4294 EventsDaemonGuardAcquisition::Held(_)
4295 ));
4296
4297 assert!(
4298 try_acquire_events_daemon_guard(&clean).is_some(),
4299 "the public Option entrance must preserve a successful acquisition"
4300 );
4301
4302 let socket = dir.path().join("blocked.sock");
4305 let lock_path = socket.with_extension("lock");
4306 std::fs::create_dir(&lock_path).expect("create directory at lock entry");
4307 let outcome = acquire_events_daemon_guard_outcome(&socket);
4308 assert!(
4309 matches!(outcome, EventsDaemonGuardAcquisition::OpenFailed(_)),
4310 "an actual lock-file open failure must not report contention: {outcome:?}"
4311 );
4312 assert!(
4313 try_acquire_events_daemon_guard(&socket).is_none(),
4314 "the public Option entrance must refuse an actual open failure"
4315 );
4316 assert!(
4317 lock_path.is_dir(),
4318 "the refused entry must remain a directory"
4319 );
4320 }
4321
4322 #[cfg(unix)]
4329 #[tokio::test]
4330 async fn read_only_runtime_never_creates_an_events_db() {
4331 use crate::{KhiveRuntime, Namespace, RuntimeConfig};
4332
4333 let dir = tempfile::tempdir().expect("tempdir");
4334 let _registry_guard = TestRegistryGuard::new(dir.path());
4335 let main_db = dir.path().join("main.db");
4336 drop(
4338 KhiveRuntime::new_for_test(RuntimeConfig {
4339 db_path: Some(main_db.clone()),
4340 ..RuntimeConfig::no_embeddings()
4341 })
4342 .expect("create main db"),
4343 );
4344 let events_db = events_db_path_beside(&main_db);
4345 assert!(!events_db.exists(), "precondition: no events db yet");
4346
4347 khive_storage::test_support::freeze_snapshot_sidecars(&main_db);
4350
4351 let split_config = |db: PathBuf| RuntimeConfig {
4352 db_path: Some(main_db.clone()),
4353 events_split: Some(EventsSplitConfig {
4354 db_path: db,
4355 socket_path: None,
4356 }),
4357 ..RuntimeConfig::no_embeddings()
4358 };
4359
4360 let ro = KhiveRuntime::new_readonly_for_test(split_config(events_db.clone()))
4363 .expect("read-only runtime");
4364 let token = ro.authorize(Namespace::local()).expect("token");
4365 let store = ro.events(&token).expect("events store");
4366 let count = store
4367 .count_events(EventFilter::default())
4368 .await
4369 .expect("count through legacy-only plane");
4370 assert_eq!(count, 0);
4371 assert!(
4372 !events_db.exists(),
4373 "a read-only runtime must not mint the events database"
4374 );
4375
4376 {
4383 let lane_backend = StorageBackend::sqlite_for_test(&events_db).expect("writable lane");
4384 lane_backend
4385 .events_for_namespace("local")
4386 .expect("lane store")
4387 .append_event(test_event("local"))
4388 .await
4389 .expect("seed lane row");
4390 }
4391 khive_storage::test_support::freeze_snapshot_sidecars(&events_db);
4392 let store = ro.events(&token).expect("events store with lane present");
4393 let count = store
4394 .count_events(EventFilter::default())
4395 .await
4396 .expect("merged count");
4397 assert_eq!(count, 1, "the pre-existing lane row must merge into reads");
4398 }
4399
4400 #[tokio::test]
4403 async fn direct_mode_appends_and_reads_without_a_daemon() {
4404 let dir = tempfile::tempdir().expect("tempdir");
4405 let _registry_guard = TestRegistryGuard::new(dir.path());
4406 let db = dir.path().join("events.db");
4407 let backend = direct_backend_for(&db).expect("direct backend");
4408 let store = backend.events_for_namespace("local").expect("store");
4409 store
4410 .append_event(test_event("local"))
4411 .await
4412 .expect("direct append");
4413 let count = store
4414 .count_events(EventFilter::default())
4415 .await
4416 .expect("count");
4417 assert_eq!(count, 1);
4418 }
4419}