1use std::collections::BTreeMap;
41use std::path::{Path, PathBuf};
42#[cfg(unix)]
43use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
44use std::sync::Arc;
45#[cfg(unix)]
46use std::time::Duration;
47
48use async_trait::async_trait;
49use khive_db::{StorageBackend, WalCeilingPolicy};
50use khive_storage::event::{
51 EventGroupBy, EventPageQuery, EventPageWindow, IdempotentEventBatchResult,
52};
53use khive_storage::{
54 BatchWriteSummary, Event, EventFilter, EventStore, Page, PageRequest, StorageError,
55 StorageResult,
56};
57use serde::{Deserialize, Serialize};
58
59#[path = "events_count_read_budget.rs"]
60mod events_count_read_budget;
61use events_count_read_budget::CountReadBudget;
62#[doc(hidden)]
63pub use events_count_read_budget::WireReadFailure;
64#[cfg(unix)]
65use events_count_read_budget::{count_storage_error_response, restore_count_read_failure};
66
67#[cfg(unix)]
68use tokio::net::{UnixListener, UnixStream};
69use uuid::Uuid;
70
71#[cfg(unix)]
72use crate::daemon::{read_frame, write_frame};
73
74pub const EVENTS_PROTOCOL_VERSION: u32 = 6;
92
93#[cfg(unix)]
97pub const DEFAULT_APPEND_QUEUE_BATCHES: usize = 4096;
98
99#[cfg(unix)]
103pub const DEFAULT_APPEND_QUEUE_BYTES: usize = 32 * 1024 * 1024;
104
105#[cfg(unix)]
110const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
111
112#[cfg(unix)]
114const FORWARDER_BACKOFF: Duration = Duration::from_secs(2);
115
116#[cfg(unix)]
123const MAX_EVENTS_CONNECTIONS: usize = 128;
124
125#[cfg(unix)]
133const CONN_IO_TIMEOUT: Duration = Duration::from_secs(60);
134
135#[cfg(unix)]
145const MAX_INFLIGHT_REQUEST_BYTES: usize = 64 * 1024 * 1024;
146
147#[cfg(unix)]
152const MAX_CACHED_NAMESPACE_STORES: usize = 1024;
153
154const EVENTS_SYMLINK_HOP_BOUND: u32 = 40;
159
160pub const MAX_QUERY_EVENTS_PAGE_ROWS: u32 = 4096;
178
179pub fn events_db_path_beside(main_db: &Path) -> PathBuf {
194 let resolved = std::fs::canonicalize(main_db)
201 .unwrap_or_else(|_| resolve_dangling_final_component(main_db));
202 let mut name = resolved
203 .file_name()
204 .map(std::ffi::OsStr::to_os_string)
205 .unwrap_or_else(|| std::ffi::OsString::from("khive.db"));
206 name.push(".events.db");
207 let path = match resolved.parent().filter(|dir| !dir.as_os_str().is_empty()) {
208 Some(dir) => std::fs::canonicalize(dir)
209 .unwrap_or_else(|_| dir.to_path_buf())
210 .join(&name),
211 None => PathBuf::from(name),
212 };
213 absolutize(&path)
214}
215
216fn resolve_dangling_final_component(path: &Path) -> PathBuf {
223 let mut current = path.to_path_buf();
224 for _ in 0..EVENTS_SYMLINK_HOP_BOUND {
229 match std::fs::read_link(¤t) {
230 Ok(target) => {
231 current = if target.is_absolute() {
232 target
233 } else {
234 match current.parent().filter(|dir| !dir.as_os_str().is_empty()) {
235 Some(dir) => dir.join(target),
236 None => target,
237 }
238 };
239 }
240 Err(_) => break,
241 }
242 }
243 current
244}
245
246fn absolutize(path: &Path) -> PathBuf {
252 if path.is_absolute() {
253 return path.to_path_buf();
254 }
255 std::env::current_dir()
256 .map(|cwd| cwd.join(path))
257 .unwrap_or_else(|_| path.to_path_buf())
258}
259
260pub fn events_socket_path_beside(events_db: &Path) -> PathBuf {
270 events_db.with_extension("sock")
271}
272
273#[cfg(unix)]
274#[path = "events_split_socket_path.rs"]
275pub(crate) mod socket_path;
276#[cfg(unix)]
277pub use socket_path::validate_events_socket_path;
278
279#[derive(Debug, Clone)]
281pub struct EventsSplitConfig {
282 pub db_path: PathBuf,
285 pub socket_path: Option<PathBuf>,
289}
290
291#[cfg(unix)]
301type ClientMap = std::collections::HashMap<PathBuf, Arc<EventsSplitClient>>;
302type BackendMap = std::collections::HashMap<PathBuf, (bool, Arc<StorageBackend>)>;
306
307#[cfg(unix)]
308fn client_registry() -> &'static std::sync::Mutex<ClientMap> {
309 static REGISTRY: std::sync::OnceLock<std::sync::Mutex<ClientMap>> = std::sync::OnceLock::new();
310 REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
311}
312
313fn direct_backend_registry() -> &'static std::sync::Mutex<BackendMap> {
314 static REGISTRY: std::sync::OnceLock<std::sync::Mutex<BackendMap>> = std::sync::OnceLock::new();
315 REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
316}
317
318#[cfg(any(test, feature = "test-internals"))]
325#[doc(hidden)]
326pub struct TestRegistryGuard<'a> {
327 root: &'a Path,
328 canonical_root: PathBuf,
329}
330
331#[cfg(any(test, feature = "test-internals"))]
332impl<'a> TestRegistryGuard<'a> {
333 pub fn new(root: &'a Path) -> Self {
335 Self {
336 root,
337 canonical_root: std::fs::canonicalize(root).expect("existing unique fixture root"),
338 }
339 }
340
341 fn remove_entries<T>(
342 &self,
343 registry: &std::sync::Mutex<std::collections::HashMap<PathBuf, T>>,
344 ) -> Vec<T> {
345 let mut entries = registry
346 .lock()
347 .unwrap_or_else(std::sync::PoisonError::into_inner);
348 let keys: Vec<_> = entries
349 .keys()
350 .filter(|path| path.starts_with(self.root) || path.starts_with(&self.canonical_root))
351 .cloned()
352 .collect();
353 keys.into_iter()
354 .filter_map(|key| entries.remove(&key))
355 .collect()
356 }
357}
358
359#[cfg(any(test, feature = "test-internals"))]
360impl Drop for TestRegistryGuard<'_> {
361 fn drop(&mut self) {
362 let backends = self.remove_entries(direct_backend_registry());
365 #[cfg(unix)]
366 let clients = self.remove_entries(client_registry());
367 drop(backends);
368 #[cfg(unix)]
369 drop(clients);
370 }
371}
372
373#[cfg(unix)]
376pub fn client_for(socket_path: &Path) -> crate::error::RuntimeResult<Arc<EventsSplitClient>> {
377 let mut registry = client_registry()
378 .lock()
379 .unwrap_or_else(std::sync::PoisonError::into_inner);
380 if let Some(existing) = registry.get(socket_path) {
381 return Ok(Arc::clone(existing));
382 }
383 let client = EventsSplitClient::new(socket_path.to_path_buf())?;
384 registry.insert(socket_path.to_path_buf(), Arc::clone(&client));
385 Ok(client)
386}
387
388pub fn direct_backend_for(db_path: &Path) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
391 direct_backend_with_max_readers(db_path, false, None)
392}
393
394pub fn direct_backend_read_only_for(
399 db_path: &Path,
400) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
401 direct_backend_with_max_readers(db_path, true, None)
402}
403
404pub(crate) fn direct_backend_with_max_readers(
406 db_path: &Path,
407 read_only: bool,
408 max_readers: Option<usize>,
409) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
410 let mut config = crate::RuntimeConfig {
411 db_path: Some(db_path.to_path_buf()),
412 ..crate::RuntimeConfig::no_embeddings()
413 };
414 let wal_ceiling = config.resolve_wal_ceiling_policy(read_only)?;
415 let disk_guard = config.resolve_disk_guard_policy(read_only)?;
416 direct_backend_with_policies(
417 db_path,
418 read_only,
419 max_readers,
420 wal_ceiling,
421 disk_guard,
422 config.volume_lock_dir,
423 )
424}
425
426pub(crate) fn direct_backend_with_policies(
430 db_path: &Path,
431 read_only: bool,
432 max_readers: Option<usize>,
433 wal_ceiling: WalCeilingPolicy,
434 disk_guard: Option<khive_db::EffectiveDiskGuardConfig>,
435 volume_lock_dir: Option<PathBuf>,
436) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
437 wal_ceiling.validate_static(true, true, read_only)?;
438 if !read_only {
439 disk_guard
440 .ok_or_else(|| {
441 crate::error::RuntimeError::Internal("missing events disk policy".into())
442 })?
443 .validate()?;
444 khive_db::require_volume_lock_dir(volume_lock_dir.clone())?;
445 }
446 let mut registry = direct_backend_registry()
447 .lock()
448 .unwrap_or_else(std::sync::PoisonError::into_inner);
449 let absolute_path = absolutize(db_path);
450 let db_path = absolute_path.as_path();
451 refuse_events_db_symlinks(db_path)
452 .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
453 #[cfg(unix)]
454 if !read_only {
455 if let Some(parent) = db_path.parent().filter(|p| !p.as_os_str().is_empty()) {
456 let _ = std::fs::create_dir_all(parent);
457 }
458 }
459 #[cfg(unix)]
465 ensure_events_db_parent_trusted(db_path)
466 .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
467 let key = std::fs::canonicalize(db_path)
468 .or_else(|error| {
469 if error.kind() != std::io::ErrorKind::NotFound {
470 return Err(error);
471 }
472 let parent = db_path.parent().ok_or(error)?;
473 let name = db_path.file_name().ok_or_else(|| {
474 std::io::Error::new(
475 std::io::ErrorKind::InvalidInput,
476 "events path has no file name",
477 )
478 })?;
479 std::fs::canonicalize(parent).map(|parent| parent.join(name))
480 })
481 .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
482 if let Some((existing_read_only, existing)) = registry.get(&key) {
483 if *existing_read_only != read_only {
484 let existing_mode = if *existing_read_only {
485 "read-only"
486 } else {
487 "writable"
488 };
489 let requested_mode = if read_only { "read-only" } else { "writable" };
490 return Err(crate::error::RuntimeError::InvalidInput(format!(
491 "events database {} is already open {existing_mode} in this process; cannot \
492 open it {requested_mode}; read-only events access requires a separate frozen snapshot",
493 key.display()
494 )));
495 }
496 let numbers = |p: Option<khive_db::EffectiveDiskGuardConfig>| {
497 p.map(|p| (p.reserve_bytes, p.guard_deadline_ms))
498 };
499 if numbers(existing.pool().effective_disk_guard_config()) != numbers(disk_guard) {
500 return Err(crate::error::RuntimeError::Internal(
501 "events database is already open with a different disk reserve/deadline policy"
502 .into(),
503 ));
504 }
505 if !read_only && existing.pool().config().volume_lock_dir != volume_lock_dir {
506 return Err(crate::error::RuntimeError::Internal(
507 "events database is already open with a different volume-lock directory".into(),
508 ));
509 }
510 let existing_bytes = existing.pool().config().wal_ceiling.bytes;
511 if existing_bytes != wal_ceiling.bytes {
512 return Err(khive_db::SqliteError::InvalidConfig(format!(
513 "events database {} is already open with wal_ceiling_bytes={existing_bytes}; \
514 requested {}; drain and restart before changing the WAL ceiling",
515 key.display(),
516 wal_ceiling.bytes
517 ))
518 .into());
519 }
520 return Ok(Arc::clone(existing));
521 }
522 let db_path = key.as_path();
523 #[cfg(unix)]
529 if !read_only {
530 use std::os::unix::fs::OpenOptionsExt;
531 let _ = std::fs::OpenOptions::new()
532 .write(true)
533 .create_new(true)
534 .mode(0o600)
535 .open(db_path);
536 let _before_open = harden_events_db_sidecars(db_path)
539 .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
540 }
541 let backend = Arc::new(if read_only {
542 StorageBackend::sqlite_read_only_with_max_readers_and_wal_ceiling(
543 db_path,
544 max_readers,
545 wal_ceiling,
546 )?
547 } else {
548 StorageBackend::sqlite_with_max_readers_and_policies(
549 db_path,
550 max_readers,
551 wal_ceiling,
552 disk_guard.ok_or_else(|| {
553 crate::error::RuntimeError::Internal("missing events disk policy".into())
554 })?,
555 khive_db::require_volume_lock_dir(volume_lock_dir)?,
556 )?
557 });
558 registry.insert(key, (read_only, Arc::clone(&backend)));
559 Ok(backend)
560}
561
562#[cfg(unix)]
565pub fn forwarding_metrics(socket_path: &Path) -> Option<EventsForwardingMetrics> {
566 let registry = client_registry()
567 .lock()
568 .unwrap_or_else(std::sync::PoisonError::into_inner);
569 registry.get(socket_path).map(|client| client.metrics())
570}
571
572#[derive(Debug, Serialize, Deserialize)]
578#[serde(tag = "op", rename_all = "snake_case")]
579pub enum EventsRequest {
580 AppendEvents {
584 protocol_version: u32,
585 namespace: String,
586 events: Vec<Event>,
587 },
588 AppendEventsIdempotent {
590 protocol_version: u32,
591 namespace: String,
592 events: Vec<Event>,
593 },
594 GetEvent {
595 protocol_version: u32,
596 namespace: String,
597 id: Uuid,
598 },
599 QueryEvents {
600 protocol_version: u32,
601 namespace: String,
602 filter: EventFilter,
603 page: PageRequest,
604 },
605 QueryEventPage {
606 protocol_version: u32,
607 namespace: String,
608 query: EventPageQuery,
609 },
610 CountEvents {
611 protocol_version: u32,
612 namespace: String,
613 #[serde(default)]
614 operator_read_budget_secs: Option<u64>,
615 filter: EventFilter,
616 },
617 CountEventsGrouped {
618 protocol_version: u32,
619 namespace: String,
620 #[serde(default)]
621 operator_read_budget_secs: Option<u64>,
622 filter: EventFilter,
623 group_by: EventGroupBy,
624 },
625}
626
627impl EventsRequest {
628 fn protocol_version(&self) -> u32 {
629 match self {
630 Self::AppendEvents {
631 protocol_version, ..
632 }
633 | Self::AppendEventsIdempotent {
634 protocol_version, ..
635 }
636 | Self::GetEvent {
637 protocol_version, ..
638 }
639 | Self::QueryEvents {
640 protocol_version, ..
641 }
642 | Self::QueryEventPage {
643 protocol_version, ..
644 }
645 | Self::CountEvents {
646 protocol_version, ..
647 }
648 | Self::CountEventsGrouped {
649 protocol_version, ..
650 } => *protocol_version,
651 }
652 }
653
654 fn count_budget_secs(&self) -> Option<Option<u64>> {
655 match self {
656 Self::CountEvents {
657 operator_read_budget_secs,
658 ..
659 }
660 | Self::CountEventsGrouped {
661 operator_read_budget_secs,
662 ..
663 } => Some(*operator_read_budget_secs),
664 _ => None,
665 }
666 }
667
668 fn namespace(&self) -> &str {
669 match self {
670 Self::AppendEvents { namespace, .. }
671 | Self::AppendEventsIdempotent { namespace, .. }
672 | Self::GetEvent { namespace, .. }
673 | Self::QueryEvents { namespace, .. }
674 | Self::QueryEventPage { namespace, .. }
675 | Self::CountEvents { namespace, .. }
676 | Self::CountEventsGrouped { namespace, .. } => namespace,
677 }
678 }
679}
680
681#[derive(Debug, Serialize, Deserialize)]
683#[serde(tag = "kind", rename_all = "snake_case")]
684pub enum EventsResponse {
685 Appended {
686 summary: BatchWriteSummary,
687 },
688 Idempotent {
689 result: IdempotentEventBatchResult,
690 },
691 Event {
692 event: Option<Event>,
693 },
694 Pageful {
695 page: Page<Event>,
696 },
697 EventPageWindow {
698 window: EventPageWindow,
699 },
700 Count {
701 count: u64,
702 },
703 GroupedCount {
704 counts: BTreeMap<String, u64>,
705 },
706 Error {
716 message: String,
717 retryable: bool,
718 #[serde(default)]
719 writer_task_failure: Option<WireWriterTaskFailure>,
720 #[serde(default, skip_serializing_if = "Option::is_none")]
722 sqlite_write_failure: Option<khive_storage::error::SqliteWriteFailure>,
723 #[serde(default, skip_serializing_if = "Option::is_none")]
724 read_failure: Option<WireReadFailure>,
725 },
726}
727
728#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
731#[serde(tag = "kind", rename_all = "snake_case")]
732pub enum WireWriterTaskFailure {
733 RequestFailed {
734 request_state: WireWriterTaskState,
735 },
736 TaskTerminated {
737 request_state: WireWriterTaskState,
738 #[serde(default, skip_serializing_if = "Option::is_none")]
739 sqlite_full_codes: Option<(i32, i32)>,
740 },
741}
742
743#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
747#[serde(rename_all = "snake_case")]
748pub enum WireWriterTaskState {
749 NotStarted,
750 TransactionRolledBack,
751 SideEffectsUnknown,
752}
753
754impl From<khive_storage::WriterTaskRequestState> for WireWriterTaskState {
755 fn from(state: khive_storage::WriterTaskRequestState) -> Self {
756 use khive_storage::WriterTaskRequestState as S;
757 match state {
758 S::NotStarted => Self::NotStarted,
759 S::TransactionRolledBack => Self::TransactionRolledBack,
760 S::SideEffectsUnknown => Self::SideEffectsUnknown,
761 }
762 }
763}
764
765impl From<WireWriterTaskState> for khive_storage::WriterTaskRequestState {
766 fn from(state: WireWriterTaskState) -> Self {
767 use khive_storage::WriterTaskRequestState as S;
768 match state {
769 WireWriterTaskState::NotStarted => S::NotStarted,
770 WireWriterTaskState::TransactionRolledBack => S::TransactionRolledBack,
771 WireWriterTaskState::SideEffectsUnknown => S::SideEffectsUnknown,
772 }
773 }
774}
775
776#[cfg(unix)]
784pub struct EventsDaemonGuard {
785 _file: std::fs::File,
786}
787
788#[cfg(unix)]
792pub fn try_acquire_events_daemon_guard(socket_path: &Path) -> Option<EventsDaemonGuard> {
793 match acquire_events_daemon_guard_outcome(socket_path) {
794 EventsDaemonGuardAcquisition::Held(guard) => Some(guard),
795 _ => None,
796 }
797}
798
799#[cfg(unix)]
802enum EventsDaemonGuardAcquisition {
803 Held(EventsDaemonGuard),
804 Contended,
805 OpenFailed(std::io::Error),
806 HardeningRefused(std::io::Error),
807}
808
809#[cfg(unix)]
810impl std::fmt::Debug for EventsDaemonGuardAcquisition {
811 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
812 match self {
813 Self::Held(_) => f.write_str("Held"),
814 Self::Contended => f.write_str("Contended"),
815 Self::OpenFailed(error) => f.debug_tuple("OpenFailed").field(error).finish(),
816 Self::HardeningRefused(error) => {
817 f.debug_tuple("HardeningRefused").field(error).finish()
818 }
819 }
820 }
821}
822
823#[cfg(unix)]
829fn acquire_events_daemon_guard_outcome(socket_path: &Path) -> EventsDaemonGuardAcquisition {
830 use EventsDaemonGuardAcquisition::{Contended, HardeningRefused, Held, OpenFailed};
831
832 let lock_path = socket_path.with_extension("lock");
833 if let Some(parent) = lock_path.parent() {
834 if let Err(error) = std::fs::create_dir_all(parent) {
835 return OpenFailed(error);
836 }
837 }
838 use std::os::unix::fs::OpenOptionsExt;
839 let file = match std::fs::OpenOptions::new()
843 .create(true)
844 .truncate(false)
845 .write(true)
846 .mode(0o600)
847 .custom_flags(libc::O_NOFOLLOW)
848 .open(&lock_path)
849 {
850 Ok(file) => file,
851 Err(error) if error.raw_os_error() == Some(libc::ELOOP) => {
852 return HardeningRefused(error);
853 }
854 Err(error) => return OpenFailed(error),
855 };
856 if let Err(error) = file.set_permissions(
862 <std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
863 ) {
864 return HardeningRefused(error);
865 }
866 use std::os::fd::AsRawFd;
867 let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
870 if rc == 0 {
871 Held(EventsDaemonGuard { _file: file })
872 } else {
873 let error = std::io::Error::last_os_error();
874 if error.raw_os_error() == Some(libc::EWOULDBLOCK)
875 || error.raw_os_error() == Some(libc::EAGAIN)
876 {
877 Contended
878 } else {
879 HardeningRefused(error)
880 }
881 }
882}
883
884fn events_db_targets(db_path: &Path) -> [PathBuf; 3] {
885 ["", "-wal", "-shm"].map(|suffix| {
886 let mut name = db_path.as_os_str().to_os_string();
887 name.push(suffix);
888 PathBuf::from(name)
889 })
890}
891
892fn refuse_events_db_symlinks(db_path: &Path) -> anyhow::Result<()> {
902 for path in events_db_targets(db_path) {
903 match std::fs::symlink_metadata(&path) {
904 Ok(meta) if meta.file_type().is_symlink() => anyhow::bail!(
905 "refusing to serve events: {} is a symlink; the events database and its \
906 sidecars must be regular files",
907 path.display()
908 ),
909 Ok(_) => {}
910 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
911 Err(e) => anyhow::bail!(
912 "refusing to serve events: cannot inspect {}: {e}",
913 path.display()
914 ),
915 }
916 }
917 Ok(())
918}
919
920#[cfg(unix)]
921#[derive(Clone, Copy, Debug, PartialEq, Eq)]
922struct EventsFileIdentity {
923 device: u64,
924 inode: u64,
925}
926
927#[cfg(unix)]
928impl EventsFileIdentity {
929 fn from_metadata(metadata: &std::fs::Metadata) -> Self {
930 use std::os::unix::fs::MetadataExt;
931 Self {
932 device: metadata.dev(),
933 inode: metadata.ino(),
934 }
935 }
936}
937
938#[cfg(unix)]
941type EventsDbIdentities = [Option<EventsFileIdentity>; 3];
942
943#[cfg(unix)]
952fn ensure_events_db_parent_trusted(db_path: &Path) -> anyhow::Result<()> {
953 let db_path = absolutize(db_path);
954 let parent = db_path.parent().filter(|p| !p.as_os_str().is_empty());
955 match parent {
956 Some(dir) => crate::daemon::ensure_store_dir_is_trusted(dir, &db_path).map_err(|e| {
957 anyhow::anyhow!("refusing to serve events from an untrusted directory: {e}")
958 }),
959 None => Ok(()),
960 }
961}
962
963#[cfg(unix)]
969fn ensure_events_db_owner_only(db_path: &Path) -> anyhow::Result<()> {
970 use std::os::unix::fs::OpenOptionsExt;
971 refuse_events_db_symlinks(db_path)?;
972 if let Some(parent) = db_path.parent() {
973 std::fs::create_dir_all(parent)?;
974 }
975 ensure_events_db_parent_trusted(db_path)?;
978 match std::fs::OpenOptions::new()
979 .write(true)
980 .create_new(true)
981 .mode(0o600)
982 .open(db_path)
983 {
984 Ok(_created) => Ok(()),
986 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()),
987 Err(e) => Err(anyhow::anyhow!(
988 "refusing to serve events: cannot create {} owner-only: {e}",
989 db_path.display()
990 )),
991 }
992}
993
994#[cfg(unix)]
1007fn harden_events_db_sidecars(db_path: &Path) -> anyhow::Result<EventsDbIdentities> {
1008 use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
1009 let mut identities = [None; 3];
1010 for (index, path) in events_db_targets(db_path).into_iter().enumerate() {
1011 let file = match std::fs::OpenOptions::new()
1018 .read(true)
1019 .custom_flags(libc::O_NOFOLLOW)
1020 .open(&path)
1021 {
1022 Ok(file) => file,
1023 Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
1024 Err(e) => {
1025 anyhow::bail!(
1026 "refusing to serve events: cannot open {} without following symlinks: \
1027 {e}. The events database and its sidecars must be regular files.",
1028 path.display()
1029 )
1030 }
1031 };
1032 let metadata = file.metadata()?;
1035 if !metadata.file_type().is_file() {
1036 anyhow::bail!(
1037 "refusing to serve events: {} is not a regular file. The events \
1038 database and its sidecars must be regular files.",
1039 path.display()
1040 );
1041 }
1042 file.set_permissions(std::fs::Permissions::from_mode(0o600))
1043 .map_err(|e| {
1044 anyhow::anyhow!(
1045 "refusing to serve events: cannot chmod 0600 {}: {e}. The events \
1046 database and its sidecars must be owner-only.",
1047 path.display()
1048 )
1049 })?;
1050 identities[index] = Some(EventsFileIdentity::from_metadata(&metadata));
1053 }
1054 Ok(identities)
1055}
1056
1057#[cfg(unix)]
1075fn verify_events_db_owner_only_unopened(
1076 db_path: &Path,
1077 before: &EventsDbIdentities,
1078) -> anyhow::Result<()> {
1079 use std::os::unix::fs::PermissionsExt;
1080 for (index, path) in events_db_targets(db_path).into_iter().enumerate() {
1081 let metadata = match std::fs::symlink_metadata(&path) {
1082 Ok(metadata) => metadata,
1083 Err(e) if e.kind() == std::io::ErrorKind::NotFound && index != 0 => continue,
1084 Err(e) if e.kind() == std::io::ErrorKind::NotFound => anyhow::bail!(
1085 "refusing to serve events: main database {} disappeared after SQLite open; \
1086 refusing to serve an unlinked database",
1087 path.display()
1088 ),
1089 Err(e) => {
1090 anyhow::bail!(
1091 "refusing to serve events: cannot stat {}: {e}",
1092 path.display()
1093 )
1094 }
1095 };
1096 if !metadata.file_type().is_file() {
1097 anyhow::bail!(
1098 "refusing to serve events: {} is not a regular file. The events database \
1099 and its sidecars must be regular files.",
1100 path.display()
1101 );
1102 }
1103 let mode = metadata.permissions().mode() & 0o777;
1104 if mode & 0o077 != 0 {
1105 anyhow::bail!(
1106 "refusing to serve events: {} is mode {mode:03o}, not owner-only. The events \
1107 database and its sidecars must be owner-only.",
1108 path.display()
1109 );
1110 }
1111 if before[index]
1112 .is_some_and(|identity| identity != EventsFileIdentity::from_metadata(&metadata))
1113 {
1114 anyhow::bail!(
1115 "refusing to serve events: {} changed identity between pre-open hardening \
1116 and post-open verification",
1117 path.display()
1118 );
1119 }
1120 }
1121 Ok(())
1122}
1123
1124#[cfg(unix)]
1127pub const EVENTS_READINESS_WINDOW: Duration = Duration::from_secs(10);
1128
1129#[cfg(unix)]
1145#[derive(Clone, Debug)]
1146pub struct EventsReadiness {
1147 socket: PathBuf,
1148 deadline: tokio::time::Instant,
1149 ready: tokio::sync::watch::Receiver<bool>,
1150}
1151
1152#[cfg(unix)]
1154#[derive(Debug)]
1155pub struct EventsReadinessSignal {
1156 deadline: tokio::time::Instant,
1157 ready: tokio::sync::watch::Sender<bool>,
1158}
1159
1160#[cfg(unix)]
1162#[derive(Clone, Debug, PartialEq, Eq)]
1163pub enum EventsNotReadyReason {
1164 WindowElapsed,
1166 SupervisorStopped,
1168}
1169
1170#[cfg(unix)]
1173#[derive(Clone, Debug, PartialEq, Eq)]
1174pub struct EventsNotReady {
1175 pub socket: PathBuf,
1176 pub reason: EventsNotReadyReason,
1177}
1178
1179#[cfg(unix)]
1180impl std::fmt::Display for EventsNotReady {
1181 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1182 match self.reason {
1183 EventsNotReadyReason::WindowElapsed => write!(
1184 f,
1185 "events daemon at {} has not answered since this daemon started; \
1186 the request was refused before dispatch",
1187 self.socket.display()
1188 ),
1189 EventsNotReadyReason::SupervisorStopped => write!(
1190 f,
1191 "events daemon supervision for {} stopped before the socket answered; \
1192 the request was refused before dispatch",
1193 self.socket.display()
1194 ),
1195 }
1196 }
1197}
1198
1199#[cfg(unix)]
1202pub fn events_readiness(
1203 socket: PathBuf,
1204 window: Duration,
1205) -> (EventsReadinessSignal, EventsReadiness) {
1206 let deadline = tokio::time::Instant::now() + window;
1207 let (sender, receiver) = tokio::sync::watch::channel(false);
1208 (
1209 EventsReadinessSignal {
1210 deadline,
1211 ready: sender,
1212 },
1213 EventsReadiness {
1214 socket,
1215 deadline,
1216 ready: receiver,
1217 },
1218 )
1219}
1220
1221#[cfg(unix)]
1222impl EventsReadinessSignal {
1223 pub fn mark_ready(&self) {
1225 self.ready
1226 .send_if_modified(|ready| !std::mem::replace(ready, true));
1227 }
1228
1229 pub fn awaiting(&self) -> bool {
1232 !*self.ready.borrow() && tokio::time::Instant::now() < self.deadline
1233 }
1234
1235 pub fn is_ready(&self) -> bool {
1237 *self.ready.borrow()
1238 }
1239}
1240
1241#[cfg(unix)]
1242impl EventsReadiness {
1243 pub fn socket(&self) -> &Path {
1245 &self.socket
1246 }
1247
1248 pub fn is_ready(&self) -> bool {
1250 *self.ready.borrow()
1251 }
1252
1253 pub async fn wait(&self) -> Result<(), EventsNotReady> {
1256 let mut ready = self.ready.clone();
1257 let outcome = tokio::time::timeout_at(self.deadline, ready.wait_for(|ready| *ready))
1258 .await
1259 .map(|answered| answered.map(|_| ()));
1260 let refused = |reason| EventsNotReady {
1261 socket: self.socket.clone(),
1262 reason,
1263 };
1264 match outcome {
1265 Ok(Ok(())) => Ok(()),
1266 Ok(Err(_)) => Err(refused(EventsNotReadyReason::SupervisorStopped)),
1267 Err(_) if self.is_ready() => Ok(()),
1268 Err(_) => Err(refused(EventsNotReadyReason::WindowElapsed)),
1269 }
1270 }
1271}
1272
1273#[cfg(unix)]
1277pub fn events_readiness_window() -> Duration {
1278 harness_duration_ms("KHIVE_TEST_EVENTS_READY_WINDOW_MS").unwrap_or(EVENTS_READINESS_WINDOW)
1279}
1280
1281#[cfg(unix)]
1285pub(crate) fn harness_duration_ms(name: &str) -> Option<Duration> {
1286 if std::env::var("KHIVE_TEST_HARNESS").as_deref() != Ok("1") {
1287 return None;
1288 }
1289 std::env::var(name)
1290 .ok()?
1291 .trim()
1292 .parse::<u64>()
1293 .ok()
1294 .map(Duration::from_millis)
1295}
1296
1297#[cfg(unix)]
1323pub async fn supervise_events_daemon(db_path: PathBuf, socket_path: PathBuf) {
1324 match standalone_daemon_policies(&db_path) {
1325 Ok((wal_ceiling, disk_guard, volume_lock_dir)) => {
1326 supervise_events_daemon_with_policies(
1327 db_path,
1328 socket_path,
1329 wal_ceiling,
1330 disk_guard,
1331 volume_lock_dir,
1332 )
1333 .await;
1334 }
1335 Err(error) => {
1336 tracing::warn!(%error, "invalid events daemon policy; events supervisor not started");
1337 }
1338 }
1339}
1340
1341#[cfg(unix)]
1343pub async fn supervise_events_daemon_with_policies(
1344 db_path: PathBuf,
1345 socket_path: PathBuf,
1346 wal_ceiling: WalCeilingPolicy,
1347 disk_guard: khive_db::EffectiveDiskGuardConfig,
1348 volume_lock_dir: PathBuf,
1349) {
1350 let (readiness, _unobserved) = events_readiness(socket_path.clone(), EVENTS_READINESS_WINDOW);
1351 supervise_events_daemon_with_readiness(
1352 db_path,
1353 socket_path,
1354 wal_ceiling,
1355 disk_guard,
1356 volume_lock_dir,
1357 readiness,
1358 )
1359 .await;
1360}
1361
1362#[cfg(unix)]
1375pub async fn supervise_events_daemon_with_readiness(
1376 db_path: PathBuf,
1377 socket_path: PathBuf,
1378 wal_ceiling: WalCeilingPolicy,
1379 disk_guard: khive_db::EffectiveDiskGuardConfig,
1380 volume_lock_dir: PathBuf,
1381 readiness: EventsReadinessSignal,
1382) {
1383 if let Err(error) = disk_guard.validate() {
1384 tracing::warn!(%error, "invalid disk policy; events supervisor not started");
1385 return;
1386 }
1387 if let Err(error) = wal_ceiling.validate_static(true, true, false) {
1388 tracing::warn!(error = %error, "invalid WAL ceiling; events supervisor not started");
1389 return;
1390 }
1391 const PROBE_INTERVAL: Duration = Duration::from_secs(15);
1392 const FAST_PROBE_INTERVAL: Duration = Duration::from_millis(20);
1393 const UNANSWERED_PROBE_INTERVAL: Duration = Duration::from_secs(1);
1394 let shutdown = crate::daemon::daemon_shutdown_token();
1395 let mut child: Option<std::process::Child> = None;
1396 let mut respawns: u64 = 0;
1397 let mut last_spawn: Option<std::time::Instant> = None;
1398 loop {
1399 if let Some(c) = child.as_mut() {
1403 match c.try_wait() {
1404 Ok(Some(status)) => {
1405 tracing::info!(%status, "events daemon child exited");
1406 child = None;
1407 }
1408 Ok(None) => {}
1409 Err(error) => {
1410 tracing::warn!(error = %error, "cannot poll events daemon child; dropping handle");
1411 child = None;
1412 }
1413 }
1414 }
1415
1416 let reachable = match connect_verified(&socket_path).await {
1417 Ok(_stream) => true,
1418 Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
1419 tracing::warn!(
1420 socket = %socket_path.display(),
1421 error = %error,
1422 "events socket answered by a foreign uid; treating as unreachable"
1423 );
1424 false
1425 }
1426 Err(_) => false,
1427 };
1428 if reachable {
1429 readiness.mark_ready();
1430 }
1431 let spawn_due = last_spawn.is_none_or(|at| at.elapsed() >= PROBE_INTERVAL);
1432 if !reachable && child.is_none() && spawn_due {
1433 last_spawn = Some(std::time::Instant::now());
1434 match std::env::current_exe() {
1435 Ok(exe) => {
1436 let spawned = events_daemon_command(
1437 &exe,
1438 &db_path,
1439 &socket_path,
1440 wal_ceiling,
1441 disk_guard,
1442 &volume_lock_dir,
1443 )
1444 .spawn();
1445 match spawned {
1446 Ok(spawned_child) => {
1447 respawns += 1;
1448 tracing::info!(
1449 pid = spawned_child.id(),
1450 respawns,
1451 socket = %socket_path.display(),
1452 "spawned events daemon"
1453 );
1454 child = Some(spawned_child);
1455 }
1456 Err(error) => {
1457 tracing::warn!(error = %error, "failed to spawn events daemon");
1458 }
1459 }
1460 }
1461 Err(error) => {
1462 tracing::warn!(error = %error, "cannot resolve current executable for events daemon spawn");
1463 }
1464 }
1465 }
1466 let pause = if readiness.awaiting() {
1467 FAST_PROBE_INTERVAL
1468 } else if readiness.is_ready() {
1469 PROBE_INTERVAL
1470 } else {
1471 UNANSWERED_PROBE_INTERVAL
1472 };
1473 tokio::select! {
1474 _ = shutdown.cancelled() => break,
1475 _ = tokio::time::sleep(pause) => {}
1476 }
1477 }
1478 if let Some(mut c) = child.take() {
1479 let _ = c.kill();
1483 let _ = c.wait();
1484 tracing::info!("events daemon child stopped with supervisor shutdown");
1485 }
1486}
1487
1488#[cfg(unix)]
1489fn events_daemon_command(
1490 executable: &Path,
1491 db_path: &Path,
1492 socket_path: &Path,
1493 wal_ceiling: WalCeilingPolicy,
1494 disk_guard: khive_db::EffectiveDiskGuardConfig,
1495 volume_lock_dir: &Path,
1496) -> std::process::Command {
1497 let source = match wal_ceiling.source {
1498 khive_db::WalCeilingSource::BackendField => "backend_field",
1499 khive_db::WalCeilingSource::Environment => "environment",
1500 khive_db::WalCeilingSource::Default => "default",
1501 };
1502 let mut command = std::process::Command::new(executable);
1503 command
1504 .arg("events-daemon")
1505 .arg("--db")
1506 .arg(db_path)
1507 .arg("--socket")
1508 .arg(socket_path)
1509 .arg("--wal-ceiling-bytes")
1510 .arg(wal_ceiling.bytes.to_string())
1511 .arg("--wal-ceiling-source")
1512 .arg(source)
1513 .arg("--disk-reserve-bytes")
1514 .arg(disk_guard.reserve_bytes.to_string())
1515 .arg("--disk-guard-deadline-ms")
1516 .arg(disk_guard.guard_deadline_ms.to_string())
1517 .arg("--disk-reserve-source")
1518 .arg(disk_guard.reserve_source.as_str())
1519 .arg("--disk-deadline-source")
1520 .arg(disk_guard.deadline_source.as_str())
1521 .arg("--disk-legacy-environment-present")
1522 .arg(disk_guard.legacy_environment_present.to_string())
1523 .arg("--volume-lock-dir")
1524 .arg(volume_lock_dir)
1525 .stdin(std::process::Stdio::null())
1526 .stdout(std::process::Stdio::null())
1527 .stderr(std::process::Stdio::null());
1528 command
1529}
1530
1531#[cfg(unix)]
1550pub async fn run_events_daemon(db_path: &Path, socket_path: &Path) -> anyhow::Result<()> {
1551 let (wal_ceiling, disk_guard, volume_lock_dir) = standalone_daemon_policies(db_path)?;
1552 run_events_daemon_with_policies(
1553 db_path,
1554 socket_path,
1555 wal_ceiling,
1556 disk_guard,
1557 volume_lock_dir,
1558 )
1559 .await
1560}
1561
1562#[cfg(unix)]
1564pub async fn run_events_daemon_with_policies(
1565 db_path: &Path,
1566 socket_path: &Path,
1567 wal_ceiling: WalCeilingPolicy,
1568 disk_guard: khive_db::EffectiveDiskGuardConfig,
1569 volume_lock_dir: PathBuf,
1570) -> anyhow::Result<()> {
1571 disk_guard.validate()?;
1572 wal_ceiling
1573 .validate_static(true, true, false)
1574 .map_err(crate::error::RuntimeError::from)?;
1575 let db_path = &absolutize(db_path);
1578 let socket_path = &absolutize(socket_path);
1579 validate_events_socket_path(socket_path)?;
1580 if let Some(parent) = socket_path.parent() {
1584 std::fs::create_dir_all(parent)?;
1585 crate::daemon::ensure_socket_dir_is_trusted(parent)?;
1586 }
1587 let _guard = match acquire_events_daemon_guard_outcome(socket_path) {
1588 EventsDaemonGuardAcquisition::Held(guard) => guard,
1589 refusal => {
1590 tracing::info!(
1591 socket = %socket_path.display(),
1592 reason = ?refusal,
1593 "events daemon lock unavailable; exiting"
1594 );
1595 return Ok(());
1596 }
1597 };
1598 ensure_events_db_owner_only(db_path)?;
1599 let before_open = harden_events_db_sidecars(db_path)?;
1612 let backend = Arc::new(
1613 StorageBackend::sqlite_with_max_readers_and_policies(
1614 db_path,
1615 None,
1616 wal_ceiling,
1617 disk_guard,
1618 volume_lock_dir,
1619 )
1620 .map_err(crate::error::RuntimeError::from)?,
1621 );
1622 backend.events()?;
1624 verify_events_db_owner_only_unopened(db_path, &before_open)?;
1625
1626 if let Some(delay) = harness_duration_ms("KHIVE_TEST_EVENTS_DAEMON_BIND_DELAY_MS") {
1629 tracing::info!(
1630 delay_ms = delay.as_millis() as u64,
1631 "test harness: delaying events socket bind"
1632 );
1633 tokio::time::sleep(delay).await;
1634 }
1635 if socket_path.exists() {
1636 std::fs::remove_file(socket_path)?;
1637 }
1638 let listener = UnixListener::bind(socket_path)?;
1639 {
1640 use std::os::unix::fs::PermissionsExt;
1641 if let Err(e) =
1645 std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))
1646 {
1647 drop(listener);
1648 let _ = std::fs::remove_file(socket_path);
1649 anyhow::bail!(
1650 "refusing to serve events: cannot chmod 0600 {}: {e}. The events socket must \
1651 be owner-only.",
1652 socket_path.display()
1653 );
1654 }
1655 }
1656 let daemon_euid = unsafe { libc::geteuid() };
1657 let stores: NamespaceStores = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
1663 let connections = Arc::new(tokio::sync::Semaphore::new(MAX_EVENTS_CONNECTIONS));
1664 let frame_budget = Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_REQUEST_BYTES));
1665 tracing::info!(
1666 socket = %socket_path.display(),
1667 db = %db_path.display(),
1668 wal_ceiling_configured_bytes = wal_ceiling.bytes,
1669 wal_ceiling_effective_bytes = wal_ceiling.effective_bytes(backend.is_read_only()),
1670 wal_ceiling_source = ?wal_ceiling.source,
1671 "events daemon listening"
1672 );
1673
1674 loop {
1675 let (stream, _addr) = match listener.accept().await {
1676 Ok(pair) => pair,
1677 Err(error) => {
1678 tracing::warn!(error = %error, "events daemon accept failed");
1679 continue;
1680 }
1681 };
1682 match crate::daemon::peer_uid(&stream) {
1683 Ok(uid) if crate::daemon::uid_is_permitted(uid, daemon_euid) => {}
1684 Ok(uid) => {
1685 tracing::warn!(peer_uid = uid, "events daemon rejected foreign-uid peer");
1686 continue;
1687 }
1688 Err(error) => {
1689 tracing::warn!(error = %error, "events daemon could not read peer credentials");
1690 continue;
1691 }
1692 }
1693 let permit = match Arc::clone(&connections).try_acquire_owned() {
1694 Ok(permit) => permit,
1695 Err(_) => {
1696 tracing::warn!(
1699 cap = MAX_EVENTS_CONNECTIONS,
1700 "events daemon at connection cap; dropping new connection"
1701 );
1702 continue;
1703 }
1704 };
1705 let backend = Arc::clone(&backend);
1706 let stores = Arc::clone(&stores);
1707 let frame_budget = Arc::clone(&frame_budget);
1708 crate::daemon::spawn_named_tracked_task("events_connection", async move {
1709 serve_events_conn(stream, backend, stores, frame_budget).await;
1710 drop(permit);
1711 });
1712 }
1713}
1714
1715#[cfg(unix)]
1716#[path = "events_split_policy_entries.rs"]
1717mod policy_entries;
1718#[cfg(unix)]
1719use policy_entries::standalone_daemon_policies;
1720#[cfg(unix)]
1721pub use policy_entries::{
1722 run_events_daemon_with_wal_ceiling, supervise_events_daemon_with_wal_ceiling,
1723};
1724
1725#[cfg(all(test, unix))]
1726#[path = "events_wal_policy_tests.rs"]
1727mod wal_policy_tests;
1728
1729#[cfg(unix)]
1737async fn read_frame_budgeted(
1738 stream: &mut UnixStream,
1739 budget: &Arc<tokio::sync::Semaphore>,
1740) -> std::io::Result<(Vec<u8>, tokio::sync::OwnedSemaphorePermit)> {
1741 use tokio::io::AsyncReadExt;
1742 let mut len_buf = [0u8; 4];
1743 stream.read_exact(&mut len_buf).await?;
1744 let len = u32::from_be_bytes(len_buf) as usize;
1745 if len > crate::daemon::MAX_FRAME_BYTES {
1746 return Err(std::io::Error::new(
1747 std::io::ErrorKind::InvalidData,
1748 format!(
1749 "daemon frame of {len} bytes exceeds {} cap",
1750 crate::daemon::MAX_FRAME_BYTES
1751 ),
1752 ));
1753 }
1754 let permit = Arc::clone(budget)
1755 .acquire_many_owned(len as u32)
1756 .await
1757 .map_err(|_| std::io::Error::other("frame budget closed"))?;
1758 let mut buf = vec![0u8; len];
1759 stream.read_exact(&mut buf).await?;
1760 Ok((buf, permit))
1761}
1762
1763#[cfg(unix)]
1764type NamespaceStores =
1765 Arc<std::sync::Mutex<std::collections::HashMap<String, Arc<dyn EventStore>>>>;
1766
1767#[cfg(unix)]
1779fn namespace_store(
1780 backend: &StorageBackend,
1781 stores: &NamespaceStores,
1782 namespace: &str,
1783) -> Result<Arc<dyn EventStore>, khive_db::SqliteError> {
1784 namespace_store_with_cap(backend, stores, namespace, MAX_CACHED_NAMESPACE_STORES)
1785}
1786
1787#[cfg(unix)]
1788fn namespace_store_with_cap(
1789 backend: &StorageBackend,
1790 stores: &NamespaceStores,
1791 namespace: &str,
1792 cap: usize,
1793) -> Result<Arc<dyn EventStore>, khive_db::SqliteError> {
1794 let key = namespace.trim();
1795 if let Some(store) = stores
1796 .lock()
1797 .unwrap_or_else(std::sync::PoisonError::into_inner)
1798 .get(key)
1799 {
1800 return Ok(Arc::clone(store));
1801 }
1802 let store = backend.events_for_namespace(namespace)?;
1803 let mut map = stores
1804 .lock()
1805 .unwrap_or_else(std::sync::PoisonError::into_inner);
1806 if map.len() >= cap && !map.contains_key(key) {
1807 if let Some(victim) = map.keys().next().cloned() {
1815 map.remove(&victim);
1816 }
1817 }
1818 map.insert(key.to_string(), Arc::clone(&store));
1819 Ok(store)
1820}
1821
1822#[cfg(unix)]
1823async fn serve_events_conn(
1824 mut stream: UnixStream,
1825 backend: Arc<StorageBackend>,
1826 stores: NamespaceStores,
1827 frame_budget: Arc<tokio::sync::Semaphore>,
1828) {
1829 loop {
1830 let (payload, budget_permit) = match tokio::time::timeout(
1836 CONN_IO_TIMEOUT,
1837 read_frame_budgeted(&mut stream, &frame_budget),
1838 )
1839 .await
1840 {
1841 Ok(Ok(pair)) => pair,
1842 Ok(Err(_)) | Err(_) => return,
1844 };
1845 let response = match serde_json::from_slice::<EventsRequest>(&payload) {
1846 Ok(request) if request.count_budget_secs().is_some() => {
1847 let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false);
1848 let dispatch = khive_storage::scope_request_read_cancellation(
1849 cancel_rx,
1850 dispatch_events_request(request, &backend, &stores),
1851 );
1852 tokio::pin!(dispatch);
1853 tokio::select! {
1854 response = &mut dispatch => response,
1855 _ = count_peer_disconnected(&stream) => {
1856 let _ = cancel_tx.send(true);
1857 let _ = dispatch.await;
1859 return;
1860 }
1861 }
1862 }
1863 Ok(request) => dispatch_events_request(request, &backend, &stores).await,
1864 Err(error) => EventsResponse::Error {
1865 message: format!("events daemon could not parse request frame: {error}"),
1866 retryable: false,
1867 writer_task_failure: None,
1868 sqlite_write_failure: None,
1869 read_failure: None,
1870 },
1871 };
1872 let bytes = match serde_json::to_vec(&response) {
1873 Ok(bytes) => bytes,
1874 Err(error) => {
1875 tracing::error!(error = %error, "events daemon response serialization failed");
1876 return;
1877 }
1878 };
1879 let bytes = if bytes.len() > crate::daemon::MAX_FRAME_BYTES {
1880 let refusal = EventsResponse::Error {
1884 message: format!(
1885 "response_frame_size_limit: {} response bytes exceed the {}-byte events IPC frame cap; request a narrower page",
1886 bytes.len(),
1887 crate::daemon::MAX_FRAME_BYTES
1888 ),
1889 retryable: false,
1890 writer_task_failure: None,
1891 sqlite_write_failure: None,
1892 read_failure: None,
1893 };
1894 match serde_json::to_vec(&refusal) {
1895 Ok(bytes) => bytes,
1896 Err(error) => {
1897 tracing::error!(error = %error, "events daemon frame-size refusal serialization failed");
1898 return;
1899 }
1900 }
1901 } else {
1902 bytes
1903 };
1904 match tokio::time::timeout(CONN_IO_TIMEOUT, write_frame(&mut stream, &bytes)).await {
1907 Ok(Ok(())) => {}
1908 Ok(Err(_)) | Err(_) => return,
1909 }
1910 drop(budget_permit);
1914 }
1915}
1916
1917#[cfg(unix)]
1918async fn count_peer_disconnected(stream: &UnixStream) {
1919 loop {
1920 if stream.readable().await.is_err() {
1921 return;
1922 }
1923 let mut byte = [0u8; 1];
1924 match stream.try_read(&mut byte) {
1925 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => continue,
1926 _ => return,
1928 }
1929 }
1930}
1931
1932#[cfg(unix)]
1933async fn dispatch_events_request(
1934 request: EventsRequest,
1935 backend: &StorageBackend,
1936 stores: &NamespaceStores,
1937) -> EventsResponse {
1938 if request.protocol_version() != EVENTS_PROTOCOL_VERSION {
1939 return EventsResponse::Error {
1940 message: format!(
1941 "events protocol version mismatch: daemon speaks {}, client sent {}",
1942 EVENTS_PROTOCOL_VERSION,
1943 request.protocol_version()
1944 ),
1945 retryable: false,
1946 writer_task_failure: None,
1947 sqlite_write_failure: None,
1948 read_failure: None,
1949 };
1950 }
1951 if let Err(error) = khive_types::Namespace::parse(request.namespace().trim()) {
1957 return EventsResponse::Error {
1958 message: format!("events request namespace rejected: {error}"),
1959 retryable: false,
1960 writer_task_failure: None,
1961 sqlite_write_failure: None,
1962 read_failure: None,
1963 };
1964 }
1965 let count_budget_secs = match request.count_budget_secs() {
1966 Some(Some(seconds))
1967 if (1..=khive_storage::request_context::MAX_REQUEST_READ_TIMEOUT_SECS)
1968 .contains(&seconds) =>
1969 {
1970 Some(seconds)
1971 }
1972 Some(_) => {
1973 return storage_error_response(&StorageError::InvalidInput {
1974 capability: khive_storage::StorageCapability::Events,
1975 operation: "events count".into(),
1976 message: "missing or invalid operator_read_budget_secs attribution".into(),
1977 })
1978 }
1979 None => None,
1980 };
1981 if let Some(seconds) = count_budget_secs {
1982 if let Err(error) = khive_storage::ensure_request_read_active("events count") {
1983 return count_storage_error_response(error, seconds);
1984 }
1985 }
1986 let store = match namespace_store(backend, stores, request.namespace()) {
1987 Ok(store) => store,
1988 Err(error) => {
1989 return EventsResponse::Error {
1990 message: format!("events store unavailable: {error}"),
1991 retryable: true,
1992 writer_task_failure: None,
1993 sqlite_write_failure: None,
1994 read_failure: None,
1995 };
1996 }
1997 };
1998 if let Some(seconds) = count_budget_secs {
1999 if let Err(error) = khive_storage::ensure_request_read_active("events count") {
2000 return count_storage_error_response(error, seconds);
2001 }
2002 }
2003 match request {
2004 EventsRequest::AppendEvents { events, .. } => match store.append_events(events).await {
2005 Ok(summary) => EventsResponse::Appended { summary },
2006 Err(error) => storage_error_response(&error),
2007 },
2008 EventsRequest::AppendEventsIdempotent { events, .. } => {
2009 match store.append_events_idempotent(events).await {
2010 Ok(result) => EventsResponse::Idempotent { result },
2011 Err(error) => storage_error_response(&error),
2012 }
2013 }
2014 EventsRequest::GetEvent { id, .. } => match store.get_event(id).await {
2015 Ok(event) => EventsResponse::Event { event },
2016 Err(error) => storage_error_response(&error),
2017 },
2018 EventsRequest::QueryEvents { filter, page, .. } => {
2019 if page.limit > MAX_QUERY_EVENTS_PAGE_ROWS {
2020 return EventsResponse::Error {
2021 message: format!(
2022 "events query page limit {} exceeds the daemon cap of {} rows; \
2023 request narrower pages",
2024 page.limit, MAX_QUERY_EVENTS_PAGE_ROWS
2025 ),
2026 retryable: false,
2027 writer_task_failure: None,
2028 sqlite_write_failure: None,
2029 read_failure: None,
2030 };
2031 }
2032 match store.query_events(filter, page).await {
2033 Ok(page) => EventsResponse::Pageful { page },
2034 Err(error) => storage_error_response(&error),
2035 }
2036 }
2037 EventsRequest::QueryEventPage { query, .. } => {
2038 if let Err(error) = crate::event_page::validate_page_query(&query) {
2039 return storage_error_response(&error);
2040 }
2041 match store.query_event_page(query).await {
2042 Ok(window) => EventsResponse::EventPageWindow { window },
2043 Err(error) => storage_error_response(&error),
2044 }
2045 }
2046 EventsRequest::CountEvents { filter, .. } => {
2047 let Some(seconds) = count_budget_secs else {
2048 return storage_error_response(&StorageError::InvalidInput {
2049 capability: khive_storage::StorageCapability::Events,
2050 operation: "events count".into(),
2051 message: "missing operator_read_budget_secs attribution".into(),
2052 });
2053 };
2054 match store.count_events(filter).await {
2055 Ok(count) => match khive_storage::ensure_request_read_active("events count") {
2056 Ok(()) => EventsResponse::Count { count },
2057 Err(error) => count_storage_error_response(error, seconds),
2058 },
2059 Err(error) => count_storage_error_response(error, seconds),
2060 }
2061 }
2062 EventsRequest::CountEventsGrouped {
2063 filter, group_by, ..
2064 } => {
2065 let Some(seconds) = count_budget_secs else {
2066 return storage_error_response(&StorageError::InvalidInput {
2067 capability: khive_storage::StorageCapability::Events,
2068 operation: "events count".into(),
2069 message: "missing operator_read_budget_secs attribution".into(),
2070 });
2071 };
2072 match store.count_events_grouped(filter, group_by).await {
2073 Ok(counts) => match khive_storage::ensure_request_read_active("events count") {
2074 Ok(()) => EventsResponse::GroupedCount { counts },
2075 Err(error) => count_storage_error_response(error, seconds),
2076 },
2077 Err(error) => count_storage_error_response(error, seconds),
2078 }
2079 }
2080 }
2081}
2082
2083#[cfg(unix)]
2084fn storage_error_response(error: &StorageError) -> EventsResponse {
2085 let (message, writer_task_failure) = match error.without_sqlite_write_stage() {
2086 StorageError::WriterTaskRequestFailed {
2087 request_state,
2088 source,
2089 } => (
2090 source.to_string(),
2091 Some(WireWriterTaskFailure::RequestFailed {
2092 request_state: (*request_state).into(),
2093 }),
2094 ),
2095 StorageError::WriterTaskTerminated {
2096 request_state,
2097 sqlite_full_codes,
2098 } => (
2099 error.to_string(),
2100 Some(WireWriterTaskFailure::TaskTerminated {
2101 request_state: (*request_state).into(),
2102 sqlite_full_codes: *sqlite_full_codes,
2103 }),
2104 ),
2105 _ => (error.to_string(), None),
2106 };
2107 EventsResponse::Error {
2108 message,
2109 retryable: error.is_retryable(),
2114 writer_task_failure,
2115 sqlite_write_failure: error.sqlite_write_failure(),
2116 read_failure: None,
2117 }
2118}
2119
2120#[cfg(unix)]
2131async fn connect_verified(socket_path: &Path) -> std::io::Result<UnixStream> {
2132 let stream = UnixStream::connect(socket_path).await?;
2133 let own_euid = unsafe { libc::geteuid() } as u32;
2135 let peer = crate::daemon::peer_uid(&stream)?;
2136 if !crate::daemon::uid_is_permitted(peer, own_euid) {
2137 return Err(std::io::Error::new(
2138 std::io::ErrorKind::PermissionDenied,
2139 format!(
2140 "events socket {} answered by uid {peer}, not this process's uid {own_euid}; \
2141 refusing to exchange event frames with an unowned daemon",
2142 socket_path.display()
2143 ),
2144 ));
2145 }
2146 Ok(stream)
2147}
2148
2149#[cfg(unix)]
2153#[derive(Debug, Clone, Copy, Serialize)]
2154pub struct EventsForwardingMetrics {
2155 pub forwarded_batches: u64,
2156 pub forwarded_events: u64,
2157 pub dropped_batches: u64,
2158 pub dropped_events: u64,
2159 pub queued_bytes: usize,
2161}
2162
2163#[cfg(unix)]
2164#[derive(Debug, Default)]
2165struct ForwardingCounters {
2166 forwarded_batches: AtomicU64,
2167 forwarded_events: AtomicU64,
2168 dropped_batches: AtomicU64,
2169 dropped_events: AtomicU64,
2170 queued_bytes: AtomicUsize,
2171}
2172
2173#[cfg(unix)]
2176#[derive(Debug)]
2177struct AppendByteReservation {
2178 counters: Arc<ForwardingCounters>,
2179 bytes: usize,
2180}
2181
2182#[cfg(unix)]
2183impl Drop for AppendByteReservation {
2184 fn drop(&mut self) {
2185 self.counters
2186 .queued_bytes
2187 .fetch_sub(self.bytes, Ordering::AcqRel);
2188 }
2189}
2190
2191#[cfg(unix)]
2192#[derive(Debug)]
2193struct QueuedAppend {
2194 request: EventsRequest,
2195 count: u64,
2196 _reservation: AppendByteReservation,
2197}
2198
2199#[cfg(all(test, unix))]
2200impl QueuedAppend {
2201 fn unmetered(namespace: &str, events: Vec<Event>, counters: Arc<ForwardingCounters>) -> Self {
2202 let count = events.len() as u64;
2203 Self {
2204 request: EventsRequest::AppendEvents {
2205 protocol_version: EVENTS_PROTOCOL_VERSION,
2206 namespace: namespace.to_string(),
2207 events,
2208 },
2209 count,
2210 _reservation: AppendByteReservation { counters, bytes: 0 },
2211 }
2212 }
2213}
2214
2215#[cfg(unix)]
2218#[derive(Default)]
2219struct BoundedFrameCounter {
2220 bytes: usize,
2221 exceeded: bool,
2222}
2223
2224#[cfg(unix)]
2225impl std::io::Write for BoundedFrameCounter {
2226 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
2227 let next = self.bytes.saturating_add(buf.len());
2228 if next > crate::daemon::MAX_FRAME_BYTES {
2229 self.exceeded = true;
2230 return Err(std::io::Error::new(
2231 std::io::ErrorKind::InvalidData,
2232 "events append request exceeds IPC frame cap",
2233 ));
2234 }
2235 self.bytes = next;
2236 Ok(buf.len())
2237 }
2238
2239 fn flush(&mut self) -> std::io::Result<()> {
2240 Ok(())
2241 }
2242}
2243
2244#[cfg(unix)]
2245fn reserve_append_bytes(
2246 counters: &Arc<ForwardingCounters>,
2247 bytes: usize,
2248 budget: usize,
2249) -> Option<AppendByteReservation> {
2250 let mut used = counters.queued_bytes.load(Ordering::Acquire);
2251 loop {
2252 let next = used.checked_add(bytes)?;
2253 if next > budget {
2254 return None;
2255 }
2256 match counters.queued_bytes.compare_exchange_weak(
2257 used,
2258 next,
2259 Ordering::AcqRel,
2260 Ordering::Acquire,
2261 ) {
2262 Ok(_) => {
2263 return Some(AppendByteReservation {
2264 counters: Arc::clone(counters),
2265 bytes,
2266 });
2267 }
2268 Err(observed) => used = observed,
2269 }
2270 }
2271}
2272
2273#[cfg(unix)]
2281pub struct EventsSplitClient {
2282 socket_path: PathBuf,
2283 append_tx: tokio::sync::mpsc::Sender<QueuedAppend>,
2284 append_queue_byte_budget: usize,
2285 counters: Arc<ForwardingCounters>,
2286 outage_logged: Arc<AtomicBool>,
2289 preflight_store: Arc<dyn EventStore>,
2291}
2292
2293#[cfg(unix)]
2294impl std::fmt::Debug for EventsSplitClient {
2295 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2296 f.debug_struct("EventsSplitClient")
2297 .field("socket_path", &self.socket_path)
2298 .finish_non_exhaustive()
2299 }
2300}
2301
2302#[cfg(unix)]
2303impl EventsSplitClient {
2304 pub fn new(socket_path: PathBuf) -> crate::error::RuntimeResult<Arc<Self>> {
2306 Self::new_with_queue_depth(socket_path, DEFAULT_APPEND_QUEUE_BATCHES)
2307 }
2308
2309 pub fn new_with_queue_depth(
2312 socket_path: PathBuf,
2313 queue_depth: usize,
2314 ) -> crate::error::RuntimeResult<Arc<Self>> {
2315 Self::new_with_queue_depth_and_delivery_timeout(socket_path, queue_depth, REQUEST_TIMEOUT)
2316 }
2317
2318 fn new_with_queue_depth_and_delivery_timeout(
2322 socket_path: PathBuf,
2323 queue_depth: usize,
2324 delivery_timeout: Duration,
2325 ) -> crate::error::RuntimeResult<Arc<Self>> {
2326 Self::new_with_limits_and_delivery_timeout(
2327 socket_path,
2328 queue_depth,
2329 DEFAULT_APPEND_QUEUE_BYTES,
2330 delivery_timeout,
2331 )
2332 }
2333
2334 fn new_with_limits_and_delivery_timeout(
2335 socket_path: PathBuf,
2336 queue_depth: usize,
2337 byte_budget: usize,
2338 delivery_timeout: Duration,
2339 ) -> crate::error::RuntimeResult<Arc<Self>> {
2340 validate_events_socket_path(&socket_path)?;
2341 let preflight_backend = StorageBackend::memory()?;
2342 let preflight_store = preflight_backend.events()?;
2343 let (append_tx, append_rx) = tokio::sync::mpsc::channel::<QueuedAppend>(queue_depth.max(1));
2347 let counters = Arc::new(ForwardingCounters::default());
2348 let outage_logged = Arc::new(AtomicBool::new(false));
2349
2350 let client = Arc::new(Self {
2351 socket_path: socket_path.clone(),
2352 append_tx,
2353 append_queue_byte_budget: byte_budget,
2354 counters: Arc::clone(&counters),
2355 outage_logged: Arc::clone(&outage_logged),
2356 preflight_store,
2357 });
2358
2359 crate::daemon::spawn_named_tracked_task(
2360 "events_forwarder",
2361 run_forwarder(
2362 socket_path,
2363 append_rx,
2364 counters,
2365 outage_logged,
2366 delivery_timeout,
2367 crate::daemon::daemon_shutdown_token(),
2368 ),
2369 );
2370 Ok(client)
2371 }
2372
2373 pub fn metrics(&self) -> EventsForwardingMetrics {
2375 EventsForwardingMetrics {
2376 forwarded_batches: self.counters.forwarded_batches.load(Ordering::Relaxed),
2377 forwarded_events: self.counters.forwarded_events.load(Ordering::Relaxed),
2378 dropped_batches: self.counters.dropped_batches.load(Ordering::Relaxed),
2379 dropped_events: self.counters.dropped_events.load(Ordering::Relaxed),
2380 queued_bytes: self.counters.queued_bytes.load(Ordering::Acquire),
2381 }
2382 }
2383
2384 fn enqueue(&self, namespace: &str, events: Vec<Event>) {
2387 let count = events.len() as u64;
2388 let request = EventsRequest::AppendEvents {
2389 protocol_version: EVENTS_PROTOCOL_VERSION,
2390 namespace: namespace.to_string(),
2391 events,
2392 };
2393 let mut frame_size = BoundedFrameCounter::default();
2394 if let Err(error) = serde_json::to_writer(&mut frame_size, &request) {
2395 self.count_append_drop(
2396 count,
2397 if frame_size.exceeded {
2398 "frame cap"
2399 } else {
2400 "serialization"
2401 },
2402 );
2403 tracing::warn!(error = %error, "events append batch could not fit a wire frame");
2404 return;
2405 }
2406 let Some(reservation) = reserve_append_bytes(
2407 &self.counters,
2408 frame_size.bytes,
2409 self.append_queue_byte_budget,
2410 ) else {
2411 self.count_append_drop(count, "queue byte budget");
2412 return;
2413 };
2414 let batch = QueuedAppend {
2415 request,
2416 count,
2417 _reservation: reservation,
2418 };
2419 match self.append_tx.try_send(batch) {
2420 Ok(()) => {}
2421 Err(_) => {
2422 self.count_append_drop(count, "queue batch limit");
2424 }
2425 }
2426 }
2427
2428 fn count_append_drop(&self, count: u64, reason: &'static str) {
2429 self.counters
2430 .dropped_batches
2431 .fetch_add(1, Ordering::Relaxed);
2432 self.counters
2433 .dropped_events
2434 .fetch_add(count, Ordering::Relaxed);
2435 if !self.outage_logged.swap(true, Ordering::Relaxed) {
2436 tracing::warn!(
2437 dropped_events = count,
2438 reason,
2439 "events append queue rejected loss-tolerant batch"
2440 );
2441 }
2442 }
2443
2444 async fn round_trip(&self, request: &EventsRequest) -> StorageResult<EventsResponse> {
2449 let op = "events daemon round-trip";
2450 let payload = serialize_events_request(request)?;
2451 let attempt = events_round_trip_attempt(&self.socket_path, &payload);
2452 let bytes = match tokio::time::timeout(REQUEST_TIMEOUT, attempt).await {
2453 Ok(Ok(bytes)) => bytes,
2454 Ok(Err(("read", error))) => {
2455 return Err(StorageError::Serialization {
2456 capability: khive_storage::StorageCapability::Events,
2457 message: format!(
2458 "events daemon closed or broke the response after connection at {}: {error}",
2459 self.socket_path.display()
2460 ),
2461 });
2462 }
2463 Ok(Err((stage, error))) => {
2464 return Err(StorageError::Pool {
2465 operation: op.into(),
2466 message: format!(
2467 "events daemon {stage} failed at {}: {error}",
2468 self.socket_path.display()
2469 ),
2470 });
2471 }
2472 Err(_elapsed) => {
2473 return Err(StorageError::Timeout {
2474 operation: op.into(),
2475 });
2476 }
2477 };
2478
2479 deserialize_events_response(&bytes)
2480 }
2481
2482 async fn round_trip_count(
2483 &self,
2484 request: &EventsRequest,
2485 budget: &CountReadBudget,
2486 ) -> StorageResult<EventsResponse> {
2487 budget.scope(async {
2488 let payload = serialize_events_request(request)?;
2489 let attempt = events_round_trip_attempt(&self.socket_path, &payload);
2490 let result = khive_storage::await_request_read_phase("events daemon count round-trip", attempt)
2491 .await.map_err(|error| budget.attribute(error))?;
2492 let bytes = match result {
2493 Ok(bytes) => bytes,
2494 Err(("read", error)) => return Err(StorageError::Serialization {
2495 capability: khive_storage::StorageCapability::Events,
2496 message: format!("events daemon closed or broke the response after connection at {}: {error}", self.socket_path.display()),
2497 }),
2498 Err((stage, error)) => return Err(StorageError::Pool {
2499 operation: "events daemon count round-trip".into(),
2500 message: format!("events daemon {stage} failed at {}: {error}", self.socket_path.display()),
2501 }),
2502 };
2503 deserialize_events_response(&bytes)
2504 }).await
2505 }
2506}
2507
2508#[cfg(unix)]
2509fn serialize_events_request(request: &EventsRequest) -> StorageResult<Vec<u8>> {
2510 serde_json::to_vec(request).map_err(|error| StorageError::Serialization {
2511 capability: khive_storage::StorageCapability::Events,
2512 message: format!("events request serialization failed: {error}"),
2513 })
2514}
2515
2516#[cfg(unix)]
2517fn deserialize_events_response(bytes: &[u8]) -> StorageResult<EventsResponse> {
2518 serde_json::from_slice(bytes).map_err(|error| StorageError::Serialization {
2519 capability: khive_storage::StorageCapability::Events,
2520 message: format!("events response deserialization failed: {error}"),
2521 })
2522}
2523
2524#[cfg(unix)]
2525async fn events_round_trip_attempt(
2526 socket_path: &Path,
2527 payload: &[u8],
2528) -> Result<Vec<u8>, (&'static str, std::io::Error)> {
2529 let mut stream = connect_verified(socket_path)
2530 .await
2531 .map_err(|error| ("connect", error))?;
2532 write_frame(&mut stream, payload)
2533 .await
2534 .map_err(|error| ("write", error))?;
2535 read_frame(&mut stream)
2536 .await
2537 .map_err(|error| ("read", error))
2538}
2539
2540#[cfg(unix)]
2547fn drain_dropped_queue(
2548 rx: &mut tokio::sync::mpsc::Receiver<QueuedAppend>,
2549 counters: &ForwardingCounters,
2550) {
2551 let mut dropped_batches = 0u64;
2552 let mut dropped_events = 0u64;
2553 while let Ok(batch) = rx.try_recv() {
2554 dropped_batches += 1;
2555 dropped_events += batch.count;
2556 }
2558 if dropped_batches > 0 {
2559 counters
2560 .dropped_batches
2561 .fetch_add(dropped_batches, Ordering::Relaxed);
2562 counters
2563 .dropped_events
2564 .fetch_add(dropped_events, Ordering::Relaxed);
2565 tracing::warn!(
2566 dropped_batches,
2567 dropped_events,
2568 "events forwarder shutting down; dropping queued loss-tolerant batches"
2569 );
2570 }
2571}
2572
2573#[cfg(unix)]
2578async fn run_forwarder(
2579 socket_path: PathBuf,
2580 mut rx: tokio::sync::mpsc::Receiver<QueuedAppend>,
2581 counters: Arc<ForwardingCounters>,
2582 outage_logged: Arc<AtomicBool>,
2583 delivery_timeout: Duration,
2584 shutdown: tokio_util::sync::CancellationToken,
2585) {
2586 let mut conn: Option<UnixStream> = None;
2594 loop {
2595 let batch = tokio::select! {
2596 _ = shutdown.cancelled() => {
2597 drain_dropped_queue(&mut rx, &counters);
2598 break;
2599 },
2600 received = rx.recv() => match received {
2601 Some(batch) => batch,
2602 None => break,
2603 },
2604 };
2605 let count = batch.count;
2606 let payload = match serde_json::to_vec(&batch.request) {
2607 Ok(payload) => payload,
2608 Err(error) => {
2609 counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
2610 counters.dropped_events.fetch_add(count, Ordering::Relaxed);
2611 tracing::error!(error = %error, "events forwarder serialization failed; batch dropped");
2612 continue;
2613 }
2614 };
2615
2616 let delivered = tokio::select! {
2624 _ = shutdown.cancelled() => {
2625 counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
2626 counters.dropped_events.fetch_add(count, Ordering::Relaxed);
2627 tracing::warn!(
2628 dropped_events = count,
2629 "events forwarder shutting down mid-delivery; in-flight batch dropped"
2630 );
2631 drain_dropped_queue(&mut rx, &counters);
2632 break;
2633 },
2634 outcome = tokio::time::timeout(
2635 delivery_timeout,
2636 deliver_batch(&socket_path, &mut conn, &payload),
2637 ) => match outcome {
2638 Ok(delivered) => delivered,
2639 Err(_elapsed) => {
2640 conn = None;
2641 false
2642 }
2643 },
2644 };
2645 if delivered {
2646 counters.forwarded_batches.fetch_add(1, Ordering::Relaxed);
2647 counters
2648 .forwarded_events
2649 .fetch_add(count, Ordering::Relaxed);
2650 if outage_logged.swap(false, Ordering::Relaxed) {
2651 tracing::info!("events daemon reachable again; forwarding resumed");
2652 }
2653 } else {
2654 counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
2655 counters.dropped_events.fetch_add(count, Ordering::Relaxed);
2656 if !outage_logged.swap(true, Ordering::Relaxed) {
2657 tracing::warn!(
2658 socket = %socket_path.display(),
2659 "events daemon unreachable; dropping loss-tolerant events until it returns"
2660 );
2661 }
2662 tokio::select! {
2663 _ = shutdown.cancelled() => {
2664 drain_dropped_queue(&mut rx, &counters);
2665 break;
2666 },
2667 _ = tokio::time::sleep(FORWARDER_BACKOFF) => {}
2668 }
2669 }
2670 }
2671}
2672
2673#[cfg(unix)]
2678async fn deliver_batch(socket_path: &Path, conn: &mut Option<UnixStream>, payload: &[u8]) -> bool {
2679 for _attempt in 0..2u8 {
2680 if conn.is_none() {
2681 match connect_verified(socket_path).await {
2682 Ok(stream) => *conn = Some(stream),
2683 Err(_) => return false,
2684 }
2685 }
2686 let stream = conn.as_mut().expect("connection populated above");
2687 let ok = async {
2688 write_frame(stream, payload).await?;
2689 let bytes = read_frame(stream).await?;
2690 std::io::Result::Ok(bytes)
2691 }
2692 .await;
2693 match ok {
2694 Ok(bytes) => {
2695 return !matches!(
2696 serde_json::from_slice::<EventsResponse>(&bytes),
2697 Ok(EventsResponse::Error { .. }) | Err(_)
2698 );
2699 }
2700 Err(_) => {
2701 *conn = None;
2704 }
2705 }
2706 }
2707 false
2708}
2709
2710#[cfg(unix)]
2719#[derive(Debug)]
2720pub struct ForwardingEventStore {
2721 namespace: String,
2722 client: Arc<EventsSplitClient>,
2723}
2724
2725#[cfg(unix)]
2726impl ForwardingEventStore {
2727 pub fn new(namespace: impl Into<String>, client: Arc<EventsSplitClient>) -> Self {
2728 Self {
2729 namespace: namespace.into(),
2730 client,
2731 }
2732 }
2733
2734 fn unexpected_count(&self, op: &'static str, response: EventsResponse) -> StorageError {
2735 match restore_count_read_failure(response) {
2736 Err(error) => error,
2737 Ok(other) => self.unexpected(op, other),
2738 }
2739 }
2740
2741 fn unexpected(&self, op: &'static str, response: EventsResponse) -> StorageError {
2742 match response {
2743 EventsResponse::Error {
2744 message,
2745 retryable,
2746 writer_task_failure,
2747 sqlite_write_failure,
2748 read_failure,
2749 } => {
2750 if read_failure.is_some() {
2751 return events_count_read_budget::malformed_read_failure(
2752 "non-count response carries read failure evidence",
2753 );
2754 }
2755 let source = if retryable {
2758 StorageError::Pool {
2759 operation: op.into(),
2760 message,
2761 }
2762 } else {
2763 StorageError::InvalidInput {
2764 capability: khive_storage::StorageCapability::Events,
2765 operation: op.into(),
2766 message,
2767 }
2768 };
2769 let source = match sqlite_write_failure {
2770 Some(failure) => source.with_sqlite_write_failure(failure),
2771 None => source,
2772 };
2773 match writer_task_failure {
2774 Some(WireWriterTaskFailure::RequestFailed { request_state }) => {
2775 StorageError::WriterTaskRequestFailed {
2776 request_state: request_state.into(),
2777 source: Box::new(source),
2778 }
2779 }
2780 Some(WireWriterTaskFailure::TaskTerminated {
2781 request_state,
2782 sqlite_full_codes,
2783 }) => {
2784 let terminal = StorageError::WriterTaskTerminated {
2785 request_state: request_state.into(),
2786 sqlite_full_codes,
2787 };
2788 match sqlite_write_failure {
2789 Some(failure) => terminal.with_sqlite_write_failure(failure),
2790 None => terminal,
2791 }
2792 }
2793 None => source,
2794 }
2795 }
2796 other => StorageError::Serialization {
2797 capability: khive_storage::StorageCapability::Events,
2798 message: format!("events daemon returned mismatched response for {op}: {other:?}"),
2799 },
2800 }
2801 }
2802}
2803
2804#[cfg(unix)]
2805#[async_trait]
2806impl EventStore for ForwardingEventStore {
2807 async fn append_event(&self, event: Event) -> StorageResult<()> {
2808 self.client.enqueue(&self.namespace, vec![event]);
2809 Ok(())
2810 }
2811
2812 async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
2813 let attempted = events.len() as u64;
2814 self.client.enqueue(&self.namespace, events);
2815 Ok(BatchWriteSummary {
2818 attempted,
2819 affected: attempted,
2820 ..BatchWriteSummary::default()
2821 })
2822 }
2823
2824 async fn get_event(&self, id: Uuid) -> StorageResult<Option<Event>> {
2825 let request = EventsRequest::GetEvent {
2826 protocol_version: EVENTS_PROTOCOL_VERSION,
2827 namespace: self.namespace.clone(),
2828 id,
2829 };
2830 match self.client.round_trip(&request).await? {
2831 EventsResponse::Event { event } => Ok(event),
2832 other => Err(self.unexpected("get_event", other)),
2833 }
2834 }
2835
2836 async fn query_events(
2837 &self,
2838 filter: EventFilter,
2839 page: PageRequest,
2840 ) -> StorageResult<Page<Event>> {
2841 let request = EventsRequest::QueryEvents {
2842 protocol_version: EVENTS_PROTOCOL_VERSION,
2843 namespace: self.namespace.clone(),
2844 filter,
2845 page,
2846 };
2847 match self.client.round_trip(&request).await? {
2848 EventsResponse::Pageful { page } => Ok(page),
2849 other => Err(self.unexpected("query_events", other)),
2850 }
2851 }
2852
2853 async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
2854 let budget = CountReadBudget::capture();
2855 let request = EventsRequest::CountEvents {
2856 protocol_version: EVENTS_PROTOCOL_VERSION,
2857 namespace: self.namespace.clone(),
2858 operator_read_budget_secs: Some(budget.operator_read_budget_secs),
2859 filter,
2860 };
2861 match self.client.round_trip_count(&request, &budget).await? {
2862 EventsResponse::Count { count } => Ok(count),
2863 other => Err(self.unexpected_count("count_events", other)),
2864 }
2865 }
2866
2867 async fn count_events_grouped(
2868 &self,
2869 filter: EventFilter,
2870 group_by: EventGroupBy,
2871 ) -> StorageResult<BTreeMap<String, u64>> {
2872 let budget = CountReadBudget::capture();
2873 let request = EventsRequest::CountEventsGrouped {
2874 protocol_version: EVENTS_PROTOCOL_VERSION,
2875 namespace: self.namespace.clone(),
2876 operator_read_budget_secs: Some(budget.operator_read_budget_secs),
2877 filter,
2878 group_by,
2879 };
2880 match self.client.round_trip_count(&request, &budget).await? {
2881 EventsResponse::GroupedCount { counts } => Ok(counts),
2882 other => Err(self.unexpected_count("count_events_grouped", other)),
2883 }
2884 }
2885
2886 async fn query_event_page(&self, query: EventPageQuery) -> StorageResult<EventPageWindow> {
2887 crate::event_page::validate_page_query(&query)?;
2888 let request = EventsRequest::QueryEventPage {
2889 protocol_version: EVENTS_PROTOCOL_VERSION,
2890 namespace: self.namespace.clone(),
2891 query: query.clone(),
2892 };
2893 match self.client.round_trip(&request).await? {
2894 EventsResponse::EventPageWindow { window } => {
2895 crate::event_page::validate_window(&query, Some(&self.namespace), &window)?;
2896 Ok(window)
2897 }
2898 error @ EventsResponse::Error { .. } => Err(self.unexpected("query_event_page", error)),
2899 _ => Err(crate::event_page::page_error(
2900 "events page response kind mismatch",
2901 )),
2902 }
2903 }
2904
2905 fn preflight_event(&self, event: &Event) -> StorageResult<()> {
2906 self.client.preflight_store.preflight_event(event)
2908 }
2909
2910 async fn append_events_idempotent(
2911 &self,
2912 events: Vec<Event>,
2913 ) -> StorageResult<IdempotentEventBatchResult> {
2914 let request = EventsRequest::AppendEventsIdempotent {
2915 protocol_version: EVENTS_PROTOCOL_VERSION,
2916 namespace: self.namespace.clone(),
2917 events,
2918 };
2919 match self.client.round_trip(&request).await? {
2920 EventsResponse::Idempotent { result } => Ok(result),
2921 other => Err(self.unexpected("append_events_idempotent", other)),
2922 }
2923 }
2924
2925 fn supports_idempotent_audit_batch(&self) -> bool {
2926 true
2927 }
2928}
2929
2930pub struct SplitEventStore {
2947 legacy: Arc<dyn EventStore>,
2948 lane: Arc<dyn EventStore>,
2949}
2950
2951impl std::fmt::Debug for SplitEventStore {
2952 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2953 f.debug_struct("SplitEventStore").finish_non_exhaustive()
2954 }
2955}
2956
2957impl SplitEventStore {
2958 pub const MAX_MERGED_WINDOW_ROWS: u64 = 100_000;
2966
2967 pub fn new(legacy: Arc<dyn EventStore>, lane: Arc<dyn EventStore>) -> Self {
2968 Self { legacy, lane }
2969 }
2970}
2971
2972#[async_trait]
2973impl EventStore for SplitEventStore {
2974 async fn append_event(&self, event: Event) -> StorageResult<()> {
2975 self.legacy.append_event(event).await
2976 }
2977
2978 async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
2979 self.legacy.append_events(events).await
2980 }
2981
2982 async fn get_event(&self, id: Uuid) -> StorageResult<Option<Event>> {
2983 if let Some(event) = self.legacy.get_event(id).await? {
2986 return Ok(Some(event));
2987 }
2988 self.lane.get_event(id).await
2989 }
2990
2991 async fn query_events(
2992 &self,
2993 filter: EventFilter,
2994 page: PageRequest,
2995 ) -> StorageResult<Page<Event>> {
2996 let window = page.offset.saturating_add(u64::from(page.limit));
3010 if window > Self::MAX_MERGED_WINDOW_ROWS {
3011 return Err(StorageError::InvalidInput {
3012 capability: khive_storage::StorageCapability::Events,
3013 operation: "query_events".into(),
3014 message: format!(
3015 "offset+limit ({window}) exceeds the merged event plane's window bound \
3016 of {}; page deep windows with a `before` cursor at offset 0, or narrow \
3017 the filter",
3018 Self::MAX_MERGED_WINDOW_ROWS
3019 ),
3020 });
3021 }
3022 let prefix = PageRequest {
3023 offset: 0,
3024 limit: window.min(u64::from(u32::MAX)) as u32,
3025 };
3026 let legacy = self
3027 .legacy
3028 .query_events(filter.clone(), prefix.clone())
3029 .await?;
3030 let lane = self.lane.query_events(filter, prefix).await?;
3031 let total = match (legacy.total, lane.total) {
3032 (Some(a), Some(b)) => Some(a + b),
3033 _ => None,
3034 };
3035 let mut items = legacy.items;
3036 items.extend(lane.items);
3037 items.sort_by(|a, b| {
3038 b.created_at
3039 .cmp(&a.created_at)
3040 .then_with(|| b.id.cmp(&a.id))
3041 });
3042 let items = items
3043 .into_iter()
3044 .skip(page.offset as usize)
3045 .take(page.limit as usize)
3046 .collect();
3047 Ok(Page { items, total })
3048 }
3049
3050 async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
3051 let budget = CountReadBudget::capture();
3052 budget
3053 .scope(async {
3054 let legacy = self
3055 .legacy
3056 .count_events(filter.clone())
3057 .await
3058 .map_err(|error| budget.attribute(error))?;
3059 let lane = self.lane.count_events(filter).await?;
3060 Ok(legacy + lane)
3061 })
3062 .await
3063 }
3064
3065 async fn count_events_grouped(
3066 &self,
3067 filter: EventFilter,
3068 group_by: EventGroupBy,
3069 ) -> StorageResult<BTreeMap<String, u64>> {
3070 let budget = CountReadBudget::capture();
3071 budget
3072 .scope(async {
3073 let mut counts = self
3074 .legacy
3075 .count_events_grouped(filter.clone(), group_by)
3076 .await
3077 .map_err(|error| budget.attribute(error))?;
3078 for (key, count) in self.lane.count_events_grouped(filter, group_by).await? {
3079 let total = counts.entry(key).or_default();
3080 *total =
3081 total
3082 .checked_add(count)
3083 .ok_or_else(|| StorageError::InvalidInput {
3084 capability: khive_storage::StorageCapability::Events,
3085 operation: "count_events_grouped".into(),
3086 message: "grouped event count overflow".into(),
3087 })?;
3088 }
3089 Ok(counts)
3090 })
3091 .await
3092 }
3093
3094 async fn query_event_page(&self, query: EventPageQuery) -> StorageResult<EventPageWindow> {
3095 crate::event_page::split_page(self.legacy.as_ref(), self.lane.as_ref(), query).await
3096 }
3097
3098 fn preflight_event(&self, event: &Event) -> StorageResult<()> {
3099 self.lane.preflight_event(event)
3102 }
3103
3104 async fn append_events_idempotent(
3105 &self,
3106 events: Vec<Event>,
3107 ) -> StorageResult<IdempotentEventBatchResult> {
3108 if events.is_empty() {
3118 return self.lane.append_events_idempotent(events).await;
3119 }
3120 let ids: Vec<Uuid> = events.iter().map(|event| event.id).collect();
3121 let probe_limit = u32::try_from(ids.len()).map_err(|_| StorageError::InvalidInput {
3122 capability: khive_storage::StorageCapability::Events,
3123 operation: "append_events_idempotent".into(),
3124 message: format!(
3125 "batch of {} rows exceeds the legacy probe window",
3126 ids.len()
3127 ),
3128 })?;
3129 let existing = self
3130 .legacy
3131 .query_events(
3132 EventFilter {
3133 ids,
3134 ..EventFilter::default()
3135 },
3136 PageRequest {
3137 offset: 0,
3138 limit: probe_limit,
3139 },
3140 )
3141 .await?;
3142 let legacy_resident: std::collections::HashSet<Uuid> =
3143 existing.items.iter().map(|event| event.id).collect();
3144 if legacy_resident.is_empty() {
3145 return self.lane.append_events_idempotent(events).await;
3146 }
3147 let mut legacy_rows = Vec::new();
3148 let mut lane_rows = Vec::new();
3149 let mut routed_to_legacy = Vec::with_capacity(events.len());
3150 for event in events {
3151 if legacy_resident.contains(&event.id) {
3152 routed_to_legacy.push(true);
3153 legacy_rows.push(event);
3154 } else {
3155 routed_to_legacy.push(false);
3156 lane_rows.push(event);
3157 }
3158 }
3159 let legacy_expected = legacy_rows.len();
3160 let lane_expected = lane_rows.len();
3161 let legacy_result = self.legacy.append_events_idempotent(legacy_rows).await?;
3162 let lane_result = if lane_expected == 0 {
3163 IdempotentEventBatchResult { rows: Vec::new() }
3164 } else {
3165 self.lane.append_events_idempotent(lane_rows).await?
3166 };
3167 if legacy_result.rows.len() != legacy_expected || lane_result.rows.len() != lane_expected {
3168 return Err(StorageError::Driver {
3169 capability: khive_storage::StorageCapability::Events,
3170 operation: "append_events_idempotent".into(),
3171 source: format!(
3172 "idempotent sub-batch result length mismatch: legacy {}/{}, lane {}/{}",
3173 legacy_result.rows.len(),
3174 legacy_expected,
3175 lane_result.rows.len(),
3176 lane_expected
3177 )
3178 .into(),
3179 });
3180 }
3181 let mut legacy_iter = legacy_result.rows.into_iter();
3182 let mut lane_iter = lane_result.rows.into_iter();
3183 let rows = routed_to_legacy
3184 .into_iter()
3185 .map(|to_legacy| {
3186 if to_legacy {
3187 legacy_iter.next().expect("length checked above")
3188 } else {
3189 lane_iter.next().expect("length checked above")
3190 }
3191 })
3192 .collect();
3193 Ok(IdempotentEventBatchResult { rows })
3194 }
3195
3196 fn supports_idempotent_audit_batch(&self) -> bool {
3197 self.lane.supports_idempotent_audit_batch() && self.legacy.supports_idempotent_audit_batch()
3201 }
3202}
3203
3204#[cfg(all(test, unix))]
3205#[path = "events_count_read_budget_tests.rs"]
3206mod count_read_budget_tests;
3207
3208#[cfg(test)]
3209#[path = "events_split_sqlite_full_tests.rs"]
3210mod sqlite_full_tests;
3211
3212#[cfg(test)]
3213mod tests {
3214 use super::*;
3215
3216 #[test]
3217 fn fixture_registry_cleanup_preserves_other_roots() {
3218 let live_dir = tempfile::tempdir().unwrap();
3219 let _live_guard = TestRegistryGuard::new(live_dir.path());
3220 let live_path = live_dir.path().join("events.db");
3221 let live_backend = direct_backend_for(&live_path).unwrap();
3222
3223 let finished_dir = tempfile::tempdir().unwrap();
3224 let finished_backend = {
3225 let _finished_guard = TestRegistryGuard::new(finished_dir.path());
3226 let backend = direct_backend_for(&finished_dir.path().join("events.db")).unwrap();
3227 let weak = Arc::downgrade(&backend);
3228 drop(backend);
3229 assert!(
3230 weak.upgrade().is_some(),
3231 "registry retains the live fixture"
3232 );
3233 weak
3234 };
3235
3236 assert!(
3237 finished_backend.upgrade().is_none(),
3238 "FD_FIXTURE_REGISTRY_RELEASED"
3239 );
3240 let same_live_backend = direct_backend_for(&live_path).unwrap();
3241 assert!(
3242 Arc::ptr_eq(&live_backend, &same_live_backend),
3243 "FD_FIXTURE_OTHER_ROOT_RETAINED"
3244 );
3245 }
3246
3247 #[cfg(unix)]
3248 mod target_filter_tests {
3249 include!("events_split_target_tests.rs");
3250 }
3251
3252 include!("events_sidecar_path_tests.rs");
3257
3258 #[cfg(unix)]
3259 mod store_trust_tests {
3260 use super::*;
3261 include!("events_store_trust_tests.rs");
3262 }
3263
3264 #[cfg(unix)]
3265 #[test]
3266 fn deep_symlink_chains_within_the_kernel_bound_derive_the_target_sidecar() {
3267 let dir = tempfile::tempdir().unwrap();
3272 let real = dir.path().join("real.db");
3273 let mut prev = real.clone();
3274 for i in 0..35 {
3275 let link = dir.path().join(format!("hop{i}.db"));
3276 std::os::unix::fs::symlink(&prev, &link).unwrap();
3277 prev = link;
3278 }
3279 assert_eq!(
3280 events_db_path_beside(&prev),
3281 events_db_path_beside(&real),
3282 "a 35-hop dangling chain must derive the target's sidecar"
3283 );
3284 assert_ne!(
3286 events_db_path_beside(&dir.path().join("unrelated.db")),
3287 events_db_path_beside(&real)
3288 );
3289 }
3290
3291 include!("events_sidecar_hardening_tests.rs");
3292
3293 #[cfg(unix)]
3294 fn write_owner_only_test_file(path: &Path) {
3295 use std::os::unix::fs::PermissionsExt;
3296 std::fs::write(path, b"").unwrap();
3297 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).unwrap();
3298 }
3299
3300 #[cfg(unix)]
3301 #[test]
3302 fn unopened_check_refuses_same_mode_regular_file_replacement() {
3303 for target_index in 0..3 {
3307 let dir = tempfile::tempdir().unwrap();
3308 let db = dir.path().join("events.db");
3309 let targets = events_db_targets(&db);
3310 for path in &targets {
3311 write_owner_only_test_file(path);
3312 }
3313 let before_open = harden_events_db_sidecars(&db).unwrap();
3314 let replacement = dir.path().join("replacement");
3315 write_owner_only_test_file(&replacement);
3316 let replacement_id = EventsFileIdentity::from_metadata(
3317 &std::fs::symlink_metadata(&replacement).unwrap(),
3318 );
3319 assert_ne!(before_open[target_index], Some(replacement_id));
3320 let _connection = rusqlite::Connection::open(&db).unwrap();
3323 verify_events_db_owner_only_unopened(&db, &before_open).unwrap();
3324 std::fs::rename(&replacement, &targets[target_index]).unwrap();
3325 let error = verify_events_db_owner_only_unopened(&db, &before_open)
3326 .unwrap_err()
3327 .to_string();
3328 assert!(error.contains("changed identity"), "{error}");
3329 assert!(
3330 error.contains(&targets[target_index].display().to_string()),
3331 "{error}"
3332 );
3333 }
3334 }
3335
3336 #[cfg(unix)]
3337 #[test]
3338 fn unopened_check_accepts_sidecars_missing_at_check_time() {
3339 let dir = tempfile::tempdir().unwrap();
3340 let db = dir.path().join("events.db");
3341 let targets = events_db_targets(&db);
3342 for path in &targets {
3343 write_owner_only_test_file(path);
3344 }
3345 let before_open = harden_events_db_sidecars(&db).unwrap();
3346 let _connection = rusqlite::Connection::open(&db).unwrap();
3347 for path in targets.iter().skip(1) {
3348 std::fs::remove_file(path).unwrap();
3349 verify_events_db_owner_only_unopened(&db, &before_open)
3350 .expect("SQLite sidecars may disappear without losing the main database");
3351 }
3352 }
3353
3354 #[cfg(unix)]
3355 #[test]
3356 fn unopened_check_refuses_main_database_removed_after_sqlite_open() {
3357 let dir = tempfile::tempdir().unwrap();
3358 let db = dir.path().join("events.db");
3359 ensure_events_db_owner_only(&db).unwrap();
3360 let before_open = harden_events_db_sidecars(&db).unwrap();
3361 let backend = StorageBackend::sqlite_for_test(&db).unwrap();
3362 backend.events().unwrap();
3363 verify_events_db_owner_only_unopened(&db, &before_open).unwrap();
3364 std::fs::remove_file(&db).unwrap();
3365 let error = verify_events_db_owner_only_unopened(&db, &before_open)
3368 .unwrap_err()
3369 .to_string();
3370 assert!(
3371 error.contains("main database") && error.contains("disappeared"),
3372 "{error}"
3373 );
3374 }
3375
3376 #[cfg(unix)]
3377 #[test]
3378 fn unopened_check_accepts_fresh_sqlite_sidecars_and_unchanged_main() {
3379 let dir = tempfile::tempdir().unwrap();
3380 let db = dir.path().join("new.events.db");
3381 assert!(!db.exists());
3382 ensure_events_db_owner_only(&db).unwrap();
3383 let before_open = harden_events_db_sidecars(&db).unwrap();
3384 assert!(before_open[0].is_some());
3385 assert_eq!(&before_open[1..], &[None, None]);
3386 let backend = StorageBackend::sqlite_for_test(&db).unwrap();
3387 backend.events().unwrap();
3388 for path in events_db_targets(&db).iter().skip(1) {
3389 assert!(path.exists(), "SQLite creates {}", path.display());
3390 }
3391 verify_events_db_owner_only_unopened(&db, &before_open)
3392 .expect("new SQLite sidecars have no pre-open identity to contradict");
3393 }
3394
3395 #[test]
3396 fn socket_derives_beside_the_sidecar() {
3397 let db = events_db_path_beside(Path::new("/data/khive.db"));
3398 let socket = events_socket_path_beside(&db);
3399 assert!(socket.ends_with("khive.db.events.sock"), "got {socket:?}");
3400 }
3401
3402 #[test]
3403 fn bare_relative_main_db_yields_an_absolute_sidecar() {
3404 let path = events_db_path_beside(Path::new("khive.db"));
3408 assert!(path.is_absolute(), "got relative {path:?}");
3409 assert!(path.ends_with("khive.db.events.db"), "got {path:?}");
3410 }
3411
3412 #[cfg(unix)]
3413 #[tokio::test]
3414 async fn over_cap_query_page_limit_is_refused_before_materialization() {
3415 let backend = Arc::new(StorageBackend::memory().unwrap());
3420 let stores: NamespaceStores =
3421 Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
3422 let request = |limit: u32| EventsRequest::QueryEvents {
3423 protocol_version: EVENTS_PROTOCOL_VERSION,
3424 namespace: "local".to_string(),
3425 filter: EventFilter::default(),
3426 page: PageRequest { offset: 0, limit },
3427 };
3428 let refused =
3429 dispatch_events_request(request(MAX_QUERY_EVENTS_PAGE_ROWS + 1), &backend, &stores)
3430 .await;
3431 match refused {
3432 EventsResponse::Error {
3433 message,
3434 retryable,
3435 writer_task_failure,
3436 ..
3437 } => {
3438 assert!(
3439 message.contains(&MAX_QUERY_EVENTS_PAGE_ROWS.to_string()),
3440 "refusal must name the cap: {message}"
3441 );
3442 assert!(!retryable, "an over-cap page is not transient");
3443 assert!(writer_task_failure.is_none());
3444 }
3445 other => panic!("over-cap query must be refused, got {other:?}"),
3446 }
3447 let at_cap =
3450 dispatch_events_request(request(MAX_QUERY_EVENTS_PAGE_ROWS), &backend, &stores).await;
3451 assert!(
3452 matches!(at_cap, EventsResponse::Pageful { .. }),
3453 "at-cap query must reach the store, got {at_cap:?}"
3454 );
3455 }
3456
3457 #[cfg(unix)]
3458 #[test]
3459 fn lease_contention_crosses_events_wire_as_retryable() {
3460 assert_eq!(EVENTS_PROTOCOL_VERSION, 6);
3461 for (phase, expected_retryable) in [
3462 (khive_storage::CapacityUnavailablePhase::LockContended, true),
3463 (khive_storage::CapacityUnavailablePhase::Lock, false),
3464 ] {
3465 let response = storage_error_response(&StorageError::CapacityUnavailable {
3466 capability: khive_storage::StorageCapability::Sql,
3467 phase,
3468 message: "lease admission failed".into(),
3469 });
3470 let encoded = serde_json::to_vec(&response).expect("serialize server error");
3471 let decoded: EventsResponse =
3472 serde_json::from_slice(&encoded).expect("decode unchanged protocol");
3473 let EventsResponse::Error { retryable, .. } = decoded else {
3474 panic!("capacity failure must be an error frame");
3475 };
3476 assert_eq!(retryable, expected_retryable, "{phase:?}");
3477 }
3478 }
3479
3480 #[cfg(unix)]
3481 #[test]
3482 fn side_effects_unknown_state_crosses_the_wire_as_terminal_failure() {
3483 let response = storage_error_response(&StorageError::writer_task_terminated(
3484 khive_storage::WriterTaskRequestState::SideEffectsUnknown,
3485 ));
3486 assert!(
3487 matches!(
3488 response,
3489 EventsResponse::Error {
3490 writer_task_failure: Some(WireWriterTaskFailure::TaskTerminated {
3491 request_state: WireWriterTaskState::SideEffectsUnknown,
3492 sqlite_full_codes: None,
3493 }),
3494 ..
3495 }
3496 ),
3497 "an unknown-commit-state termination must carry its state on the wire"
3498 );
3499 let busy = storage_error_response(&StorageError::WriterTaskBusy { timeout_ms: 5 });
3501 assert!(matches!(
3502 busy,
3503 EventsResponse::Error {
3504 writer_task_failure: None,
3505 sqlite_write_failure: None,
3506 read_failure: None,
3507 ..
3508 }
3509 ));
3510 }
3511
3512 #[cfg(unix)]
3513 #[test]
3514 fn proven_rollback_crosses_the_wire_without_claiming_task_termination() {
3515 let response = storage_error_response(&StorageError::WriterTaskRequestFailed {
3516 request_state: khive_storage::WriterTaskRequestState::TransactionRolledBack,
3517 source: Box::new(StorageError::Pool {
3518 operation: "writer_task_commit".into(),
3519 message: "commit refused".into(),
3520 }),
3521 });
3522 assert!(matches!(
3523 response,
3524 EventsResponse::Error {
3525 writer_task_failure: Some(WireWriterTaskFailure::RequestFailed {
3526 request_state: WireWriterTaskState::TransactionRolledBack,
3527 }),
3528 ..
3529 }
3530 ));
3531 }
3532
3533 #[test]
3534 fn error_frames_without_writer_failure_still_parse() {
3535 let bytes = br#"{"kind":"error","message":"boom","retryable":true}"#;
3539 let parsed: EventsResponse = serde_json::from_slice(bytes).expect("stateless frame parses");
3540 assert!(matches!(
3541 parsed,
3542 EventsResponse::Error {
3543 retryable: true,
3544 writer_task_failure: None,
3545 sqlite_write_failure: None,
3546 read_failure: None,
3547 ..
3548 }
3549 ));
3550 }
3551
3552 fn split_retry_event(verb: &str) -> Event {
3553 Event::new(
3554 "test",
3555 verb,
3556 khive_types::EventKind::RecallExecuted,
3557 khive_types::SubstrateKind::Note,
3558 "agent:test",
3559 )
3560 }
3561
3562 fn store_pair(dir: &Path) -> (Arc<dyn EventStore>, Arc<dyn EventStore>) {
3563 let legacy = direct_backend_for(&dir.join("legacy.db"))
3564 .expect("legacy backend")
3565 .events_for_namespace("test")
3566 .expect("legacy store");
3567 let lane = direct_backend_for(&dir.join("lane.db"))
3568 .expect("lane backend")
3569 .events_for_namespace("test")
3570 .expect("lane store");
3571 (legacy, lane)
3572 }
3573
3574 #[tokio::test]
3579 async fn idempotent_retry_of_legacy_resident_rows_does_not_duplicate() {
3580 use khive_storage::event::EventAppendDisposition;
3581
3582 let dir = tempfile::tempdir().unwrap();
3583 let _registry_guard = TestRegistryGuard::new(dir.path());
3584 let (legacy, lane) = store_pair(dir.path());
3585
3586 let resident = split_retry_event("recall");
3587 legacy
3588 .append_events_idempotent(vec![resident.clone()])
3589 .await
3590 .expect("pre-cutover landing");
3591
3592 let split = SplitEventStore::new(Arc::clone(&legacy), Arc::clone(&lane));
3593 let fresh = split_retry_event("search");
3594 let result = split
3595 .append_events_idempotent(vec![resident.clone(), fresh.clone()])
3596 .await
3597 .expect("mixed retry batch");
3598 assert_eq!(
3599 result.rows,
3600 vec![
3601 EventAppendDisposition::AlreadyPresentIdentical,
3602 EventAppendDisposition::Inserted,
3603 ],
3604 "input order must be preserved across the two sub-batches"
3605 );
3606 assert_eq!(
3607 lane.count_events(EventFilter::default()).await.unwrap(),
3608 1,
3609 "the legacy-resident row must not reach the lane"
3610 );
3611 assert_eq!(
3612 split.count_events(EventFilter::default()).await.unwrap(),
3613 2,
3614 "merged count must not double-count the retried id"
3615 );
3616
3617 let mut mutated = resident.clone();
3620 mutated.verb = "other".to_string();
3621 let conflict = split
3622 .append_events_idempotent(vec![mutated])
3623 .await
3624 .expect("conflicting retry");
3625 assert_eq!(
3626 conflict.rows,
3627 vec![EventAppendDisposition::IdentityConflict]
3628 );
3629 assert_eq!(split.count_events(EventFilter::default()).await.unwrap(), 2);
3630 }
3631
3632 #[tokio::test]
3635 async fn merged_offset_window_is_bounded() {
3636 let dir = tempfile::tempdir().unwrap();
3637 let _registry_guard = TestRegistryGuard::new(dir.path());
3638 let (legacy, lane) = store_pair(dir.path());
3639 let split = SplitEventStore::new(legacy, lane);
3640
3641 let err = split
3642 .query_events(
3643 EventFilter::default(),
3644 PageRequest {
3645 offset: SplitEventStore::MAX_MERGED_WINDOW_ROWS,
3646 limit: 1,
3647 },
3648 )
3649 .await
3650 .expect_err("a window past the bound must be refused");
3651 assert!(
3652 matches!(err, StorageError::InvalidInput { .. }),
3653 "got {err:?}"
3654 );
3655 assert!(
3656 err.to_string().contains("before"),
3657 "the refusal must name the cursor remedy: {err}"
3658 );
3659
3660 let at_bound = split
3661 .query_events(
3662 EventFilter::default(),
3663 PageRequest {
3664 offset: SplitEventStore::MAX_MERGED_WINDOW_ROWS - 1,
3665 limit: 1,
3666 },
3667 )
3668 .await
3669 .expect("a window at the bound is admitted");
3670 assert!(at_bound.items.is_empty());
3671 }
3672
3673 #[cfg(unix)]
3674 #[tokio::test]
3675 async fn stateless_non_retryable_error_maps_terminal_not_writer_terminated() {
3676 let client = EventsSplitClient::new(std::path::PathBuf::from("/tmp/never-bound.sock"))
3683 .expect("client builds");
3684 let store = ForwardingEventStore::new("test", client);
3685 let bytes = br#"{"kind":"error","message":"refused","retryable":false}"#;
3686 let parsed: EventsResponse = serde_json::from_slice(bytes).expect("frame parses");
3687 assert!(matches!(
3688 store.unexpected("append", parsed),
3689 StorageError::InvalidInput { .. }
3690 ));
3691 let with_state = br#"{"kind":"error","message":"died","retryable":false,"writer_task_failure":{"kind":"task_terminated","request_state":"side_effects_unknown"}}"#;
3695 let parsed: EventsResponse =
3696 serde_json::from_slice(with_state).expect("stateful frame parses");
3697 assert!(matches!(
3698 store.unexpected("append", parsed),
3699 StorageError::WriterTaskTerminated { .. }
3700 ));
3701 }
3702
3703 #[cfg(unix)]
3704 #[tokio::test]
3705 async fn writer_task_states_cross_the_wire_verbatim() {
3706 use khive_storage::WriterTaskRequestState as S;
3712 let dir = tempfile::tempdir().unwrap();
3713 let client =
3714 EventsSplitClient::new(dir.path().join("never-bound.sock")).expect("client builds");
3715 let store = ForwardingEventStore::new("test", client);
3716 for state in [
3717 S::NotStarted,
3718 S::TransactionRolledBack,
3719 S::SideEffectsUnknown,
3720 ] {
3721 let response = storage_error_response(&StorageError::writer_task_terminated(state));
3722 let bytes = serde_json::to_vec(&response).unwrap();
3724 let parsed: EventsResponse = serde_json::from_slice(&bytes).unwrap();
3725 let err = store.unexpected("append_events_idempotent", parsed);
3726 assert!(
3727 matches!(
3728 err,
3729 StorageError::WriterTaskTerminated { request_state, .. } if request_state == state
3730 ),
3731 "state {state:?} did not survive the socket: got {err:?}"
3732 );
3733 }
3734 let busy = storage_error_response(&StorageError::WriterTaskBusy { timeout_ms: 5 });
3736 assert!(matches!(
3737 busy,
3738 EventsResponse::Error {
3739 writer_task_failure: None,
3740 sqlite_write_failure: None,
3741 read_failure: None,
3742 ..
3743 }
3744 ));
3745 }
3746
3747 #[cfg(unix)]
3748 #[tokio::test]
3749 async fn native_write_stages_survive_direct_and_request_error_frames() {
3750 use khive_storage::error::{SqliteWriteFailure, SqliteWriteStage};
3751 use khive_storage::WriterTaskRequestState;
3752 let dir = tempfile::tempdir().unwrap();
3753 let client = EventsSplitClient::new(dir.path().join("never-bound.sock")).unwrap();
3754 let store = ForwardingEventStore::new("test", client);
3755 for (stage, extended) in [
3756 (SqliteWriteStage::Begin, rusqlite::ffi::SQLITE_BUSY_RECOVERY),
3757 (
3758 SqliteWriteStage::Statement,
3759 rusqlite::ffi::SQLITE_CONSTRAINT_UNIQUE,
3760 ),
3761 (SqliteWriteStage::Commit, rusqlite::ffi::SQLITE_IOERR_FSYNC),
3762 (SqliteWriteStage::Statement, rusqlite::ffi::SQLITE_FULL),
3763 ] {
3764 for request in [false, true] {
3765 for retryable in [false, true] {
3766 let failure = SqliteWriteFailure {
3767 stage,
3768 primary_code: extended & 0xff,
3769 extended_code: extended,
3770 settlement_unknown: false,
3771 };
3772 let source = if retryable {
3773 StorageError::Pool {
3774 operation: "append".into(),
3775 message: "no code in this message".into(),
3776 }
3777 } else {
3778 StorageError::InvalidInput {
3779 capability: khive_storage::StorageCapability::Events,
3780 operation: "append".into(),
3781 message: "no code in this message".into(),
3782 }
3783 }
3784 .with_sqlite_write_failure(failure);
3785 let source = if request {
3786 StorageError::WriterTaskRequestFailed {
3787 request_state: WriterTaskRequestState::TransactionRolledBack,
3788 source: Box::new(source),
3789 }
3790 } else {
3791 source
3792 };
3793 let bytes = serde_json::to_vec(&storage_error_response(&source)).unwrap();
3794 let error = store.unexpected("append", serde_json::from_slice(&bytes).unwrap());
3795 assert_eq!(error.sqlite_write_failure(), Some(failure));
3796 assert_eq!(error.is_retryable(), retryable);
3797 assert_eq!(
3798 matches!(
3799 &error,
3800 StorageError::WriterTaskRequestFailed {
3801 request_state: WriterTaskRequestState::TransactionRolledBack,
3802 ..
3803 }
3804 ),
3805 request
3806 );
3807 let wire =
3808 crate::runtime_error_value(error.into(), crate::DomainDisposition::Unknown);
3809 let expected = if extended == rusqlite::ffi::SQLITE_FULL {
3810 "sqlite_disk_full"
3811 } else {
3812 stage.as_str()
3813 };
3814 assert_eq!(wire["stage"], expected);
3815 assert_eq!(wire["sqlite_primary_code"], failure.primary_code);
3816 assert_eq!(wire["sqlite_extended_code"], extended);
3817 if request {
3818 assert_eq!(wire["task_terminated"], false);
3819 }
3820 }
3821 }
3822 }
3823 }
3824
3825 #[cfg(unix)]
3826 #[tokio::test]
3827 async fn terminal_native_evidence_keeps_unknown_outcome_precedence() {
3828 use khive_storage::error::{SqliteWriteFailure, SqliteWriteStage};
3829 use khive_storage::WriterTaskRequestState;
3830 let dir = tempfile::tempdir().unwrap();
3831 let client = EventsSplitClient::new(dir.path().join("never-bound.sock")).unwrap();
3832 let store = ForwardingEventStore::new("test", client);
3833 for terminal in [false, true] {
3834 for extended in [
3835 rusqlite::ffi::SQLITE_IOERR_FSYNC,
3836 rusqlite::ffi::SQLITE_FULL,
3837 ] {
3838 let failure = SqliteWriteFailure {
3839 stage: SqliteWriteStage::Commit,
3840 primary_code: extended & 0xff,
3841 extended_code: extended,
3842 settlement_unknown: false,
3843 };
3844 let source = if terminal {
3846 StorageError::WriterTaskTerminated {
3847 request_state: WriterTaskRequestState::SideEffectsUnknown,
3848 sqlite_full_codes: (extended == rusqlite::ffi::SQLITE_FULL)
3849 .then_some((extended, extended)),
3850 }
3851 } else {
3852 StorageError::WriterTaskRequestFailed {
3853 request_state: WriterTaskRequestState::SideEffectsUnknown,
3854 source: Box::new(StorageError::Internal("original cause".into())),
3855 }
3856 }
3857 .with_sqlite_write_failure(failure);
3858 let bytes = serde_json::to_vec(&storage_error_response(&source)).unwrap();
3859 let error = store.unexpected("append", serde_json::from_slice(&bytes).unwrap());
3860 assert_eq!(error.sqlite_write_failure(), Some(failure));
3861 if terminal {
3862 assert!(
3863 std::error::Error::source(error.without_sqlite_write_stage()).is_none()
3864 );
3865 }
3866 let wire =
3867 crate::runtime_error_value(error.into(), crate::DomainDisposition::Unknown);
3868 let expected = if extended == rusqlite::ffi::SQLITE_FULL {
3869 "sqlite_disk_full"
3870 } else if terminal {
3871 "writer_task_terminated"
3872 } else {
3873 "writer_task_request_failed"
3874 };
3875 assert_eq!(wire["code"], expected);
3876 assert_eq!(wire["stage"], expected);
3877 assert_eq!(wire["sqlite_write_stage"], "sqlite_commit_failure");
3878 assert_eq!(wire["sqlite_extended_code"], extended);
3879 assert_eq!(wire["request_state"], "side_effects_unknown");
3880 assert_eq!(wire["task_terminated"], terminal);
3881 assert_eq!(wire["retryable"], false);
3882 }
3883 }
3884 }
3885
3886 #[cfg(unix)]
3887 #[tokio::test]
3888 async fn poisoned_commit_evidence_does_not_claim_definite_failure() {
3889 use khive_storage::error::{SqliteWriteFailure, SqliteWriteStage};
3890 let dir = tempfile::tempdir().unwrap();
3891 let client = EventsSplitClient::new(dir.path().join("never-bound.sock")).unwrap();
3892 let store = ForwardingEventStore::new("test", client);
3893 let failure = SqliteWriteFailure {
3894 stage: SqliteWriteStage::Commit,
3895 primary_code: rusqlite::ffi::SQLITE_AUTH,
3896 extended_code: rusqlite::ffi::SQLITE_AUTH,
3897 settlement_unknown: true,
3898 };
3899 let original = StorageError::driver(
3900 khive_storage::StorageCapability::Sql,
3901 "execute_batch",
3902 rusqlite::Error::SqliteFailure(
3903 rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_AUTH),
3904 None,
3905 ),
3906 );
3907 let source = original.with_sqlite_write_failure(failure);
3908 let bytes = serde_json::to_vec(&storage_error_response(&source)).unwrap();
3909 let error = store.unexpected("append", serde_json::from_slice(&bytes).unwrap());
3910 assert_eq!(error.sqlite_write_failure(), Some(failure));
3911 assert!(!error.is_retryable());
3912 let wire = crate::runtime_error_value(error.into(), crate::DomainDisposition::Unknown);
3913 assert!(wire.get("code").is_none());
3914 assert!(wire.get("stage").is_none());
3915 assert_eq!(wire["sqlite_write_stage"], "sqlite_commit_failure");
3916 assert_eq!(wire["sqlite_primary_code"], rusqlite::ffi::SQLITE_AUTH);
3917 assert_eq!(wire["domain_disposition"], "unknown");
3918 }
3919
3920 #[cfg(unix)]
3921 #[tokio::test]
3922 async fn proven_rollback_round_trips_as_non_terminal_request_failure() {
3923 use khive_storage::WriterTaskRequestState as S;
3924
3925 let dir = tempfile::tempdir().unwrap();
3926 let client =
3927 EventsSplitClient::new(dir.path().join("never-bound.sock")).expect("client builds");
3928 let store = ForwardingEventStore::new("test", client);
3929 let response = storage_error_response(&StorageError::WriterTaskRequestFailed {
3930 request_state: S::TransactionRolledBack,
3931 source: Box::new(StorageError::Pool {
3932 operation: "writer_task_commit".into(),
3933 message: "commit refused".into(),
3934 }),
3935 });
3936 let bytes = serde_json::to_vec(&response).unwrap();
3937 let parsed: EventsResponse = serde_json::from_slice(&bytes).unwrap();
3938 let err = store.unexpected("append_events_idempotent", parsed);
3939 assert!(
3940 matches!(
3941 err,
3942 StorageError::WriterTaskRequestFailed {
3943 request_state: S::TransactionRolledBack,
3944 ..
3945 }
3946 ),
3947 "proven rollback must not reconstruct as a terminal writer: {err:?}"
3948 );
3949 }
3950
3951 #[cfg(unix)]
3952 #[test]
3953 fn direct_backend_hardens_preexisting_db_and_sidecars() {
3954 use std::os::unix::fs::PermissionsExt;
3955 let dir = tempfile::tempdir().unwrap();
3956 let _registry_guard = TestRegistryGuard::new(dir.path());
3957 let db = dir.path().join("pre-existing.events.db");
3958 let wal = dir.path().join("pre-existing.events.db-wal");
3959 std::fs::write(&db, b"").unwrap();
3960 std::fs::write(&wal, b"").unwrap();
3961 for path in [&db, &wal] {
3962 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o644)).unwrap();
3963 }
3964 assert_eq!(
3966 std::fs::metadata(&db).unwrap().permissions().mode() & 0o777,
3967 0o644
3968 );
3969 direct_backend_for(&db).expect("writable open succeeds");
3970 for path in [&db, &wal] {
3971 assert_eq!(
3972 std::fs::metadata(path).unwrap().permissions().mode() & 0o777,
3973 0o600,
3974 "pre-existing {} must be tightened to owner-only",
3975 path.display()
3976 );
3977 }
3978 }
3979
3980 #[cfg(unix)]
3981 #[test]
3982 fn namespace_store_cache_is_bounded_and_trim_normalized() {
3983 let backend = StorageBackend::memory().unwrap();
3984 let stores: NamespaceStores =
3985 Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
3986 namespace_store_with_cap(&backend, &stores, "alpha", 1).expect("first store");
3989 namespace_store_with_cap(&backend, &stores, "beta", 1).expect("second store evicts");
3990 {
3991 let map = stores.lock().unwrap();
3992 assert_eq!(map.len(), 1, "cache must not grow past its cap");
3993 assert!(
3994 map.contains_key("beta"),
3995 "the newest namespace must be admitted at the cap"
3996 );
3997 }
3998 namespace_store_with_cap(&backend, &stores, "beta", 1).expect("cached beta");
4001 namespace_store_with_cap(&backend, &stores, " beta ", 1).expect("trimmed spelling");
4002 {
4003 let map = stores.lock().unwrap();
4004 assert_eq!(
4005 map.len(),
4006 1,
4007 "spellings of one namespace must share one entry"
4008 );
4009 assert!(map.contains_key("beta"), "trimmed key is the cache key");
4010 }
4011 }
4012
4013 #[cfg(unix)]
4014 #[tokio::test]
4015 async fn frame_budget_admits_before_allocating_and_releases_after() {
4016 let (mut client, mut server) = UnixStream::pair().expect("socketpair");
4017 let budget = Arc::new(tokio::sync::Semaphore::new(1024));
4018 let payload = vec![7u8; 100];
4019 write_frame(&mut client, &payload).await.expect("write");
4020 let (bytes, permit) = read_frame_budgeted(&mut server, &budget)
4021 .await
4022 .expect("budgeted read");
4023 assert_eq!(bytes, payload);
4024 assert_eq!(budget.available_permits(), 1024 - 100);
4026 drop(permit);
4028 assert_eq!(budget.available_permits(), 1024);
4029 let mut oversized = Vec::from(((crate::daemon::MAX_FRAME_BYTES + 1) as u32).to_be_bytes());
4032 oversized.extend_from_slice(&[0u8; 8]);
4033 use tokio::io::AsyncWriteExt;
4034 client.write_all(&oversized).await.expect("raw prefix");
4035 let err = read_frame_budgeted(&mut server, &budget)
4036 .await
4037 .expect_err("oversized declaration must refuse");
4038 assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
4039 assert_eq!(
4040 budget.available_permits(),
4041 1024,
4042 "refusal must not consume budget"
4043 );
4044 }
4045
4046 #[cfg(unix)]
4047 #[tokio::test]
4048 async fn dispatch_rejects_invalid_wire_namespace() {
4049 let backend = Arc::new(StorageBackend::memory().unwrap());
4050 let stores: NamespaceStores =
4051 Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
4052 let huge = "n".repeat(64 * 1024);
4056 let response = dispatch_events_request(
4057 EventsRequest::CountEvents {
4058 protocol_version: EVENTS_PROTOCOL_VERSION,
4059 namespace: huge,
4060 operator_read_budget_secs: Some(30),
4061 filter: Default::default(),
4062 },
4063 &backend,
4064 &stores,
4065 )
4066 .await;
4067 assert!(
4068 matches!(
4069 &response,
4070 EventsResponse::Error {
4071 retryable: false,
4072 writer_task_failure: None,
4073 sqlite_write_failure: None,
4074 read_failure: None,
4075 ..
4076 }
4077 ),
4078 "oversized namespace must be a typed refusal, got {response:?}"
4079 );
4080 assert_eq!(
4081 stores.lock().unwrap().len(),
4082 0,
4083 "a rejected namespace must never enter the cache"
4084 );
4085 let ok = dispatch_events_request(
4087 EventsRequest::CountEvents {
4088 protocol_version: EVENTS_PROTOCOL_VERSION,
4089 namespace: "local".to_string(),
4090 operator_read_budget_secs: Some(30),
4091 filter: Default::default(),
4092 },
4093 &backend,
4094 &stores,
4095 )
4096 .await;
4097 assert!(
4098 matches!(ok, EventsResponse::Count { .. }),
4099 "valid namespace must dispatch, got {ok:?}"
4100 );
4101 }
4102
4103 include!("events_split_shutdown_tests.rs");
4104 include!("events_split_readiness_tests.rs");
4105
4106 #[cfg(unix)]
4107 #[tokio::test]
4108 async fn forwarder_abandons_hung_delivery() {
4109 let dir = tempfile::tempdir().unwrap();
4113 let socket = dir.path().join("hung.sock");
4114 let listener = tokio::net::UnixListener::bind(&socket).unwrap();
4115 let _server = tokio::spawn(async move {
4117 let mut held = Vec::new();
4118 loop {
4119 if let Ok((stream, _)) = listener.accept().await {
4120 held.push(stream);
4121 }
4122 }
4123 });
4124 let client = EventsSplitClient::new_with_queue_depth_and_delivery_timeout(
4125 socket.clone(),
4126 4,
4127 Duration::from_millis(100),
4128 )
4129 .expect("client builds");
4130 let event = Event::new(
4131 "test",
4132 "noop",
4133 khive_types::EventKind::Audit,
4134 khive_types::SubstrateKind::Event,
4135 "tester",
4136 );
4137 client.enqueue("test", vec![event]);
4138 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
4139 loop {
4140 if client.metrics().dropped_batches >= 1 {
4141 break;
4142 }
4143 assert!(
4144 tokio::time::Instant::now() < deadline,
4145 "forwarder never abandoned the hung delivery: {:?}",
4146 client.metrics()
4147 );
4148 tokio::time::sleep(Duration::from_millis(20)).await;
4149 }
4150 }
4151
4152 #[cfg(unix)]
4153 #[test]
4154 fn wire_retryability_follows_the_storage_classifier() {
4155 let busy = storage_error_response(&StorageError::WriterTaskBusy { timeout_ms: 5 });
4156 assert!(
4157 matches!(
4158 busy,
4159 EventsResponse::Error {
4160 retryable: true,
4161 ..
4162 }
4163 ),
4164 "transient writer contention must stay retryable across the socket"
4165 );
4166 let terminated = storage_error_response(&StorageError::writer_task_terminated(
4169 khive_storage::WriterTaskRequestState::SideEffectsUnknown,
4170 ));
4171 assert!(
4172 matches!(
4173 terminated,
4174 EventsResponse::Error {
4175 retryable: false,
4176 ..
4177 }
4178 ),
4179 "a terminated writer must not be reported transient"
4180 );
4181 }
4182
4183 use khive_storage::event::EventAppendDisposition;
4184 use khive_types::{EventKind, SubstrateKind};
4185
4186 fn test_event(namespace: &str) -> Event {
4187 Event::new(
4188 namespace,
4189 "test.verb",
4190 EventKind::Audit,
4191 SubstrateKind::Event,
4192 "actor:test",
4193 )
4194 }
4195
4196 #[cfg(unix)]
4198 async fn boot_daemon(dir: &tempfile::TempDir) -> (PathBuf, PathBuf) {
4199 let db = dir.path().join("events.db");
4200 let socket = dir.path().join("events.sock");
4201 let (db_clone, socket_clone) = (db.clone(), socket.clone());
4202 tokio::spawn(async move {
4203 let _ = run_events_daemon(&db_clone, &socket_clone).await;
4204 });
4205 for _ in 0..100 {
4206 if UnixStream::connect(&socket).await.is_ok() {
4207 return (db, socket);
4208 }
4209 tokio::time::sleep(Duration::from_millis(20)).await;
4210 }
4211 panic!("events daemon did not come up on {}", socket.display());
4212 }
4213
4214 #[tokio::test]
4219 async fn split_store_routes_plain_to_legacy_idempotent_to_lane_and_merges_reads() {
4220 let dir = tempfile::tempdir().expect("tempdir");
4221 let _registry_guard = TestRegistryGuard::new(dir.path());
4222 let legacy_backend =
4223 direct_backend_for(&dir.path().join("legacy.db")).expect("legacy backend");
4224 let lane_backend = direct_backend_for(&dir.path().join("lane.db")).expect("lane backend");
4225 let legacy = legacy_backend
4226 .events_for_namespace("local")
4227 .expect("legacy store");
4228 let lane = lane_backend
4229 .events_for_namespace("local")
4230 .expect("lane store");
4231 let split = SplitEventStore::new(Arc::clone(&legacy), Arc::clone(&lane));
4232
4233 let plain = test_event("local");
4234 let plain_id = plain.id;
4235 split.append_event(plain).await.expect("plain append");
4236
4237 let audit = test_event("local");
4238 let audit_id = audit.id;
4239 let result = split
4240 .append_events_idempotent(vec![audit])
4241 .await
4242 .expect("idempotent append");
4243 assert_eq!(result.rows, vec![EventAppendDisposition::Inserted]);
4244
4245 assert!(legacy
4247 .get_event(plain_id)
4248 .await
4249 .expect("legacy get")
4250 .is_some());
4251 assert!(lane.get_event(plain_id).await.expect("lane get").is_none());
4252 assert!(legacy
4253 .get_event(audit_id)
4254 .await
4255 .expect("legacy get")
4256 .is_none());
4257 assert!(lane.get_event(audit_id).await.expect("lane get").is_some());
4258
4259 assert!(split
4261 .get_event(plain_id)
4262 .await
4263 .expect("split get")
4264 .is_some());
4265 assert!(split
4266 .get_event(audit_id)
4267 .await
4268 .expect("split get")
4269 .is_some());
4270 assert_eq!(
4271 split
4272 .count_events(EventFilter::default())
4273 .await
4274 .expect("split count"),
4275 2
4276 );
4277 let page = split
4278 .query_events(
4279 EventFilter::default(),
4280 PageRequest {
4281 offset: 0,
4282 limit: 10,
4283 },
4284 )
4285 .await
4286 .expect("split query");
4287 let ids: Vec<Uuid> = page.items.iter().map(|e| e.id).collect();
4288 assert!(ids.contains(&plain_id) && ids.contains(&audit_id));
4289
4290 let mut seen = Vec::new();
4293 for offset in 0..2 {
4294 let page = split
4295 .query_events(EventFilter::default(), PageRequest { offset, limit: 1 })
4296 .await
4297 .expect("windowed query");
4298 assert_eq!(page.items.len(), 1);
4299 seen.push(page.items[0].id);
4300 }
4301 seen.sort();
4302 let mut expected = vec![plain_id, audit_id];
4303 expected.sort();
4304 assert_eq!(seen, expected);
4305 }
4306
4307 #[tokio::test]
4320 async fn raw_sql_consumers_of_the_legacy_events_table_still_see_plain_appends() {
4321 use khive_storage::types::{SqlStatement, SqlValue};
4322
4323 let dir = tempfile::tempdir().expect("tempdir");
4324 let _registry_guard = TestRegistryGuard::new(dir.path());
4325 let legacy_backend =
4326 direct_backend_for(&dir.path().join("legacy-guard.db")).expect("legacy backend");
4327 let lane_backend =
4328 direct_backend_for(&dir.path().join("lane-guard.db")).expect("lane backend");
4329 let split = SplitEventStore::new(
4330 legacy_backend
4331 .events_for_namespace("local")
4332 .expect("legacy store"),
4333 lane_backend
4334 .events_for_namespace("local")
4335 .expect("lane store"),
4336 );
4337
4338 let provenance = test_event("local");
4341 let provenance_id = provenance.id;
4342 split.append_event(provenance).await.expect("plain append");
4343 let audit = test_event("local");
4344 let audit_id = audit.id;
4345 split
4346 .append_events_idempotent(vec![audit])
4347 .await
4348 .expect("idempotent append");
4349
4350 let count_by_raw_sql = |backend: Arc<StorageBackend>, id: Uuid| async move {
4351 let mut reader = backend.sql().reader().await.expect("sql reader");
4352 let rows = reader
4353 .query_all(SqlStatement {
4354 sql: "SELECT actor FROM events WHERE id = ?1".to_string(),
4355 params: vec![SqlValue::Text(id.to_string())],
4356 label: None,
4357 })
4358 .await
4359 .expect("raw events query");
4360 rows.len()
4361 };
4362
4363 assert_eq!(
4366 count_by_raw_sql(Arc::clone(&legacy_backend), provenance_id).await,
4367 1,
4368 "plain appends must stay visible to raw-SQL consumers of the legacy events table"
4369 );
4370 assert_eq!(
4373 count_by_raw_sql(Arc::clone(&legacy_backend), audit_id).await,
4374 0,
4375 "audit-lane rows must not land in the legacy events table"
4376 );
4377 assert_eq!(
4381 count_by_raw_sql(Arc::clone(&lane_backend), audit_id).await,
4382 1,
4383 "the raw query must prove it can find rows where they actually live"
4384 );
4385 }
4386
4387 #[cfg(unix)]
4390 #[tokio::test]
4391 async fn idempotent_append_round_trips_through_the_daemon() {
4392 let dir = tempfile::tempdir().expect("tempdir");
4393 let (_db, socket) = boot_daemon(&dir).await;
4394 let client = EventsSplitClient::new(socket).expect("client");
4395 let store = ForwardingEventStore::new("local", client);
4396
4397 let event = test_event("local");
4398 let id = event.id;
4399 let result = store
4400 .append_events_idempotent(vec![event.clone()])
4401 .await
4402 .expect("idempotent append over socket");
4403 assert_eq!(result.rows, vec![EventAppendDisposition::Inserted]);
4404
4405 let retry = store
4408 .append_events_idempotent(vec![event])
4409 .await
4410 .expect("idempotent retry");
4411 assert_eq!(
4412 retry.rows,
4413 vec![EventAppendDisposition::AlreadyPresentIdentical]
4414 );
4415
4416 let fetched = store.get_event(id).await.expect("get over socket");
4417 assert_eq!(fetched.map(|e| e.id), Some(id));
4418 let count = store
4419 .count_events(EventFilter::default())
4420 .await
4421 .expect("count over socket");
4422 assert_eq!(count, 1);
4423 }
4424
4425 #[cfg(unix)]
4428 #[tokio::test]
4429 async fn fire_and_forget_append_lands_in_the_daemon_store() {
4430 let dir = tempfile::tempdir().expect("tempdir");
4431 let (_db, socket) = boot_daemon(&dir).await;
4432 let client = EventsSplitClient::new(socket).expect("client");
4433 let store = ForwardingEventStore::new("local", Arc::clone(&client));
4434
4435 store
4436 .append_event(test_event("local"))
4437 .await
4438 .expect("append_event is fire-and-forget");
4439
4440 let (count, metrics) = tokio::time::timeout(Duration::from_secs(30), async {
4444 loop {
4445 let count = store
4446 .count_events(EventFilter::default())
4447 .await
4448 .expect("count over socket");
4449 let metrics = client.metrics();
4450 if (count == 1 && metrics.forwarded_events >= 1) || metrics.dropped_events > 0 {
4451 break (count, metrics);
4452 }
4453 tokio::time::sleep(Duration::from_millis(20)).await;
4454 }
4455 })
4456 .await
4457 .expect("forwarder must deliver or report a drop within 30 seconds");
4458 assert_eq!(
4459 metrics.dropped_events, 0,
4460 "forwarder dropped an event before it reached the daemon store"
4461 );
4462 assert_eq!(count, 1, "forwarded event must land in the daemon store");
4463 assert!(metrics.forwarded_events >= 1, "delivery must be counted");
4464 }
4465
4466 #[cfg(unix)]
4470 #[tokio::test]
4471 async fn queue_overflow_drops_and_counts_but_never_errors() {
4472 let dir = tempfile::tempdir().expect("tempdir");
4473 let dead_socket = dir.path().join("nobody-home.sock");
4474 let client = EventsSplitClient::new_with_queue_depth(dead_socket, 2).expect("client");
4475 let store = ForwardingEventStore::new("local", Arc::clone(&client));
4476
4477 for _ in 0..20 {
4478 store
4479 .append_event(test_event("local"))
4480 .await
4481 .expect("append_event must not error under overflow");
4482 }
4483 let metrics = client.metrics();
4484 assert!(
4485 metrics.dropped_events > 0,
4486 "overflow must register in the drop counter, got {metrics:?}"
4487 );
4488 }
4489
4490 #[cfg(unix)]
4494 #[tokio::test]
4495 async fn dead_socket_reads_fail_typed_and_preflight_stays_local() {
4496 let dir = tempfile::tempdir().expect("tempdir");
4497 let dead_socket = dir.path().join("nobody-home.sock");
4498 let client = EventsSplitClient::new(dead_socket).expect("client");
4499 let store = ForwardingEventStore::new("local", client);
4500
4501 let error = store
4502 .get_event(Uuid::new_v4())
4503 .await
4504 .expect_err("read against a dead socket must fail");
4505 assert!(
4506 matches!(error, StorageError::Pool { .. }),
4507 "expected the typed unreachable error, got {error:?}"
4508 );
4509
4510 assert!(store.supports_idempotent_audit_batch());
4513 store
4514 .preflight_event(&test_event("local"))
4515 .expect("offline preflight validates a well-formed event");
4516 }
4517
4518 #[cfg(unix)]
4521 #[tokio::test]
4522 async fn connected_events_peer_closing_without_response_is_not_unreachable() {
4523 let dir = tempfile::tempdir().unwrap();
4524 let socket = dir.path().join("closes-after-request.sock");
4525 let listener = UnixListener::bind(&socket).unwrap();
4526 let server = tokio::spawn(async move {
4527 let (mut stream, _) = listener.accept().await.unwrap();
4528 read_frame(&mut stream)
4529 .await
4530 .expect("complete request arrives");
4531 });
4533 let client = EventsSplitClient::new(socket).unwrap();
4534 let store = ForwardingEventStore::new("local", client);
4535 let error = store.get_event(Uuid::new_v4()).await.unwrap_err();
4536 assert!(
4537 matches!(error, StorageError::Serialization { .. }),
4538 "a post-connect response failure must not be Pool/unreachable: {error:?}"
4539 );
4540 server.await.unwrap();
4541 }
4542
4543 #[cfg(unix)]
4547 #[tokio::test]
4548 async fn query_page_over_frame_cap_returns_typed_size_refusal() {
4549 let dir = tempfile::tempdir().unwrap();
4550 let socket = dir.path().join("large-query.sock");
4551 let backend = Arc::new(StorageBackend::memory().unwrap());
4552 let event_store = backend.events_for_namespace("local").unwrap();
4553 let events: Vec<_> = (0..MAX_QUERY_EVENTS_PAGE_ROWS)
4554 .map(|_| {
4555 test_event("local").with_payload(serde_json::json!({"data": "x".repeat(3 * 1024)}))
4556 })
4557 .collect();
4558 event_store
4559 .append_events(events)
4560 .await
4561 .expect("seed large page");
4562
4563 let listener = UnixListener::bind(&socket).unwrap();
4564 let stores: NamespaceStores =
4565 Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
4566 let server = tokio::spawn(async move {
4567 let (stream, _) = listener.accept().await.unwrap();
4568 serve_events_conn(
4569 stream,
4570 backend,
4571 stores,
4572 Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_REQUEST_BYTES)),
4573 )
4574 .await;
4575 });
4576 let client = EventsSplitClient::new(socket).unwrap();
4577 let store = ForwardingEventStore::new("local", client);
4578 let error = store
4579 .query_events(
4580 EventFilter::default(),
4581 PageRequest {
4582 offset: 0,
4583 limit: MAX_QUERY_EVENTS_PAGE_ROWS,
4584 },
4585 )
4586 .await
4587 .expect_err("the full page cannot fit one frame");
4588 assert!(
4589 matches!(error, StorageError::InvalidInput { .. }),
4590 "expected non-retryable frame-size refusal, got {error:?}"
4591 );
4592 assert!(
4593 error.to_string().contains("response_frame_size_limit"),
4594 "{error}"
4595 );
4596 assert!(
4597 error.to_string().contains("request a narrower page"),
4598 "{error}"
4599 );
4600 server.abort();
4601 }
4602
4603 #[cfg(unix)]
4607 #[tokio::test]
4608 async fn forwarding_queue_enforces_serialized_byte_and_frame_limits_with_drop_metrics() {
4609 let dir = tempfile::tempdir().unwrap();
4610 let socket = dir.path().join("stalled-forwarder.sock");
4611 let listener = UnixListener::bind(&socket).unwrap();
4612 let server = tokio::spawn(async move {
4613 let (stream, _) = listener.accept().await.unwrap();
4614 let _hold_open = stream;
4615 std::future::pending::<()>().await;
4616 });
4617 let event = test_event("local").with_payload(serde_json::json!({"data": "x".repeat(4096)}));
4618 let first = vec![event.clone(), event.clone()];
4619 let request_bytes = serde_json::to_vec(&EventsRequest::AppendEvents {
4620 protocol_version: EVENTS_PROTOCOL_VERSION,
4621 namespace: "local".into(),
4622 events: first.clone(),
4623 })
4624 .unwrap()
4625 .len();
4626 let byte_budget = request_bytes + 64;
4627 let client = EventsSplitClient::new_with_limits_and_delivery_timeout(
4628 socket,
4629 8,
4630 byte_budget,
4631 Duration::from_secs(30),
4632 )
4633 .unwrap();
4634 let store = ForwardingEventStore::new("local", Arc::clone(&client));
4635 store.append_events(first.clone()).await.unwrap();
4636 assert_eq!(client.metrics().queued_bytes, request_bytes);
4637
4638 store.append_events(first).await.unwrap();
4639 let metrics = client.metrics();
4640 assert_eq!(metrics.dropped_batches, 1);
4641 assert_eq!(metrics.dropped_events, 2);
4642 assert_eq!(metrics.queued_bytes, request_bytes);
4643 assert!(metrics.queued_bytes <= byte_budget);
4644
4645 let oversized = test_event("local")
4648 .with_payload(serde_json::json!({"data": "x".repeat(crate::daemon::MAX_FRAME_BYTES)}));
4649 store.append_event(oversized).await.unwrap();
4650 let metrics = client.metrics();
4651 assert_eq!(metrics.dropped_batches, 2);
4652 assert_eq!(metrics.dropped_events, 3);
4653 assert_eq!(metrics.queued_bytes, request_bytes);
4654 server.abort();
4655 }
4656
4657 #[cfg(unix)]
4660 #[tokio::test]
4661 async fn protocol_version_skew_is_a_typed_refusal() {
4662 let dir = tempfile::tempdir().expect("tempdir");
4663 let (_db, socket) = boot_daemon(&dir).await;
4664
4665 let mut stream = UnixStream::connect(&socket).await.expect("connect");
4666 let request = EventsRequest::CountEvents {
4667 protocol_version: EVENTS_PROTOCOL_VERSION + 1,
4668 namespace: "local".into(),
4669 operator_read_budget_secs: Some(30),
4670 filter: EventFilter::default(),
4671 };
4672 let payload = serde_json::to_vec(&request).expect("serialize");
4673 write_frame(&mut stream, &payload).await.expect("write");
4674 let bytes = read_frame(&mut stream).await.expect("read");
4675 let response: EventsResponse = serde_json::from_slice(&bytes).expect("parse");
4676 match response {
4677 EventsResponse::Error {
4678 message, retryable, ..
4679 } => {
4680 assert!(!retryable, "version skew is not retryable");
4681 assert!(message.contains("protocol version"), "message: {message}");
4682 }
4683 other => panic!("expected a typed refusal, got {other:?}"),
4684 }
4685 }
4686
4687 #[cfg(unix)]
4691 #[test]
4692 fn events_daemon_guard_is_exclusive_then_reusable() {
4693 let dir = tempfile::tempdir().expect("tempdir");
4694 let socket = dir.path().join("events.sock");
4695 let first = match acquire_events_daemon_guard_outcome(&socket) {
4696 EventsDaemonGuardAcquisition::Held(guard) => guard,
4697 other => panic!("first acquire must succeed: {other:?}"),
4698 };
4699 let second = acquire_events_daemon_guard_outcome(&socket);
4700 assert!(
4701 matches!(second, EventsDaemonGuardAcquisition::Contended),
4702 "second acquire must report contention while the first guard is held: {second:?}"
4703 );
4704 drop(first);
4705 const MAX_ATTEMPTS: usize = 100;
4709 for attempt in 1..=MAX_ATTEMPTS {
4710 match acquire_events_daemon_guard_outcome(&socket) {
4711 EventsDaemonGuardAcquisition::Held(_) => return,
4712 EventsDaemonGuardAcquisition::Contended if attempt < MAX_ATTEMPTS => {
4713 std::thread::sleep(Duration::from_millis(10));
4714 }
4715 other => panic!(
4716 "acquire must succeed after release within {MAX_ATTEMPTS} attempts; \
4717 attempt {attempt}: {other:?}"
4718 ),
4719 }
4720 }
4721 unreachable!("the final acquisition attempt returns or reports its refusal");
4722 }
4723
4724 #[cfg(unix)]
4725 #[test]
4726 fn events_daemon_guard_open_failure_is_not_contention() {
4727 let dir = tempfile::tempdir().expect("tempdir");
4728 let clean = dir.path().join("clean.sock");
4729 assert!(matches!(
4730 acquire_events_daemon_guard_outcome(&clean),
4731 EventsDaemonGuardAcquisition::Held(_)
4732 ));
4733
4734 assert!(
4735 try_acquire_events_daemon_guard(&clean).is_some(),
4736 "the public Option entrance must preserve a successful acquisition"
4737 );
4738
4739 let socket = dir.path().join("blocked.sock");
4742 let lock_path = socket.with_extension("lock");
4743 std::fs::create_dir(&lock_path).expect("create directory at lock entry");
4744 let outcome = acquire_events_daemon_guard_outcome(&socket);
4745 assert!(
4746 matches!(outcome, EventsDaemonGuardAcquisition::OpenFailed(_)),
4747 "an actual lock-file open failure must not report contention: {outcome:?}"
4748 );
4749 assert!(
4750 try_acquire_events_daemon_guard(&socket).is_none(),
4751 "the public Option entrance must refuse an actual open failure"
4752 );
4753 assert!(
4754 lock_path.is_dir(),
4755 "the refused entry must remain a directory"
4756 );
4757 }
4758
4759 #[cfg(unix)]
4766 #[tokio::test]
4767 async fn read_only_runtime_never_creates_an_events_db() {
4768 use crate::{KhiveRuntime, Namespace, RuntimeConfig};
4769
4770 let dir = tempfile::tempdir().expect("tempdir");
4771 let _registry_guard = TestRegistryGuard::new(dir.path());
4772 let main_db = dir.path().join("main.db");
4773 drop(
4775 KhiveRuntime::new_for_test(RuntimeConfig {
4776 db_path: Some(main_db.clone()),
4777 ..RuntimeConfig::no_embeddings()
4778 })
4779 .expect("create main db"),
4780 );
4781 let events_db = events_db_path_beside(&main_db);
4782 assert!(!events_db.exists(), "precondition: no events db yet");
4783
4784 khive_storage::test_support::freeze_snapshot_sidecars(&main_db);
4787
4788 let split_config = |db: PathBuf| RuntimeConfig {
4789 db_path: Some(main_db.clone()),
4790 events_split: Some(EventsSplitConfig {
4791 db_path: db,
4792 socket_path: None,
4793 }),
4794 ..RuntimeConfig::no_embeddings()
4795 };
4796
4797 let ro = KhiveRuntime::new_readonly_for_test(split_config(events_db.clone()))
4800 .expect("read-only runtime");
4801 let token = ro.authorize(Namespace::local()).expect("token");
4802 let store = ro.events(&token).expect("events store");
4803 let count = store
4804 .count_events(EventFilter::default())
4805 .await
4806 .expect("count through legacy-only plane");
4807 assert_eq!(count, 0);
4808 assert!(
4809 !events_db.exists(),
4810 "a read-only runtime must not mint the events database"
4811 );
4812
4813 {
4820 let lane_backend = StorageBackend::sqlite_for_test(&events_db).expect("writable lane");
4821 lane_backend
4822 .events_for_namespace("local")
4823 .expect("lane store")
4824 .append_event(test_event("local"))
4825 .await
4826 .expect("seed lane row");
4827 }
4828 khive_storage::test_support::freeze_snapshot_sidecars(&events_db);
4829 let store = ro.events(&token).expect("events store with lane present");
4830 let count = store
4831 .count_events(EventFilter::default())
4832 .await
4833 .expect("merged count");
4834 assert_eq!(count, 1, "the pre-existing lane row must merge into reads");
4835 }
4836
4837 #[tokio::test]
4840 async fn direct_mode_appends_and_reads_without_a_daemon() {
4841 let dir = tempfile::tempdir().expect("tempdir");
4842 let _registry_guard = TestRegistryGuard::new(dir.path());
4843 let db = dir.path().join("events.db");
4844 let backend = direct_backend_for(&db).expect("direct backend");
4845 let store = backend.events_for_namespace("local").expect("store");
4846 store
4847 .append_event(test_event("local"))
4848 .await
4849 .expect("direct append");
4850 let count = store
4851 .count_events(EventFilter::default())
4852 .await
4853 .expect("count");
4854 assert_eq!(count, 1);
4855 }
4856}