Skip to main content

khive_runtime/
events_split.rs

1//! Events-daemon split (ADR-170): the audit lane leaves the domain store.
2//!
3//! The ADR-133 idempotent audit batch is the measured bulk of event write
4//! volume, and in a single-store deployment its rows queue on the same SQLite
5//! writer lane as domain mutations. This module moves that lane into a
6//! dedicated events daemon that owns the events database, reachable over a
7//! Unix socket with the same length-prefixed framing and peer-uid admission the
8//! main daemon socket uses. Plain event appends stay on the domain store —
9//! the legacy `events` table has raw-SQL consumers (schedule provenance, kg
10//! projection guards, graph-query substrate unions) whose correctness
11//! depends on finding those rows there.
12//!
13//! Cooperating pieces:
14//!
15//! - [`run_events_daemon`] — the server loop the `events-daemon` subcommand
16//!   runs: binds the events socket, owns the only resident writer of the
17//!   events database, and serves append/read requests through the ordinary
18//!   `SqlEventStore`.
19//! - [`EventsSplitClient`] — one per domain process. Plain appends ride a
20//!   bounded in-memory queue drained by a background forwarder
21//!   (fire-and-forget; overflow or a dead daemon drops the batch, counts it,
22//!   and logs — the loss-tolerant durability class made concrete; unused by
23//!   the default routing until telemetry producers opt in). Idempotent
24//!   audit-batch appends and reads are synchronous framed round-trips with
25//!   bounded timeouts, because their callers are background flushers or query
26//!   paths, never the dispatch hot path.
27//! - [`ForwardingEventStore`] — the lane-side [`EventStore`] over the socket.
28//!   Preflight validation delegates to an in-memory `SqlEventStore`, so the
29//!   ADR-133 audit-batch seam keeps its pre-enqueue shape check without any
30//!   I/O on the dispatch path.
31//! - [`SplitEventStore`] — the per-namespace handle
32//!   [`crate::runtime::KhiveRuntime::events`] returns when the split is
33//!   configured: routes the idempotent lane to the events store, plain
34//!   appends to the legacy store, and merges reads across both.
35//!
36//! Domain availability never depends on events-daemon liveness: every failure
37//! path here degrades (drop + count + log, or a typed storage error for the
38//! synchronous lanes) instead of blocking the caller.
39
40use std::path::{Path, PathBuf};
41#[cfg(unix)]
42use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
43use std::sync::Arc;
44#[cfg(unix)]
45use std::time::Duration;
46
47use async_trait::async_trait;
48use khive_db::StorageBackend;
49use khive_storage::event::IdempotentEventBatchResult;
50use khive_storage::{
51    BatchWriteSummary, Event, EventFilter, EventStore, Page, PageRequest, StorageError,
52    StorageResult,
53};
54use serde::{Deserialize, Serialize};
55
56#[cfg(unix)]
57use tokio::net::{UnixListener, UnixStream};
58use uuid::Uuid;
59
60#[cfg(unix)]
61use crate::daemon::{read_frame, write_frame};
62
63/// Bump whenever the request or response frame shape changes incompatibly.
64/// The server rejects frames whose version it does not speak, so a skewed
65/// client gets a typed refusal instead of a deserialization panic.
66///
67/// Version 2 is the first version any release ships: version 1 carried a
68/// `side_effects_unknown` error field that was replaced by the
69/// `writer_task_state` carrier before this module reached any released ref,
70/// so v1 speakers existed only on unreleased development heads. The bump
71/// exists so that even such a process gets the version refusal above rather
72/// than having its retryable writer states silently mapped to terminal
73/// `InvalidInput`.
74pub const EVENTS_PROTOCOL_VERSION: u32 = 2;
75
76/// Default bound on the fire-and-forget append queue, in batches. The loss
77/// window on overflow is this depth times the batch size in flight; the value
78/// is deliberately generous because entries are pointers, not rows.
79#[cfg(unix)]
80pub const DEFAULT_APPEND_QUEUE_BATCHES: usize = 4096;
81
82/// Timeout for one synchronous round-trip (idempotent appends, reads).
83/// Covers the WHOLE attempt — connect, peer verification, write, read —
84/// because each round-trip owns its connection (no shared lock to wait on
85/// outside the clock).
86#[cfg(unix)]
87const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
88
89/// Reconnect backoff for the background forwarder after a failed connect.
90#[cfg(unix)]
91const FORWARDER_BACKOFF: Duration = Duration::from_secs(2);
92
93/// Cap on concurrently served daemon connections. Admission control, not a
94/// correctness bound: every permitted-uid process on the machine shares the
95/// daemon, and without a cap each accepted connection holds a task and a file
96/// descriptor for as long as the peer keeps the socket open — a stalled or
97/// hostile same-uid peer could exhaust both. Excess connects are dropped
98/// (peer sees EOF) and retried by the clients' own reconnect paths.
99#[cfg(unix)]
100const MAX_EVENTS_CONNECTIONS: usize = 128;
101
102/// Per-frame I/O deadline on a served connection: the longest the daemon
103/// waits for one request frame to finish arriving, or one response frame to
104/// finish sending. A peer that sends a partial frame and stalls is closed
105/// instead of holding its task and descriptor forever. Idle well-behaved
106/// clients are closed too — that is fine by construction: round-trips open a
107/// fresh connection per call, and the forwarder's delivery path retries once
108/// on a stale connection.
109#[cfg(unix)]
110const CONN_IO_TIMEOUT: Duration = Duration::from_secs(60);
111
112/// Aggregate cap on request-frame bytes buffered across all served
113/// connections at once. The per-frame cap bounds one buffer and the
114/// connection cap bounds the task count, but their product is the real
115/// allocation exposure (128 connections × 8 MiB declared frames ≈ 1 GiB).
116/// Body buffers are admitted against this shared byte budget before they are
117/// allocated; a connection whose frame cannot be admitted waits inside its
118/// own I/O deadline, so exhaustion degrades into per-connection timeouts,
119/// never into daemon memory growth. Must be at least `MAX_FRAME_BYTES`, or a
120/// maximum-size frame could never be admitted.
121#[cfg(unix)]
122const MAX_INFLIGHT_REQUEST_BYTES: usize = 64 * 1024 * 1024;
123
124/// Cap on the per-namespace store cache. Stores are cheap pool handles, but
125/// the namespace string is client-supplied, so an unbounded map is
126/// attacker-controlled memory growth. Beyond the cap, stores are built
127/// per-request instead of cached — slower, never wrong.
128#[cfg(unix)]
129const MAX_CACHED_NAMESPACE_STORES: usize = 1024;
130
131/// Symlink-chain hop ceiling for events-path resolution. Matches the bound
132/// the daemon's socket-path traversal guard enforces, which is itself the
133/// kernel's total-links ceiling on Linux (40): both resolvers must agree,
134/// or a chain one accepts derives a different identity than the other.
135const EVENTS_SYMLINK_HOP_BOUND: u32 = 40;
136
137/// Cap on `QueryEvents` page size, in rows. The wire `PageRequest.limit` is
138/// client-supplied `u32`, and the daemon materializes the full page as a
139/// `Vec<Event>` and serializes it into one response frame — so an unbounded
140/// limit is attacker-controlled memory and serialization work in a process
141/// that lives for months. Over-cap requests get a typed refusal naming the
142/// cap, never a silently clamped page: a caller that asked for more rows
143/// than it got would otherwise read the short page as the end of the data.
144/// The split client's merged read requests a prefix of `offset + limit`
145/// rows, so this cap also bounds the deep-offset window a socket client can
146/// demand in one request.
147///
148/// Public (and defined on every platform) because in-tree consumers that
149/// read events through a possibly-split store must size their page requests
150/// under it — a deep read is a `before`-cursor walk at `offset: 0` in pages
151/// of at most this many rows, never one wide page.
152pub const MAX_QUERY_EVENTS_PAGE_ROWS: u32 = 4096;
153
154/// Default events database file, beside the main database file.
155///
156/// The name is derived from the main database's full file name (`khive.db` →
157/// `khive.db.events.db`), never from its stem and never a fixed name in the
158/// parent directory: two independent databases that happen to share a
159/// directory (`a.db`, `b.db`) — or a stem (`a.db`, `a.sqlite`) — must each
160/// get their own event plane, not silently share one. The whole path is
161/// canonicalized when the database file exists — final-component symlink
162/// aliases of one database must derive the same sidecar and socket as the
163/// target spelling, because backend identity already treats those aliases as
164/// one database. When the file does not exist yet, the parent directory alone
165/// is canonicalized when it resolves, so path aliases of a fresh database
166/// (relative spellings, symlinked directories) still map to one sidecar
167/// instead of minting a distinct events database per spelling.
168pub fn events_db_path_beside(main_db: &Path) -> PathBuf {
169    // When the database file does not exist yet, the path can still be a
170    // symlink — a dangling alias whose target the first open will create.
171    // `canonicalize` refuses dangling links, so resolve final-component
172    // links by hand on that arm: alias-first and target-first cold starts
173    // must derive the same sidecar, or the first process to open each
174    // spelling writes audit records the other never reads.
175    let resolved = std::fs::canonicalize(main_db)
176        .unwrap_or_else(|_| resolve_dangling_final_component(main_db));
177    let mut name = resolved
178        .file_name()
179        .map(std::ffi::OsStr::to_os_string)
180        .unwrap_or_else(|| std::ffi::OsString::from("khive.db"));
181    name.push(".events.db");
182    let path = match resolved.parent().filter(|dir| !dir.as_os_str().is_empty()) {
183        Some(dir) => std::fs::canonicalize(dir)
184            .unwrap_or_else(|_| dir.to_path_buf())
185            .join(&name),
186        None => PathBuf::from(name),
187    };
188    absolutize(&path)
189}
190
191/// Follow final-component symlinks whose eventual target need not exist.
192/// The identity being recovered is the target's NAME — the file the first
193/// open through the alias will actually create — so a chain of links is
194/// walked (relative targets anchored at each link's parent) up to the same
195/// bound kernels use; a cycle or over-long chain stops at the last spelling,
196/// which `open()` will refuse anyway.
197fn resolve_dangling_final_component(path: &Path) -> PathBuf {
198    let mut current = path.to_path_buf();
199    // Same hop bound as the daemon's socket-path traversal guard (and the
200    // kernel's own total-links ceiling on Linux, 40): a chain the kernel
201    // would resolve must derive the target's sidecar, or a 33-hop alias
202    // opens fine yet splits its audit records from the target spelling.
203    for _ in 0..EVENTS_SYMLINK_HOP_BOUND {
204        match std::fs::read_link(&current) {
205            Ok(target) => {
206                current = if target.is_absolute() {
207                    target
208                } else {
209                    match current.parent().filter(|dir| !dir.as_os_str().is_empty()) {
210                        Some(dir) => dir.join(target),
211                        None => target,
212                    }
213                };
214            }
215            Err(_) => break,
216        }
217    }
218    current
219}
220
221/// Anchor a relative path to the current working directory. A bare relative
222/// spelling (`khive.db`) yields `Some("")` from `Path::parent`, and every
223/// consumer downstream — lock-file parenting, socket-directory trust
224/// validation, the daemon spawn contract — needs a real directory to stat,
225/// not an empty string. Absolute paths pass through untouched.
226fn absolutize(path: &Path) -> PathBuf {
227    if path.is_absolute() {
228        return path.to_path_buf();
229    }
230    std::env::current_dir()
231        .map(|cwd| cwd.join(path))
232        .unwrap_or_else(|_| path.to_path_buf())
233}
234
235/// Events daemon socket path, beside the events database it serves.
236///
237/// Derived from the events db path (not a process-global location, not a
238/// fixed name in the parent directory) so every events database gets its own
239/// daemon and socket: a shared socket would route events from any second
240/// database (another seat, a test tempdir) to whichever daemon happens to
241/// own it, persisting them beside the wrong main store. `khive.db.events.db`
242/// yields `khive.db.events.sock`; the daemon's advisory lock is the same
243/// path with a `.lock` extension.
244pub fn events_socket_path_beside(events_db: &Path) -> PathBuf {
245    events_db.with_extension("sock")
246}
247
248/// How a runtime reaches event storage when the split is configured.
249#[derive(Debug, Clone)]
250pub struct EventsSplitConfig {
251    /// The events database file. The events daemon is its only writer in
252    /// daemon deployments; embedded mode writes it directly.
253    pub db_path: PathBuf,
254    /// `Some(socket)` = forward appends to the events daemon at this socket
255    /// (daemon deployments). `None` = embedded mode: open `db_path` directly
256    /// in-process (one-shot CLI, tests).
257    pub socket_path: Option<PathBuf>,
258}
259
260// ---------------------------------------------------------------------------
261// Process-global handles
262// ---------------------------------------------------------------------------
263//
264// One events-split client (or one direct backend) per process, whatever the
265// number of runtimes and clones — the same shape as the daemon module's other
266// process-global state. Keyed by path so tests exercising two distinct paths
267// in one process stay isolated.
268
269#[cfg(unix)]
270type ClientMap = std::collections::HashMap<PathBuf, Arc<EventsSplitClient>>;
271/// Keyed by (path, read_only): a read-only open and a writable open of the
272/// same file are different pools with different guarantees and must never be
273/// handed out interchangeably.
274type BackendMap = std::collections::HashMap<(PathBuf, bool), Arc<StorageBackend>>;
275
276#[cfg(unix)]
277fn client_registry() -> &'static std::sync::Mutex<ClientMap> {
278    static REGISTRY: std::sync::OnceLock<std::sync::Mutex<ClientMap>> = std::sync::OnceLock::new();
279    REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
280}
281
282fn direct_backend_registry() -> &'static std::sync::Mutex<BackendMap> {
283    static REGISTRY: std::sync::OnceLock<std::sync::Mutex<BackendMap>> = std::sync::OnceLock::new();
284    REGISTRY.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
285}
286
287/// The process-wide client for `socket_path`, created (and its forwarder
288/// spawned) on first use. Requires a tokio runtime context on first call.
289#[cfg(unix)]
290pub fn client_for(socket_path: &Path) -> crate::error::RuntimeResult<Arc<EventsSplitClient>> {
291    let mut registry = client_registry()
292        .lock()
293        .unwrap_or_else(std::sync::PoisonError::into_inner);
294    if let Some(existing) = registry.get(socket_path) {
295        return Ok(Arc::clone(existing));
296    }
297    let client = EventsSplitClient::new(socket_path.to_path_buf())?;
298    registry.insert(socket_path.to_path_buf(), Arc::clone(&client));
299    Ok(client)
300}
301
302/// The process-wide direct (embedded-mode) backend for `db_path`, opened
303/// read-write on first use.
304pub fn direct_backend_for(db_path: &Path) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
305    direct_backend(db_path, false)
306}
307
308/// The process-wide direct backend for `db_path`, opened READ-ONLY on first
309/// use. The file must already exist — this constructor never creates or
310/// schema-initializes an events database, which is what a read-only runtime's
311/// no-DB-creation contract requires of its event lane.
312pub fn direct_backend_read_only_for(
313    db_path: &Path,
314) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
315    direct_backend(db_path, true)
316}
317
318fn direct_backend(
319    db_path: &Path,
320    read_only: bool,
321) -> crate::error::RuntimeResult<Arc<StorageBackend>> {
322    let mut registry = direct_backend_registry()
323        .lock()
324        .unwrap_or_else(std::sync::PoisonError::into_inner);
325    let key = (db_path.to_path_buf(), read_only);
326    if let Some(existing) = registry.get(&key) {
327        return Ok(Arc::clone(existing));
328    }
329    refuse_events_db_symlinks(db_path)
330        .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
331    #[cfg(unix)]
332    if !read_only {
333        if let Some(parent) = db_path.parent().filter(|p| !p.as_os_str().is_empty()) {
334            let _ = std::fs::create_dir_all(parent);
335        }
336    }
337    // Directory trust comes after creation (a fresh parent in a trusted
338    // ancestor must be statable) and before any open: SQLite's open is
339    // path-based, so the only defense against a component swapped between
340    // validation and open is that no untrusted local user can write the
341    // directories the path traverses.
342    #[cfg(unix)]
343    ensure_events_db_parent_trusted(db_path)
344        .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
345    // Embedded writable mode holds the events database to the daemon's own
346    // contract: owner-only from the first byte, never at the process umask,
347    // and — because event rows carry the same audit payloads either way — a
348    // pre-existing database or -wal/-shm sidecar is tightened to 0600 too,
349    // fail-closed. A create race just means SQLite finds the file present.
350    #[cfg(unix)]
351    if !read_only {
352        use std::os::unix::fs::OpenOptionsExt;
353        let _ = std::fs::OpenOptions::new()
354            .write(true)
355            .create_new(true)
356            .mode(0o600)
357            .open(db_path);
358        harden_events_db_sidecars(db_path)
359            .map_err(|e| crate::error::RuntimeError::Internal(e.to_string()))?;
360    }
361    let backend = Arc::new(if read_only {
362        StorageBackend::sqlite_read_only(db_path)?
363    } else {
364        StorageBackend::sqlite(db_path)?
365    });
366    registry.insert(key, Arc::clone(&backend));
367    Ok(backend)
368}
369
370/// Forwarding metrics for the process-wide client at `socket_path`, if one
371/// exists. `None` means the split never initialized in this process.
372#[cfg(unix)]
373pub fn forwarding_metrics(socket_path: &Path) -> Option<EventsForwardingMetrics> {
374    let registry = client_registry()
375        .lock()
376        .unwrap_or_else(std::sync::PoisonError::into_inner);
377    registry.get(socket_path).map(|client| client.metrics())
378}
379
380// ---------------------------------------------------------------------------
381// Wire protocol
382// ---------------------------------------------------------------------------
383
384/// Request frame sent from a domain process to the events daemon.
385#[derive(Debug, Serialize, Deserialize)]
386#[serde(tag = "op", rename_all = "snake_case")]
387pub enum EventsRequest {
388    /// Fire-and-forget append lane. The daemon replies with a summary, but the
389    /// forwarder treats a failed reply as a counted drop, never an error to
390    /// the original caller.
391    AppendEvents {
392        protocol_version: u32,
393        namespace: String,
394        events: Vec<Event>,
395    },
396    /// ADR-133 audit-batch lane: real dispositions come back.
397    AppendEventsIdempotent {
398        protocol_version: u32,
399        namespace: String,
400        events: Vec<Event>,
401    },
402    GetEvent {
403        protocol_version: u32,
404        namespace: String,
405        id: Uuid,
406    },
407    QueryEvents {
408        protocol_version: u32,
409        namespace: String,
410        filter: EventFilter,
411        page: PageRequest,
412    },
413    CountEvents {
414        protocol_version: u32,
415        namespace: String,
416        filter: EventFilter,
417    },
418}
419
420impl EventsRequest {
421    fn protocol_version(&self) -> u32 {
422        match self {
423            Self::AppendEvents {
424                protocol_version, ..
425            }
426            | Self::AppendEventsIdempotent {
427                protocol_version, ..
428            }
429            | Self::GetEvent {
430                protocol_version, ..
431            }
432            | Self::QueryEvents {
433                protocol_version, ..
434            }
435            | Self::CountEvents {
436                protocol_version, ..
437            } => *protocol_version,
438        }
439    }
440
441    fn namespace(&self) -> &str {
442        match self {
443            Self::AppendEvents { namespace, .. }
444            | Self::AppendEventsIdempotent { namespace, .. }
445            | Self::GetEvent { namespace, .. }
446            | Self::QueryEvents { namespace, .. }
447            | Self::CountEvents { namespace, .. } => namespace,
448        }
449    }
450}
451
452/// Response frame from the events daemon.
453#[derive(Debug, Serialize, Deserialize)]
454#[serde(tag = "kind", rename_all = "snake_case")]
455pub enum EventsResponse {
456    Appended {
457        summary: BatchWriteSummary,
458    },
459    Idempotent {
460        result: IdempotentEventBatchResult,
461    },
462    Event {
463        event: Option<Event>,
464    },
465    Pageful {
466        page: Page<Event>,
467    },
468    Count {
469        count: u64,
470    },
471    /// Typed refusal. `retryable` distinguishes transient daemon-side
472    /// conditions from contract errors (bad frame, version skew).
473    /// `writer_task_state` is the single carrier of the ADR-133 dispositions
474    /// the retryable bit cannot express: the daemon-side writer terminated
475    /// in a state the audit-batch retry classifier keys on per-variant —
476    /// `NotStarted` and `TransactionRolledBack` are safe replays,
477    /// `SideEffectsUnknown` gates double-send decisions. Flattening any of
478    /// them into a generic refusal turns a mandated retry into a terminal
479    /// failure on the client side. `None` means the error is not a
480    /// writer-task termination at all (parse errors, version refusals,
481    /// non-writer storage errors). No second spelling exists: within one
482    /// protocol version every storage-class Error frame carries this field,
483    /// and cross-version frames never reach this mapping because
484    /// `dispatch_events_request` refuses a mismatched version outright —
485    /// a refusal that happens BEFORE any write executes, so no retryable
486    /// writer state can exist to lose. That guarantee is why
487    /// [`EVENTS_PROTOCOL_VERSION`] must be bumped with any change to this
488    /// field's shape: two shapes sharing one version would make this exact
489    /// sentence false (v2 exists because v1 briefly had a second spelling
490    /// on unreleased development heads).
491    Error {
492        message: String,
493        retryable: bool,
494        #[serde(default)]
495        writer_task_state: Option<WireWriterTaskState>,
496    },
497}
498
499/// Wire mirror of [`khive_storage::WriterTaskRequestState`]. A separate type
500/// because the storage enum is not serializable and the wire shape must stay
501/// under this module's protocol-version control, not the storage crate's.
502#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
503#[serde(rename_all = "snake_case")]
504pub enum WireWriterTaskState {
505    NotStarted,
506    TransactionRolledBack,
507    SideEffectsUnknown,
508}
509
510impl From<khive_storage::WriterTaskRequestState> for WireWriterTaskState {
511    fn from(state: khive_storage::WriterTaskRequestState) -> Self {
512        use khive_storage::WriterTaskRequestState as S;
513        match state {
514            S::NotStarted => Self::NotStarted,
515            S::TransactionRolledBack => Self::TransactionRolledBack,
516            S::SideEffectsUnknown => Self::SideEffectsUnknown,
517        }
518    }
519}
520
521impl From<WireWriterTaskState> for khive_storage::WriterTaskRequestState {
522    fn from(state: WireWriterTaskState) -> Self {
523        use khive_storage::WriterTaskRequestState as S;
524        match state {
525            WireWriterTaskState::NotStarted => S::NotStarted,
526            WireWriterTaskState::TransactionRolledBack => S::TransactionRolledBack,
527            WireWriterTaskState::SideEffectsUnknown => S::SideEffectsUnknown,
528        }
529    }
530}
531
532// ---------------------------------------------------------------------------
533// Server side
534// ---------------------------------------------------------------------------
535
536/// Advisory lock guaranteeing at most one events daemon per socket path.
537/// Held for the daemon's lifetime; a second daemon exits instead of stealing
538/// the socket path from the live one.
539#[cfg(unix)]
540pub struct EventsDaemonGuard {
541    _file: std::fs::File,
542}
543
544/// Try to become the events daemon for `socket_path`. `None` = the lock
545/// could not be safely acquired — either another events daemon holds it, or
546/// a hardening step refused (symlinked lock entry, failed chmod). Both mean
547/// the caller must not serve; callers that need the socket directory
548/// validated must run `ensure_socket_dir_is_trusted` on the parent BEFORE
549/// calling this, so no lock-path operation happens in an untrusted
550/// directory.
551#[cfg(unix)]
552pub fn try_acquire_events_daemon_guard(socket_path: &Path) -> Option<EventsDaemonGuard> {
553    let lock_path = socket_path.with_extension("lock");
554    if let Some(parent) = lock_path.parent() {
555        std::fs::create_dir_all(parent).ok()?;
556    }
557    use std::os::unix::fs::OpenOptionsExt;
558    // `O_NOFOLLOW` pins the open to the final component: a symlink planted
559    // at the lock name is refused instead of redirecting the open (and the
560    // chmod below) to an attacker-selected target.
561    let file = std::fs::OpenOptions::new()
562        .create(true)
563        .truncate(false)
564        .write(true)
565        .mode(0o600)
566        .custom_flags(libc::O_NOFOLLOW)
567        .open(&lock_path)
568        .ok()?;
569    // `mode` applies only at creation; tighten a pre-existing lock file too.
570    // Descriptor-based (`fchmod` on the handle just opened), never a second
571    // path lookup — and fail closed: with the inode pinned, a failed chmod
572    // is abnormal, and serving behind a lock file another user can open is
573    // exactly what the hardening exists to refuse.
574    file.set_permissions(
575        <std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o600),
576    )
577    .ok()?;
578    use std::os::fd::AsRawFd;
579    // SAFETY: `fd` is a live descriptor owned by `file` for the duration of
580    // the call; `flock` reads nothing else.
581    let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
582    if rc == 0 {
583        Some(EventsDaemonGuard { _file: file })
584    } else {
585        None
586    }
587}
588
589/// Refuse to serve an events database whose path — or whose `-wal`/`-shm`
590/// sidecar path — is a pre-existing symlink. These paths are derived, never
591/// user-chosen (`events_db_path_beside` canonicalizes the main database
592/// spelling first), so a link here is a planted redirect, not an alias:
593/// permission hardening and SQLite would otherwise follow it and tighten or
594/// write event rows through to whatever file the link's author chose
595/// (CWE-59). Runs before any open on both the embedded and daemon arms; a
596/// link planted after admission is bounded by the daemon's trusted-directory
597/// contract on the socket parent.
598fn refuse_events_db_symlinks(db_path: &Path) -> anyhow::Result<()> {
599    let mut targets = vec![db_path.to_path_buf()];
600    for suffix in ["-wal", "-shm"] {
601        let mut name = db_path.as_os_str().to_os_string();
602        name.push(suffix);
603        targets.push(PathBuf::from(name));
604    }
605    for path in targets {
606        match std::fs::symlink_metadata(&path) {
607            Ok(meta) if meta.file_type().is_symlink() => anyhow::bail!(
608                "refusing to serve events: {} is a symlink; the events database and its \
609                 sidecars must be regular files",
610                path.display()
611            ),
612            Ok(_) => {}
613            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
614            Err(e) => anyhow::bail!(
615                "refusing to serve events: cannot inspect {}: {e}",
616                path.display()
617            ),
618        }
619    }
620    Ok(())
621}
622
623/// Require the directory holding the events database to be trusted before
624/// anything opens a path inside it. The symlink pre-checks and the fd-pinned
625/// chmod close the races they can see, but SQLite's own open is path-based
626/// and cannot be inode-pinned from here — so the remaining defense is the
627/// same one the daemon socket uses: no untrusted local user may be able to
628/// write (or swap components of) the directory the path traverses. Delegates
629/// to the socket-path walk, which validates every ancestor the kernel will
630/// visit, with the same ownership and sticky-bit rules.
631#[cfg(unix)]
632fn ensure_events_db_parent_trusted(db_path: &Path) -> anyhow::Result<()> {
633    let parent = absolutize(db_path);
634    let parent = parent.parent().filter(|p| !p.as_os_str().is_empty());
635    match parent {
636        Some(dir) => crate::daemon::ensure_socket_dir_is_trusted(dir).map_err(|e| {
637            anyhow::anyhow!("refusing to serve events from an untrusted directory: {e}")
638        }),
639        None => Ok(()),
640    }
641}
642
643/// Create the events database file owner-only if it does not exist yet.
644/// The 0600 socket and peer-uid admission bound who can *talk to* the
645/// daemon; they bound nothing if the database file itself is readable by
646/// other local users, so the daemon creates it 0600 before SQLite ever
647/// opens it (SQLite would otherwise create it at the process umask).
648#[cfg(unix)]
649fn ensure_events_db_owner_only(db_path: &Path) -> anyhow::Result<()> {
650    use std::os::unix::fs::OpenOptionsExt;
651    refuse_events_db_symlinks(db_path)?;
652    if let Some(parent) = db_path.parent() {
653        std::fs::create_dir_all(parent)?;
654    }
655    // After creation so a fresh parent can be validated; a directory this
656    // process just created in a trusted ancestor passes by construction.
657    ensure_events_db_parent_trusted(db_path)?;
658    match std::fs::OpenOptions::new()
659        .write(true)
660        .create_new(true)
661        .mode(0o600)
662        .open(db_path)
663    {
664        // A zero-byte file is a valid empty SQLite database.
665        Ok(_created) => Ok(()),
666        Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()),
667        Err(e) => Err(anyhow::anyhow!(
668            "refusing to serve events: cannot create {} owner-only: {e}",
669            db_path.display()
670        )),
671    }
672}
673
674/// Tighten the events database and its SQLite sidecars to owner-only.
675/// Fail closed, same contract as the socket chmod: a daemon that cannot
676/// keep its database owner-only must not serve it.
677#[cfg(unix)]
678fn harden_events_db_sidecars(db_path: &Path) -> anyhow::Result<()> {
679    use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
680    let mut targets = vec![db_path.to_path_buf()];
681    for suffix in ["-wal", "-shm"] {
682        let mut name = db_path.as_os_str().to_os_string();
683        name.push(suffix);
684        targets.push(PathBuf::from(name));
685    }
686    for path in targets {
687        // Pin the inode before touching it: `O_NOFOLLOW` makes the open
688        // itself refuse a symlink at the final component, and the chmod is
689        // then issued on the returned handle (fchmod), so no path re-lookup
690        // exists between validation and the permission change for a swapped
691        // entry to exploit. A path-based lstat-then-chmod pair here would
692        // re-open the exact race it is defending against.
693        let file = match std::fs::OpenOptions::new()
694            .read(true)
695            .custom_flags(libc::O_NOFOLLOW)
696            .open(&path)
697        {
698            Ok(file) => file,
699            Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
700            Err(e) => {
701                anyhow::bail!(
702                    "refusing to serve events: cannot open {} without following symlinks: \
703                     {e}. The events database and its sidecars must be regular files.",
704                    path.display()
705                )
706            }
707        };
708        file.set_permissions(std::fs::Permissions::from_mode(0o600))
709            .map_err(|e| {
710                anyhow::anyhow!(
711                    "refusing to serve events: cannot chmod 0600 {}: {e}. The events \
712                     database and its sidecars must be owner-only.",
713                    path.display()
714                )
715            })?;
716    }
717    Ok(())
718}
719
720/// Supervise the events daemon from the main daemon process: probe the
721/// socket periodically and (re)spawn the daemon subcommand when unreachable.
722///
723/// The spawned command contract is fixed here once: the current executable
724/// re-invoked as `events-daemon --db <db> --socket <socket>` — the subcommand
725/// the kernel binary registers for [`run_events_daemon`]. The child holds the
726/// per-socket advisory lock, so a probe/spawn race resolves to one survivor.
727///
728/// Lifecycle: the loop observes the process-wide daemon shutdown token, so
729/// `drain()` never waits on it forever, and it retains the handle of the
730/// child it spawned — reaping it with `try_wait` on every probe (no zombie
731/// accumulation during a persistent startup failure) and never stacking a
732/// second spawn on a still-live child. On shutdown, a child this supervisor
733/// spawned is killed and reaped; a pre-existing events daemon it never
734/// spawned is left alone.
735///
736/// The reachability probe uses the peer-verified connect: a socket answered
737/// by a foreign-uid process is treated as UNREACHABLE (and logged loudly),
738/// so a pre-bound spoof socket triggers a real-daemon spawn instead of being
739/// reported healthy.
740#[cfg(unix)]
741pub async fn supervise_events_daemon(db_path: PathBuf, socket_path: PathBuf) {
742    const PROBE_INTERVAL: Duration = Duration::from_secs(15);
743    let shutdown = crate::daemon::daemon_shutdown_token();
744    let mut child: Option<std::process::Child> = None;
745    let mut respawns: u64 = 0;
746    loop {
747        // Reap first: a child that exited (crashed, lost the advisory lock
748        // race, refused an untrusted directory) must not linger as a zombie,
749        // and clearing the slot is what re-arms the spawn below.
750        if let Some(c) = child.as_mut() {
751            match c.try_wait() {
752                Ok(Some(status)) => {
753                    tracing::info!(%status, "events daemon child exited");
754                    child = None;
755                }
756                Ok(None) => {}
757                Err(error) => {
758                    tracing::warn!(error = %error, "cannot poll events daemon child; dropping handle");
759                    child = None;
760                }
761            }
762        }
763
764        let reachable = match connect_verified(&socket_path).await {
765            Ok(_stream) => true,
766            Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
767                tracing::warn!(
768                    socket = %socket_path.display(),
769                    error = %error,
770                    "events socket answered by a foreign uid; treating as unreachable"
771                );
772                false
773            }
774            Err(_) => false,
775        };
776        if !reachable && child.is_none() {
777            match std::env::current_exe() {
778                Ok(exe) => {
779                    let spawned = std::process::Command::new(exe)
780                        .arg("events-daemon")
781                        .arg("--db")
782                        .arg(&db_path)
783                        .arg("--socket")
784                        .arg(&socket_path)
785                        .stdin(std::process::Stdio::null())
786                        .stdout(std::process::Stdio::null())
787                        .stderr(std::process::Stdio::null())
788                        .spawn();
789                    match spawned {
790                        Ok(spawned_child) => {
791                            respawns += 1;
792                            tracing::info!(
793                                pid = spawned_child.id(),
794                                respawns,
795                                socket = %socket_path.display(),
796                                "spawned events daemon"
797                            );
798                            child = Some(spawned_child);
799                        }
800                        Err(error) => {
801                            tracing::warn!(error = %error, "failed to spawn events daemon");
802                        }
803                    }
804                }
805                Err(error) => {
806                    tracing::warn!(error = %error, "cannot resolve current executable for events daemon spawn");
807                }
808            }
809        }
810        tokio::select! {
811            _ = shutdown.cancelled() => break,
812            _ = tokio::time::sleep(PROBE_INTERVAL) => {}
813        }
814    }
815    if let Some(mut c) = child.take() {
816        // Our child, our cleanup: the events daemon holds no volatile queue
817        // state (SQLite is the durability), and the next daemon host's
818        // supervisor respawns it.
819        let _ = c.kill();
820        let _ = c.wait();
821        tracing::info!("events daemon child stopped with supervisor shutdown");
822    }
823}
824
825/// Serve the events daemon loop on `socket_path`, owning `db_path`.
826///
827/// Binds the socket (removing a stale path first), then accepts connections
828/// for the process lifetime. Each connection is served sequentially:
829/// same-uid admission, then a read-frame → dispatch → write-frame loop until
830/// the peer disconnects. All storage goes through `SqlEventStore` on a
831/// backend opened read-write against `db_path`; the events schema is ensured
832/// once at boot.
833///
834/// Bind-path trust mirrors the main daemon socket: the socket directory must
835/// pass the same ownership/mode/swap-resistance validation, and the bound
836/// socket entry is chmod'd 0600 fail-closed. Pathname reachability is not
837/// identity — clients additionally verify the peer uid on every connect —
838/// but a hardened bind path is what keeps the *bind* itself out of another
839/// user's hands.
840#[cfg(unix)]
841pub async fn run_events_daemon(db_path: &Path, socket_path: &Path) -> anyhow::Result<()> {
842    // The subcommand's `--db`/`--socket` arrive from argv and may be
843    // relative; anchor them before anything derives a parent from them.
844    let db_path = &absolutize(db_path);
845    let socket_path = &absolutize(socket_path);
846    // Directory trust comes FIRST: the lock guard below opens and chmods a
847    // path in this directory, and validating only before the later bind
848    // would let those operations run in a directory another user controls.
849    if let Some(parent) = socket_path.parent() {
850        std::fs::create_dir_all(parent)?;
851        crate::daemon::ensure_socket_dir_is_trusted(parent)?;
852    }
853    let Some(_guard) = try_acquire_events_daemon_guard(socket_path) else {
854        tracing::info!(
855            socket = %socket_path.display(),
856            "events daemon lock unavailable (held by another daemon, or hardening refused); exiting"
857        );
858        return Ok(());
859    };
860    ensure_events_db_owner_only(db_path)?;
861    let backend = Arc::new(StorageBackend::sqlite(db_path)?);
862    // Ensure the schema once, loudly, before accepting traffic.
863    backend.events()?;
864    // The `-wal`/`-shm` sidecars inherit the database file's mode at
865    // creation; tighten any that already exist from before the hardening.
866    harden_events_db_sidecars(db_path)?;
867
868    if socket_path.exists() {
869        std::fs::remove_file(socket_path)?;
870    }
871    let listener = UnixListener::bind(socket_path)?;
872    {
873        use std::os::unix::fs::PermissionsExt;
874        // Fail closed, same contract as the main daemon socket: if the entry
875        // cannot be made owner-only, drop the listener and remove it rather
876        // than serve a world-reachable socket the design never covered.
877        if let Err(e) =
878            std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))
879        {
880            drop(listener);
881            let _ = std::fs::remove_file(socket_path);
882            anyhow::bail!(
883                "refusing to serve events: cannot chmod 0600 {}: {e}. The events socket must \
884                 be owner-only.",
885                socket_path.display()
886            );
887        }
888    }
889    let daemon_euid = unsafe { libc::geteuid() };
890    // Per-namespace store cache: `events_for_namespace` takes a writer-lane
891    // checkout and re-runs the schema DDL on every call, so paying it once
892    // per namespace instead of once per request keeps the writer lane for
893    // actual writes. The namespace is client-supplied, so the cache is
894    // bounded (`MAX_CACHED_NAMESPACE_STORES`) rather than trusted small.
895    let stores: NamespaceStores = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
896    let connections = Arc::new(tokio::sync::Semaphore::new(MAX_EVENTS_CONNECTIONS));
897    let frame_budget = Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_REQUEST_BYTES));
898    tracing::info!(
899        socket = %socket_path.display(),
900        db = %db_path.display(),
901        "events daemon listening"
902    );
903
904    loop {
905        let (stream, _addr) = match listener.accept().await {
906            Ok(pair) => pair,
907            Err(error) => {
908                tracing::warn!(error = %error, "events daemon accept failed");
909                continue;
910            }
911        };
912        match crate::daemon::peer_uid(&stream) {
913            Ok(uid) if crate::daemon::uid_is_permitted(uid, daemon_euid) => {}
914            Ok(uid) => {
915                tracing::warn!(peer_uid = uid, "events daemon rejected foreign-uid peer");
916                continue;
917            }
918            Err(error) => {
919                tracing::warn!(error = %error, "events daemon could not read peer credentials");
920                continue;
921            }
922        }
923        let permit = match Arc::clone(&connections).try_acquire_owned() {
924            Ok(permit) => permit,
925            Err(_) => {
926                // At the cap: drop the stream (peer sees EOF and retries via
927                // its own reconnect path) instead of queueing unbounded work.
928                tracing::warn!(
929                    cap = MAX_EVENTS_CONNECTIONS,
930                    "events daemon at connection cap; dropping new connection"
931                );
932                continue;
933            }
934        };
935        let backend = Arc::clone(&backend);
936        let stores = Arc::clone(&stores);
937        let frame_budget = Arc::clone(&frame_budget);
938        crate::daemon::spawn_tracked_task(async move {
939            serve_events_conn(stream, backend, stores, frame_budget).await;
940            drop(permit);
941        });
942    }
943}
944
945/// Read one length-prefixed request frame, admitting the body buffer against
946/// the shared byte budget before allocating it. The returned permit holds
947/// the admitted bytes for as long as the buffer may be alive — the caller
948/// drops it after the response is written. The per-frame cap is checked
949/// before admission, so a single frame is always satisfiable against the
950/// full budget (`MAX_INFLIGHT_REQUEST_BYTES >= MAX_FRAME_BYTES`) and a
951/// waiter can never deadlock on an impossible request.
952#[cfg(unix)]
953async fn read_frame_budgeted(
954    stream: &mut UnixStream,
955    budget: &Arc<tokio::sync::Semaphore>,
956) -> std::io::Result<(Vec<u8>, tokio::sync::OwnedSemaphorePermit)> {
957    use tokio::io::AsyncReadExt;
958    let mut len_buf = [0u8; 4];
959    stream.read_exact(&mut len_buf).await?;
960    let len = u32::from_be_bytes(len_buf) as usize;
961    if len > crate::daemon::MAX_FRAME_BYTES {
962        return Err(std::io::Error::new(
963            std::io::ErrorKind::InvalidData,
964            format!(
965                "daemon frame of {len} bytes exceeds {} cap",
966                crate::daemon::MAX_FRAME_BYTES
967            ),
968        ));
969    }
970    let permit = Arc::clone(budget)
971        .acquire_many_owned(len as u32)
972        .await
973        .map_err(|_| std::io::Error::other("frame budget closed"))?;
974    let mut buf = vec![0u8; len];
975    stream.read_exact(&mut buf).await?;
976    Ok((buf, permit))
977}
978
979#[cfg(unix)]
980type NamespaceStores =
981    Arc<std::sync::Mutex<std::collections::HashMap<String, Arc<dyn EventStore>>>>;
982
983/// The cached per-namespace store, constructing (and caching) it on first
984/// use. Construction failures are not cached — the next request retries.
985///
986/// The cache key is the trimmed namespace, matching the backend's own
987/// normalization, so spellings that resolve to one store share one handle.
988/// The map is bounded: at `MAX_CACHED_NAMESPACE_STORES` entries an arbitrary
989/// existing entry is evicted to admit the new namespace, so the map never
990/// grows past the cap and no namespace is permanently condemned to
991/// per-request store rebuilds. Keys reaching this cache have already passed
992/// `Namespace` validation at dispatch (charset + length bound), so cap ×
993/// bounded key is the worst-case retained memory.
994#[cfg(unix)]
995fn namespace_store(
996    backend: &StorageBackend,
997    stores: &NamespaceStores,
998    namespace: &str,
999) -> Result<Arc<dyn EventStore>, khive_db::SqliteError> {
1000    namespace_store_with_cap(backend, stores, namespace, MAX_CACHED_NAMESPACE_STORES)
1001}
1002
1003#[cfg(unix)]
1004fn namespace_store_with_cap(
1005    backend: &StorageBackend,
1006    stores: &NamespaceStores,
1007    namespace: &str,
1008    cap: usize,
1009) -> Result<Arc<dyn EventStore>, khive_db::SqliteError> {
1010    let key = namespace.trim();
1011    if let Some(store) = stores
1012        .lock()
1013        .unwrap_or_else(std::sync::PoisonError::into_inner)
1014        .get(key)
1015    {
1016        return Ok(Arc::clone(store));
1017    }
1018    let store = backend.events_for_namespace(namespace)?;
1019    let mut map = stores
1020        .lock()
1021        .unwrap_or_else(std::sync::PoisonError::into_inner);
1022    if map.len() >= cap && !map.contains_key(key) {
1023        // At the cap, evict an arbitrary entry instead of refusing
1024        // admission: stores are cheap pool handles, so eviction costs one
1025        // rebuild on the victim's next request, while refusing admission
1026        // would make every request beyond the cap rebuild its store and
1027        // rerun schema init on the SQLite writer forever. Keys are
1028        // validated `Namespace` values (bounded length), so the map's
1029        // worst-case memory is cap × small key + cap handles.
1030        if let Some(victim) = map.keys().next().cloned() {
1031            map.remove(&victim);
1032        }
1033    }
1034    map.insert(key.to_string(), Arc::clone(&store));
1035    Ok(store)
1036}
1037
1038#[cfg(unix)]
1039async fn serve_events_conn(
1040    mut stream: UnixStream,
1041    backend: Arc<StorageBackend>,
1042    stores: NamespaceStores,
1043    frame_budget: Arc<tokio::sync::Semaphore>,
1044) {
1045    loop {
1046        // Deadline on the whole frame read — budget admission included: a
1047        // peer that opens a connection and stalls, or that cannot be
1048        // admitted because other connections hold the byte budget, is
1049        // closed instead of holding this task and its descriptor
1050        // indefinitely.
1051        let (payload, budget_permit) = match tokio::time::timeout(
1052            CONN_IO_TIMEOUT,
1053            read_frame_budgeted(&mut stream, &frame_budget),
1054        )
1055        .await
1056        {
1057            Ok(Ok(pair)) => pair,
1058            // Includes clean EOF on peer disconnect, and the expired deadline.
1059            Ok(Err(_)) | Err(_) => return,
1060        };
1061        let response = match serde_json::from_slice::<EventsRequest>(&payload) {
1062            Ok(request) => dispatch_events_request(request, &backend, &stores).await,
1063            Err(error) => EventsResponse::Error {
1064                message: format!("events daemon could not parse request frame: {error}"),
1065                retryable: false,
1066                writer_task_state: None,
1067            },
1068        };
1069        let bytes = match serde_json::to_vec(&response) {
1070            Ok(bytes) => bytes,
1071            Err(error) => {
1072                tracing::error!(error = %error, "events daemon response serialization failed");
1073                return;
1074            }
1075        };
1076        // Same deadline on the response write: a peer that stops reading
1077        // would otherwise park this task in a full socket buffer.
1078        match tokio::time::timeout(CONN_IO_TIMEOUT, write_frame(&mut stream, &bytes)).await {
1079            Ok(Ok(())) => {}
1080            Ok(Err(_)) | Err(_) => return,
1081        }
1082        // The request buffer is long dropped; release its byte budget only
1083        // now, after the whole request lifecycle, so admission tracks live
1084        // work rather than just live buffers.
1085        drop(budget_permit);
1086    }
1087}
1088
1089#[cfg(unix)]
1090async fn dispatch_events_request(
1091    request: EventsRequest,
1092    backend: &StorageBackend,
1093    stores: &NamespaceStores,
1094) -> EventsResponse {
1095    if request.protocol_version() != EVENTS_PROTOCOL_VERSION {
1096        return EventsResponse::Error {
1097            message: format!(
1098                "events protocol version mismatch: daemon speaks {}, client sent {}",
1099                EVENTS_PROTOCOL_VERSION,
1100                request.protocol_version()
1101            ),
1102            retryable: false,
1103            writer_task_state: None,
1104        };
1105    }
1106    // The wire namespace is an unrestricted client-supplied String until this
1107    // point. Validate it as a real `Namespace` (charset plus the 256-byte
1108    // length bound) before it can become a cache key, a store, or a database
1109    // row — without this, near-frame-sized strings are attacker-controlled
1110    // retained memory in a process that lives for months.
1111    if let Err(error) = khive_types::Namespace::parse(request.namespace().trim()) {
1112        return EventsResponse::Error {
1113            message: format!("events request namespace rejected: {error}"),
1114            retryable: false,
1115            writer_task_state: None,
1116        };
1117    }
1118    let store = match namespace_store(backend, stores, request.namespace()) {
1119        Ok(store) => store,
1120        Err(error) => {
1121            return EventsResponse::Error {
1122                message: format!("events store unavailable: {error}"),
1123                retryable: true,
1124                writer_task_state: None,
1125            };
1126        }
1127    };
1128    match request {
1129        EventsRequest::AppendEvents { events, .. } => match store.append_events(events).await {
1130            Ok(summary) => EventsResponse::Appended { summary },
1131            Err(error) => storage_error_response(&error),
1132        },
1133        EventsRequest::AppendEventsIdempotent { events, .. } => {
1134            match store.append_events_idempotent(events).await {
1135                Ok(result) => EventsResponse::Idempotent { result },
1136                Err(error) => storage_error_response(&error),
1137            }
1138        }
1139        EventsRequest::GetEvent { id, .. } => match store.get_event(id).await {
1140            Ok(event) => EventsResponse::Event { event },
1141            Err(error) => storage_error_response(&error),
1142        },
1143        EventsRequest::QueryEvents { filter, page, .. } => {
1144            if page.limit > MAX_QUERY_EVENTS_PAGE_ROWS {
1145                return EventsResponse::Error {
1146                    message: format!(
1147                        "events query page limit {} exceeds the daemon cap of {} rows; \
1148                         request narrower pages",
1149                        page.limit, MAX_QUERY_EVENTS_PAGE_ROWS
1150                    ),
1151                    retryable: false,
1152                    writer_task_state: None,
1153                };
1154            }
1155            match store.query_events(filter, page).await {
1156                Ok(page) => EventsResponse::Pageful { page },
1157                Err(error) => storage_error_response(&error),
1158            }
1159        }
1160        EventsRequest::CountEvents { filter, .. } => match store.count_events(filter).await {
1161            Ok(count) => EventsResponse::Count { count },
1162            Err(error) => storage_error_response(&error),
1163        },
1164    }
1165}
1166
1167#[cfg(unix)]
1168fn storage_error_response(error: &StorageError) -> EventsResponse {
1169    // Writer-task terminations carry their request state verbatim: the
1170    // ADR-133 retry classifier decides per-variant (`NotStarted` and
1171    // `TransactionRolledBack` are mandated retries), so the state must
1172    // survive the socket even though `is_retryable()` reports the family
1173    // non-transient.
1174    let writer_task_state = match error {
1175        StorageError::WriterTaskTerminated { request_state } => {
1176            Some(WireWriterTaskState::from(*request_state))
1177        }
1178        _ => None,
1179    };
1180    EventsResponse::Error {
1181        message: error.to_string(),
1182        // Defer to the storage layer's own transience classifier rather than
1183        // re-enumerating variants here: a hand-rolled subset silently turns
1184        // transient writer contention (`WriterTaskBusy`, `Transaction`) into
1185        // a terminal error on the client side of the socket.
1186        retryable: error.is_retryable(),
1187        writer_task_state,
1188    }
1189}
1190
1191// ---------------------------------------------------------------------------
1192// Client side
1193// ---------------------------------------------------------------------------
1194
1195/// Connect to the events socket and verify who answered. Pathname
1196/// reachability is not identity: any process that can write the directory can
1197/// pre-bind the path. The kernel-reported peer uid is the one identity the
1198/// far side cannot choose, so every client connect — round-trips, the
1199/// forwarder, the supervisor probe — refuses a socket answered by a foreign
1200/// uid before a single frame is written.
1201#[cfg(unix)]
1202async fn connect_verified(socket_path: &Path) -> std::io::Result<UnixStream> {
1203    let stream = UnixStream::connect(socket_path).await?;
1204    // SAFETY: `geteuid` is always successful and takes no arguments.
1205    let own_euid = unsafe { libc::geteuid() } as u32;
1206    let peer = crate::daemon::peer_uid(&stream)?;
1207    if !crate::daemon::uid_is_permitted(peer, own_euid) {
1208        return Err(std::io::Error::new(
1209            std::io::ErrorKind::PermissionDenied,
1210            format!(
1211                "events socket {} answered by uid {peer}, not this process's uid {own_euid}; \
1212                 refusing to exchange event frames with an unowned daemon",
1213                socket_path.display()
1214            ),
1215        ));
1216    }
1217    Ok(stream)
1218}
1219
1220/// Counters describing the fire-and-forget lane's degradation. Zero drops is
1221/// the healthy state; any non-zero `dropped_batches` means the loss-tolerant
1222/// contract was exercised and says so.
1223#[cfg(unix)]
1224#[derive(Debug, Clone, Copy, Serialize)]
1225pub struct EventsForwardingMetrics {
1226    pub forwarded_batches: u64,
1227    pub forwarded_events: u64,
1228    pub dropped_batches: u64,
1229    pub dropped_events: u64,
1230}
1231
1232#[cfg(unix)]
1233#[derive(Debug, Default)]
1234struct ForwardingCounters {
1235    forwarded_batches: AtomicU64,
1236    forwarded_events: AtomicU64,
1237    dropped_batches: AtomicU64,
1238    dropped_events: AtomicU64,
1239}
1240
1241/// One per domain process: the connection to the events daemon plus the
1242/// bounded fire-and-forget append queue.
1243///
1244/// Synchronous round-trips each own a fresh connection: there is no shared
1245/// request connection and no lock in front of one, so concurrent reads and
1246/// audit flushes never queue behind a single stalled call, and the round-trip
1247/// timeout bounds each caller's whole attempt.
1248#[cfg(unix)]
1249pub struct EventsSplitClient {
1250    socket_path: PathBuf,
1251    append_tx: tokio::sync::mpsc::Sender<(String, Vec<Event>)>,
1252    counters: Arc<ForwardingCounters>,
1253    /// Flipped by the forwarder while the daemon is unreachable so the drop
1254    /// log fires once per outage, not once per batch.
1255    outage_logged: Arc<AtomicBool>,
1256    /// In-memory validator backing `preflight_event` without I/O.
1257    preflight_store: Arc<dyn EventStore>,
1258}
1259
1260#[cfg(unix)]
1261impl std::fmt::Debug for EventsSplitClient {
1262    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1263        f.debug_struct("EventsSplitClient")
1264            .field("socket_path", &self.socket_path)
1265            .finish_non_exhaustive()
1266    }
1267}
1268
1269#[cfg(unix)]
1270impl EventsSplitClient {
1271    /// Build the client and spawn its background forwarder.
1272    pub fn new(socket_path: PathBuf) -> crate::error::RuntimeResult<Arc<Self>> {
1273        Self::new_with_queue_depth(socket_path, DEFAULT_APPEND_QUEUE_BATCHES)
1274    }
1275
1276    /// [`Self::new`] with an explicit fire-and-forget queue bound. Tests use a
1277    /// tiny depth to exercise the overflow drop arm deterministically.
1278    pub fn new_with_queue_depth(
1279        socket_path: PathBuf,
1280        queue_depth: usize,
1281    ) -> crate::error::RuntimeResult<Arc<Self>> {
1282        Self::new_with_queue_depth_and_delivery_timeout(socket_path, queue_depth, REQUEST_TIMEOUT)
1283    }
1284
1285    /// [`Self::new_with_queue_depth`] with an explicit per-batch delivery
1286    /// timeout, so tests can prove the forwarder abandons a hung peer without
1287    /// waiting out the production clock.
1288    fn new_with_queue_depth_and_delivery_timeout(
1289        socket_path: PathBuf,
1290        queue_depth: usize,
1291        delivery_timeout: Duration,
1292    ) -> crate::error::RuntimeResult<Arc<Self>> {
1293        let preflight_backend = StorageBackend::memory()?;
1294        let preflight_store = preflight_backend.events()?;
1295        // The in-memory backend must outlive the store handle; the store holds
1296        // the pool Arc internally, so dropping the backend wrapper here is fine.
1297
1298        let (append_tx, append_rx) =
1299            tokio::sync::mpsc::channel::<(String, Vec<Event>)>(queue_depth.max(1));
1300        let counters = Arc::new(ForwardingCounters::default());
1301        let outage_logged = Arc::new(AtomicBool::new(false));
1302
1303        let client = Arc::new(Self {
1304            socket_path: socket_path.clone(),
1305            append_tx,
1306            counters: Arc::clone(&counters),
1307            outage_logged: Arc::clone(&outage_logged),
1308            preflight_store,
1309        });
1310
1311        crate::daemon::spawn_tracked_task(run_forwarder(
1312            socket_path,
1313            append_rx,
1314            counters,
1315            outage_logged,
1316            delivery_timeout,
1317            crate::daemon::daemon_shutdown_token(),
1318        ));
1319        Ok(client)
1320    }
1321
1322    /// Snapshot of the fire-and-forget lane's health.
1323    pub fn metrics(&self) -> EventsForwardingMetrics {
1324        EventsForwardingMetrics {
1325            forwarded_batches: self.counters.forwarded_batches.load(Ordering::Relaxed),
1326            forwarded_events: self.counters.forwarded_events.load(Ordering::Relaxed),
1327            dropped_batches: self.counters.dropped_batches.load(Ordering::Relaxed),
1328            dropped_events: self.counters.dropped_events.load(Ordering::Relaxed),
1329        }
1330    }
1331
1332    /// Enqueue a batch on the fire-and-forget lane. Never blocks; a full
1333    /// queue is a counted, logged drop.
1334    fn enqueue(&self, namespace: &str, events: Vec<Event>) {
1335        let count = events.len() as u64;
1336        match self.append_tx.try_send((namespace.to_string(), events)) {
1337            Ok(()) => {}
1338            Err(_) => {
1339                self.counters
1340                    .dropped_batches
1341                    .fetch_add(1, Ordering::Relaxed);
1342                self.counters
1343                    .dropped_events
1344                    .fetch_add(count, Ordering::Relaxed);
1345                if !self.outage_logged.swap(true, Ordering::Relaxed) {
1346                    tracing::warn!(
1347                        dropped_events = count,
1348                        "events append queue full; dropping loss-tolerant events until it drains"
1349                    );
1350                }
1351            }
1352        }
1353    }
1354
1355    /// One synchronous framed round-trip with a bounded timeout, on its own
1356    /// fresh connection. No connection is shared between concurrent callers,
1357    /// so no caller ever waits on another's stall, and `REQUEST_TIMEOUT`
1358    /// bounds the whole attempt — connect, peer verification, write, read.
1359    async fn round_trip(&self, request: &EventsRequest) -> StorageResult<EventsResponse> {
1360        let op = "events daemon round-trip";
1361        let payload = serde_json::to_vec(request).map_err(|error| StorageError::Serialization {
1362            capability: khive_storage::StorageCapability::Events,
1363            message: format!("events request serialization failed: {error}"),
1364        })?;
1365
1366        let attempt = async {
1367            let mut stream = connect_verified(&self.socket_path).await?;
1368            write_frame(&mut stream, &payload).await?;
1369            let bytes = read_frame(&mut stream).await?;
1370            std::io::Result::Ok(bytes)
1371        };
1372        let bytes = match tokio::time::timeout(REQUEST_TIMEOUT, attempt).await {
1373            Ok(Ok(bytes)) => bytes,
1374            Ok(Err(error)) => {
1375                return Err(StorageError::Pool {
1376                    operation: op.into(),
1377                    message: format!(
1378                        "events daemon unreachable at {}: {error}",
1379                        self.socket_path.display()
1380                    ),
1381                });
1382            }
1383            Err(_elapsed) => {
1384                return Err(StorageError::Timeout {
1385                    operation: op.into(),
1386                });
1387            }
1388        };
1389
1390        serde_json::from_slice::<EventsResponse>(&bytes).map_err(|error| {
1391            StorageError::Serialization {
1392                capability: khive_storage::StorageCapability::Events,
1393                message: format!("events response deserialization failed: {error}"),
1394            }
1395        })
1396    }
1397}
1398
1399/// Drain every batch currently sitting in the fire-and-forget queue,
1400/// counting each as a dropped batch/events and logging one summary line
1401/// (never one line per batch — a full queue at shutdown is exactly the
1402/// bursty case a per-batch log would flood). Non-blocking: `try_recv` only
1403/// consumes what is already queued, so it terminates as soon as the queue
1404/// (temporarily) empties even though the sender half is still live.
1405#[cfg(unix)]
1406fn drain_dropped_queue(
1407    rx: &mut tokio::sync::mpsc::Receiver<(String, Vec<Event>)>,
1408    counters: &ForwardingCounters,
1409) {
1410    let mut dropped_batches = 0u64;
1411    let mut dropped_events = 0u64;
1412    while let Ok((_, events)) = rx.try_recv() {
1413        dropped_batches += 1;
1414        dropped_events += events.len() as u64;
1415    }
1416    if dropped_batches > 0 {
1417        counters
1418            .dropped_batches
1419            .fetch_add(dropped_batches, Ordering::Relaxed);
1420        counters
1421            .dropped_events
1422            .fetch_add(dropped_events, Ordering::Relaxed);
1423        tracing::warn!(
1424            dropped_batches,
1425            dropped_events,
1426            "events forwarder shutting down; dropping queued loss-tolerant batches"
1427        );
1428    }
1429}
1430
1431/// The background fire-and-forget forwarder: drains the bounded queue into
1432/// framed appends on its own connection, reconnecting with backoff. A batch
1433/// that cannot be delivered is dropped and counted — never retried, never
1434/// blocking the queue behind it.
1435#[cfg(unix)]
1436async fn run_forwarder(
1437    socket_path: PathBuf,
1438    mut rx: tokio::sync::mpsc::Receiver<(String, Vec<Event>)>,
1439    counters: Arc<ForwardingCounters>,
1440    outage_logged: Arc<AtomicBool>,
1441    delivery_timeout: Duration,
1442    shutdown: tokio_util::sync::CancellationToken,
1443) {
1444    // The client lives in a process-global registry, so its sender is never
1445    // dropped and `rx.recv()` alone would keep this tracked task alive
1446    // through daemon shutdown, forcing `drain()` to its full timeout.
1447    // Observe the shutdown token directly: queued batches at shutdown are a
1448    // counted drop, exactly the lane's loss-tolerant contract — enforced by
1449    // draining and counting `rx` below on every shutdown exit, not just
1450    // implied by the comment.
1451    let mut conn: Option<UnixStream> = None;
1452    loop {
1453        let (namespace, events) = tokio::select! {
1454            _ = shutdown.cancelled() => {
1455                drain_dropped_queue(&mut rx, &counters);
1456                break;
1457            },
1458            received = rx.recv() => match received {
1459                Some(batch) => batch,
1460                None => break,
1461            },
1462        };
1463        let count = events.len() as u64;
1464        let request = EventsRequest::AppendEvents {
1465            protocol_version: EVENTS_PROTOCOL_VERSION,
1466            namespace,
1467            events,
1468        };
1469        let payload = match serde_json::to_vec(&request) {
1470            Ok(payload) => payload,
1471            Err(error) => {
1472                counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
1473                counters.dropped_events.fetch_add(count, Ordering::Relaxed);
1474                tracing::error!(error = %error, "events forwarder serialization failed; batch dropped");
1475                continue;
1476            }
1477        };
1478
1479        // The delivery itself must stay both cancellable and bounded: a
1480        // connected but non-responding daemon would otherwise park this
1481        // tracked task inside write_frame/read_frame, beyond the reach of the
1482        // recv-side shutdown select — and daemon drain would wait its full
1483        // timeout on it. Shutdown mid-delivery abandons the batch (the
1484        // lane's loss-tolerant contract); a timeout poisons the connection,
1485        // since the peer may answer the abandoned frame later.
1486        let delivered = tokio::select! {
1487            _ = shutdown.cancelled() => {
1488                counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
1489                counters.dropped_events.fetch_add(count, Ordering::Relaxed);
1490                tracing::warn!(
1491                    dropped_events = count,
1492                    "events forwarder shutting down mid-delivery; in-flight batch dropped"
1493                );
1494                drain_dropped_queue(&mut rx, &counters);
1495                break;
1496            },
1497            outcome = tokio::time::timeout(
1498                delivery_timeout,
1499                deliver_batch(&socket_path, &mut conn, &payload),
1500            ) => match outcome {
1501                Ok(delivered) => delivered,
1502                Err(_elapsed) => {
1503                    conn = None;
1504                    false
1505                }
1506            },
1507        };
1508        if delivered {
1509            counters.forwarded_batches.fetch_add(1, Ordering::Relaxed);
1510            counters
1511                .forwarded_events
1512                .fetch_add(count, Ordering::Relaxed);
1513            if outage_logged.swap(false, Ordering::Relaxed) {
1514                tracing::info!("events daemon reachable again; forwarding resumed");
1515            }
1516        } else {
1517            counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
1518            counters.dropped_events.fetch_add(count, Ordering::Relaxed);
1519            if !outage_logged.swap(true, Ordering::Relaxed) {
1520                tracing::warn!(
1521                    socket = %socket_path.display(),
1522                    "events daemon unreachable; dropping loss-tolerant events until it returns"
1523                );
1524            }
1525            tokio::select! {
1526                _ = shutdown.cancelled() => {
1527                    drain_dropped_queue(&mut rx, &counters);
1528                    break;
1529                },
1530                _ = tokio::time::sleep(FORWARDER_BACKOFF) => {}
1531            }
1532        }
1533    }
1534}
1535
1536/// Try to deliver one framed append over the forwarder connection,
1537/// (re)connecting at most once. Returns whether the daemon acknowledged.
1538/// Connections are peer-verified: a socket answered by a foreign uid is a
1539/// failed connect, never a delivery target.
1540#[cfg(unix)]
1541async fn deliver_batch(socket_path: &Path, conn: &mut Option<UnixStream>, payload: &[u8]) -> bool {
1542    for _attempt in 0..2u8 {
1543        if conn.is_none() {
1544            match connect_verified(socket_path).await {
1545                Ok(stream) => *conn = Some(stream),
1546                Err(_) => return false,
1547            }
1548        }
1549        let stream = conn.as_mut().expect("connection populated above");
1550        let ok = async {
1551            write_frame(stream, payload).await?;
1552            let bytes = read_frame(stream).await?;
1553            std::io::Result::Ok(bytes)
1554        }
1555        .await;
1556        match ok {
1557            Ok(bytes) => {
1558                return !matches!(
1559                    serde_json::from_slice::<EventsResponse>(&bytes),
1560                    Ok(EventsResponse::Error { .. }) | Err(_)
1561                );
1562            }
1563            Err(_) => {
1564                // Stale connection (daemon restarted): drop it and retry once
1565                // with a fresh connect; a second failure is a real outage.
1566                *conn = None;
1567            }
1568        }
1569    }
1570    false
1571}
1572
1573// ---------------------------------------------------------------------------
1574// The per-namespace EventStore handle
1575// ---------------------------------------------------------------------------
1576
1577/// [`EventStore`] implementation the runtime hands out when the events split
1578/// runs in daemon mode. Appends are fire-and-forget through the client's
1579/// bounded queue; the ADR-133 idempotent lane and all reads are synchronous
1580/// round-trips to the events daemon.
1581#[cfg(unix)]
1582#[derive(Debug)]
1583pub struct ForwardingEventStore {
1584    namespace: String,
1585    client: Arc<EventsSplitClient>,
1586}
1587
1588#[cfg(unix)]
1589impl ForwardingEventStore {
1590    pub fn new(namespace: impl Into<String>, client: Arc<EventsSplitClient>) -> Self {
1591        Self {
1592            namespace: namespace.into(),
1593            client,
1594        }
1595    }
1596
1597    fn unexpected(&self, op: &'static str, response: EventsResponse) -> StorageError {
1598        match response {
1599            EventsResponse::Error {
1600                message,
1601                retryable,
1602                writer_task_state,
1603            } => {
1604                if let Some(state) = writer_task_state {
1605                    // Reconstruct the exact writer-task variant: the ADR-133
1606                    // retry classifier decides per-state (`NotStarted` and
1607                    // `TransactionRolledBack` are mandated retries,
1608                    // `SideEffectsUnknown` gates double-send decisions), so
1609                    // flattening any of them into a generic refusal breaks
1610                    // the retry contract on the client side of the socket.
1611                    // The field is the single carrier: cross-version frames
1612                    // never reach this mapping (the daemon refuses a
1613                    // mismatched protocol version before dispatch, and the
1614                    // refusal precedes any write, so a version-skewed peer
1615                    // has no writer state to lose — the guarantee that
1616                    // obligates a version bump on any field-shape change).
1617                    StorageError::WriterTaskTerminated {
1618                        request_state: state.into(),
1619                    }
1620                } else if retryable {
1621                    StorageError::Pool {
1622                        operation: op.into(),
1623                        message,
1624                    }
1625                } else {
1626                    StorageError::InvalidInput {
1627                        capability: khive_storage::StorageCapability::Events,
1628                        operation: op.into(),
1629                        message,
1630                    }
1631                }
1632            }
1633            other => StorageError::Serialization {
1634                capability: khive_storage::StorageCapability::Events,
1635                message: format!("events daemon returned mismatched response for {op}: {other:?}"),
1636            },
1637        }
1638    }
1639}
1640
1641#[cfg(unix)]
1642#[async_trait]
1643impl EventStore for ForwardingEventStore {
1644    async fn append_event(&self, event: Event) -> StorageResult<()> {
1645        self.client.enqueue(&self.namespace, vec![event]);
1646        Ok(())
1647    }
1648
1649    async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
1650        let attempted = events.len() as u64;
1651        self.client.enqueue(&self.namespace, events);
1652        // Fire-and-forget: the hand-off succeeded or was counted as a drop;
1653        // either way the caller's contract is "accepted for forwarding".
1654        Ok(BatchWriteSummary {
1655            attempted,
1656            affected: attempted,
1657            failed: 0,
1658            first_error: String::new(),
1659        })
1660    }
1661
1662    async fn get_event(&self, id: Uuid) -> StorageResult<Option<Event>> {
1663        let request = EventsRequest::GetEvent {
1664            protocol_version: EVENTS_PROTOCOL_VERSION,
1665            namespace: self.namespace.clone(),
1666            id,
1667        };
1668        match self.client.round_trip(&request).await? {
1669            EventsResponse::Event { event } => Ok(event),
1670            other => Err(self.unexpected("get_event", other)),
1671        }
1672    }
1673
1674    async fn query_events(
1675        &self,
1676        filter: EventFilter,
1677        page: PageRequest,
1678    ) -> StorageResult<Page<Event>> {
1679        let request = EventsRequest::QueryEvents {
1680            protocol_version: EVENTS_PROTOCOL_VERSION,
1681            namespace: self.namespace.clone(),
1682            filter,
1683            page,
1684        };
1685        match self.client.round_trip(&request).await? {
1686            EventsResponse::Pageful { page } => Ok(page),
1687            other => Err(self.unexpected("query_events", other)),
1688        }
1689    }
1690
1691    async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
1692        let request = EventsRequest::CountEvents {
1693            protocol_version: EVENTS_PROTOCOL_VERSION,
1694            namespace: self.namespace.clone(),
1695            filter,
1696        };
1697        match self.client.round_trip(&request).await? {
1698            EventsResponse::Count { count } => Ok(count),
1699            other => Err(self.unexpected("count_events", other)),
1700        }
1701    }
1702
1703    fn preflight_event(&self, event: &Event) -> StorageResult<()> {
1704        // Same validation code path as the daemon-side store, zero I/O.
1705        self.client.preflight_store.preflight_event(event)
1706    }
1707
1708    async fn append_events_idempotent(
1709        &self,
1710        events: Vec<Event>,
1711    ) -> StorageResult<IdempotentEventBatchResult> {
1712        let request = EventsRequest::AppendEventsIdempotent {
1713            protocol_version: EVENTS_PROTOCOL_VERSION,
1714            namespace: self.namespace.clone(),
1715            events,
1716        };
1717        match self.client.round_trip(&request).await? {
1718            EventsResponse::Idempotent { result } => Ok(result),
1719            other => Err(self.unexpected("append_events_idempotent", other)),
1720        }
1721    }
1722
1723    fn supports_idempotent_audit_batch(&self) -> bool {
1724        true
1725    }
1726}
1727
1728/// The store the runtime hands out when the events split is configured:
1729/// routes by APPEND CLASS rather than moving the whole event plane.
1730///
1731/// - The ADR-133 idempotent audit-batch lane — the measured bulk of event
1732///   write volume (verb-dispatch audit plus the config-lock rows that ride
1733///   the same flusher) — goes to the events lane (the events database, forwarded or
1734///   direct).
1735/// - Plain appends stay on the legacy store, because the legacy `events`
1736///   table has raw-SQL consumers whose correctness depends on finding those
1737///   rows there: the schedule drain's creator-provenance fence, the kg
1738///   projection worker's guarded event INSERT (transactional with main-db
1739///   state), and GraphQuery's cross-substrate UNION. Those events are
1740///   low-volume domain facts; the split's contention relief does not need
1741///   them moved, and moving them breaks the consumers by construction.
1742/// - Reads merge both stores so trait-level consumers
1743///   (`brain.event_counts`, event getters) observe one event plane.
1744pub struct SplitEventStore {
1745    legacy: Arc<dyn EventStore>,
1746    lane: Arc<dyn EventStore>,
1747}
1748
1749impl std::fmt::Debug for SplitEventStore {
1750    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1751        f.debug_struct("SplitEventStore").finish_non_exhaustive()
1752    }
1753}
1754
1755impl SplitEventStore {
1756    /// Bound on `offset + limit` for a merged `query_events` window. Offset
1757    /// pagination over two stores materializes the whole prefix in memory
1758    /// (see `query_events`), so an unbounded offset would let a single
1759    /// request buffer both stores wholesale. The bound comfortably admits
1760    /// the largest legitimate bounded window in the tree
1761    /// (`brain.event_counts`' 50k page); deep walks page with a `before`
1762    /// cursor at `offset: 0`, which never grows the materialized prefix.
1763    pub const MAX_MERGED_WINDOW_ROWS: u64 = 100_000;
1764
1765    pub fn new(legacy: Arc<dyn EventStore>, lane: Arc<dyn EventStore>) -> Self {
1766        Self { legacy, lane }
1767    }
1768}
1769
1770#[async_trait]
1771impl EventStore for SplitEventStore {
1772    async fn append_event(&self, event: Event) -> StorageResult<()> {
1773        self.legacy.append_event(event).await
1774    }
1775
1776    async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
1777        self.legacy.append_events(events).await
1778    }
1779
1780    async fn get_event(&self, id: Uuid) -> StorageResult<Option<Event>> {
1781        // Domain events (legacy) are the likelier and cheaper hit; fall back
1782        // to the lane for audit-batch rows.
1783        if let Some(event) = self.legacy.get_event(id).await? {
1784            return Ok(Some(event));
1785        }
1786        self.lane.get_event(id).await
1787    }
1788
1789    async fn query_events(
1790        &self,
1791        filter: EventFilter,
1792        page: PageRequest,
1793    ) -> StorageResult<Page<Event>> {
1794        // Offset pagination cannot be split across two stores: fetch each
1795        // store's prefix covering the requested window, merge in the stores'
1796        // shared order (created_at DESC, id DESC), then window in memory.
1797        // Cost is O(offset + limit) rows materialized from each store — the
1798        // floor for offset semantics over two sources, since the split point
1799        // is unknowable without both prefixes. The sort below is a single
1800        // merge pass in practice (std's stable sort is adaptive on the
1801        // concatenation of two sorted runs). That materialization is why the
1802        // window is bounded below: without a bound, one authenticated
1803        // request with a pathological offset would make both stores buffer
1804        // and sort every matching row. Deep walks page with a `before`
1805        // cursor (strict `created_at <` bound in `EventFilter`) at
1806        // `offset: 0`, which stays inside the bound at any depth.
1807        let window = page.offset.saturating_add(u64::from(page.limit));
1808        if window > Self::MAX_MERGED_WINDOW_ROWS {
1809            return Err(StorageError::InvalidInput {
1810                capability: khive_storage::StorageCapability::Events,
1811                operation: "query_events".into(),
1812                message: format!(
1813                    "offset+limit ({window}) exceeds the merged event plane's window bound \
1814                     of {}; page deep windows with a `before` cursor at offset 0, or narrow \
1815                     the filter",
1816                    Self::MAX_MERGED_WINDOW_ROWS
1817                ),
1818            });
1819        }
1820        let prefix = PageRequest {
1821            offset: 0,
1822            limit: window.min(u64::from(u32::MAX)) as u32,
1823        };
1824        let legacy = self
1825            .legacy
1826            .query_events(filter.clone(), prefix.clone())
1827            .await?;
1828        let lane = self.lane.query_events(filter, prefix).await?;
1829        let total = match (legacy.total, lane.total) {
1830            (Some(a), Some(b)) => Some(a + b),
1831            _ => None,
1832        };
1833        let mut items = legacy.items;
1834        items.extend(lane.items);
1835        items.sort_by(|a, b| {
1836            b.created_at
1837                .cmp(&a.created_at)
1838                .then_with(|| b.id.cmp(&a.id))
1839        });
1840        let items = items
1841            .into_iter()
1842            .skip(page.offset as usize)
1843            .take(page.limit as usize)
1844            .collect();
1845        Ok(Page { items, total })
1846    }
1847
1848    async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
1849        let legacy = self.legacy.count_events(filter.clone()).await?;
1850        let lane = self.lane.count_events(filter).await?;
1851        Ok(legacy + lane)
1852    }
1853
1854    fn preflight_event(&self, event: &Event) -> StorageResult<()> {
1855        // Validation is store-independent; the lane's implementation is the
1856        // local zero-I/O validator in forwarding mode.
1857        self.lane.preflight_event(event)
1858    }
1859
1860    async fn append_events_idempotent(
1861        &self,
1862        events: Vec<Event>,
1863    ) -> StorageResult<IdempotentEventBatchResult> {
1864        // A retried batch may have landed — fully or partially — on the
1865        // legacy store before the split cutover, and merged reads stay
1866        // duplicate-free only while an id lives in exactly one store. Probe
1867        // the legacy store for the batch's ids and route resident rows back
1868        // through the legacy store's own idempotent machinery, which compares
1869        // every persisted column without re-inserting; only genuinely new
1870        // rows reach the lane. Without the probe, a legacy-resident id would
1871        // be inserted a second time into the lane and every merged query and
1872        // count would double-count it.
1873        if events.is_empty() {
1874            return self.lane.append_events_idempotent(events).await;
1875        }
1876        let ids: Vec<Uuid> = events.iter().map(|event| event.id).collect();
1877        let probe_limit = u32::try_from(ids.len()).map_err(|_| StorageError::InvalidInput {
1878            capability: khive_storage::StorageCapability::Events,
1879            operation: "append_events_idempotent".into(),
1880            message: format!(
1881                "batch of {} rows exceeds the legacy probe window",
1882                ids.len()
1883            ),
1884        })?;
1885        let existing = self
1886            .legacy
1887            .query_events(
1888                EventFilter {
1889                    ids,
1890                    ..EventFilter::default()
1891                },
1892                PageRequest {
1893                    offset: 0,
1894                    limit: probe_limit,
1895                },
1896            )
1897            .await?;
1898        let legacy_resident: std::collections::HashSet<Uuid> =
1899            existing.items.iter().map(|event| event.id).collect();
1900        if legacy_resident.is_empty() {
1901            return self.lane.append_events_idempotent(events).await;
1902        }
1903        let mut legacy_rows = Vec::new();
1904        let mut lane_rows = Vec::new();
1905        let mut routed_to_legacy = Vec::with_capacity(events.len());
1906        for event in events {
1907            if legacy_resident.contains(&event.id) {
1908                routed_to_legacy.push(true);
1909                legacy_rows.push(event);
1910            } else {
1911                routed_to_legacy.push(false);
1912                lane_rows.push(event);
1913            }
1914        }
1915        let legacy_expected = legacy_rows.len();
1916        let lane_expected = lane_rows.len();
1917        let legacy_result = self.legacy.append_events_idempotent(legacy_rows).await?;
1918        let lane_result = if lane_expected == 0 {
1919            IdempotentEventBatchResult { rows: Vec::new() }
1920        } else {
1921            self.lane.append_events_idempotent(lane_rows).await?
1922        };
1923        if legacy_result.rows.len() != legacy_expected || lane_result.rows.len() != lane_expected {
1924            return Err(StorageError::Driver {
1925                capability: khive_storage::StorageCapability::Events,
1926                operation: "append_events_idempotent".into(),
1927                source: format!(
1928                    "idempotent sub-batch result length mismatch: legacy {}/{}, lane {}/{}",
1929                    legacy_result.rows.len(),
1930                    legacy_expected,
1931                    lane_result.rows.len(),
1932                    lane_expected
1933                )
1934                .into(),
1935            });
1936        }
1937        let mut legacy_iter = legacy_result.rows.into_iter();
1938        let mut lane_iter = lane_result.rows.into_iter();
1939        let rows = routed_to_legacy
1940            .into_iter()
1941            .map(|to_legacy| {
1942                if to_legacy {
1943                    legacy_iter.next().expect("length checked above")
1944                } else {
1945                    lane_iter.next().expect("length checked above")
1946                }
1947            })
1948            .collect();
1949        Ok(IdempotentEventBatchResult { rows })
1950    }
1951
1952    fn supports_idempotent_audit_batch(&self) -> bool {
1953        // The legacy-resident probe above routes retried rows through the
1954        // legacy store's idempotent path, so the split supports the audit
1955        // batch only when BOTH stores do.
1956        self.lane.supports_idempotent_audit_batch() && self.legacy.supports_idempotent_audit_batch()
1957    }
1958}
1959
1960#[cfg(test)]
1961mod tests {
1962    use super::*;
1963
1964    // The full daemon/forwarding test suite for this module lands with the
1965    // final slice of this series; these tests cover the naming and
1966    // classification contracts the module itself defines.
1967
1968    #[test]
1969    fn sidecar_name_derives_from_the_full_file_name() {
1970        let path = events_db_path_beside(Path::new("/data/khive.db"));
1971        assert!(path.ends_with("khive.db.events.db"), "got {path:?}");
1972    }
1973
1974    #[test]
1975    fn databases_sharing_a_stem_get_distinct_sidecars() {
1976        let a = events_db_path_beside(Path::new("/data/a.db"));
1977        let b = events_db_path_beside(Path::new("/data/a.sqlite"));
1978        assert_ne!(a, b, "a.db and a.sqlite must not share an event plane");
1979        assert!(a.ends_with("a.db.events.db"), "got {a:?}");
1980        assert!(b.ends_with("a.sqlite.events.db"), "got {b:?}");
1981    }
1982
1983    #[cfg(unix)]
1984    #[test]
1985    fn directory_aliases_resolve_to_one_sidecar() {
1986        let dir = tempfile::tempdir().unwrap();
1987        let real = dir.path().join("real");
1988        std::fs::create_dir(&real).unwrap();
1989        let alias = dir.path().join("alias");
1990        std::os::unix::fs::symlink(&real, &alias).unwrap();
1991        assert_eq!(
1992            events_db_path_beside(&real.join("khive.db")),
1993            events_db_path_beside(&alias.join("khive.db")),
1994            "a symlinked spelling of one directory must not mint a second sidecar"
1995        );
1996    }
1997
1998    #[cfg(unix)]
1999    #[test]
2000    fn final_component_aliases_of_one_database_share_one_sidecar() {
2001        // Backend identity canonicalizes the whole database path, so a
2002        // final-component symlink alias is the same database; its sidecar
2003        // and socket must be the same too, or a process opening one spelling
2004        // writes event rows a process opening the other never reads.
2005        let dir = tempfile::tempdir().unwrap();
2006        let real = dir.path().join("real.db");
2007        std::fs::write(&real, b"").unwrap();
2008        let alias = dir.path().join("alias.db");
2009        std::os::unix::fs::symlink(&real, &alias).unwrap();
2010        let real_sidecar = events_db_path_beside(&real);
2011        let alias_sidecar = events_db_path_beside(&alias);
2012        assert_eq!(
2013            real_sidecar, alias_sidecar,
2014            "a symlink alias of one database file must not mint a second event store"
2015        );
2016        assert_eq!(
2017            events_socket_path_beside(&real_sidecar),
2018            events_socket_path_beside(&alias_sidecar),
2019        );
2020        // Control: a genuinely distinct database in the same directory keeps
2021        // its own sidecar — resolution must not collapse different files.
2022        let other = dir.path().join("other.db");
2023        std::fs::write(&other, b"").unwrap();
2024        assert_ne!(events_db_path_beside(&other), real_sidecar);
2025    }
2026
2027    #[cfg(unix)]
2028    #[test]
2029    fn dangling_alias_derives_the_target_sidecar_on_cold_start() {
2030        // Event-path derivation runs before backend creation, so the alias
2031        // can be consulted while its target does not exist yet — the first
2032        // open through the alias is what creates the target. A dangling
2033        // link must therefore already derive the TARGET's sidecar, or an
2034        // alias-first cold start and a later target-spelled process split
2035        // the event store between them.
2036        let dir = tempfile::tempdir().unwrap();
2037        let real = dir.path().join("real.db");
2038        let alias = dir.path().join("alias.db");
2039        std::os::unix::fs::symlink(&real, &alias).unwrap();
2040        // Neither real.db nor its sidecar exists at this point.
2041        assert_eq!(
2042            events_db_path_beside(&alias),
2043            events_db_path_beside(&real),
2044            "a dangling alias must derive the same sidecar its target will use"
2045        );
2046        // Chain: a link to a link resolves all the way down.
2047        let chain = dir.path().join("chain.db");
2048        std::os::unix::fs::symlink(&alias, &chain).unwrap();
2049        assert_eq!(events_db_path_beside(&chain), events_db_path_beside(&real));
2050        // Control: a distinct nonexistent file still gets its own sidecar.
2051        assert_ne!(
2052            events_db_path_beside(&dir.path().join("unrelated.db")),
2053            events_db_path_beside(&real)
2054        );
2055    }
2056
2057    #[cfg(unix)]
2058    #[test]
2059    fn planted_symlink_at_the_sidecar_path_is_refused() {
2060        // The sidecar path is derived, never user-chosen, so a pre-existing
2061        // symlink there is a planted redirect: following it would tighten
2062        // permissions on and write event rows into the link's target.
2063        let dir = tempfile::tempdir().unwrap();
2064        let victim = dir.path().join("victim.txt");
2065        std::fs::write(&victim, b"victim-bytes").unwrap();
2066        let mode_before = victim.metadata().unwrap().permissions();
2067        let sidecar = dir.path().join("khive.db.events.db");
2068        std::os::unix::fs::symlink(&victim, &sidecar).unwrap();
2069        let err = match direct_backend_for(&sidecar) {
2070            Ok(_) => panic!("a planted symlink at the sidecar path must be refused"),
2071            Err(e) => e,
2072        };
2073        assert!(err.to_string().contains("symlink"), "got: {err}");
2074        // The link's target is untouched: content intact, mode not tightened.
2075        assert_eq!(std::fs::read(&victim).unwrap(), b"victim-bytes");
2076        assert_eq!(victim.metadata().unwrap().permissions(), mode_before);
2077        // A dangling link is refused too, and nothing is created at its
2078        // target — without the refusal, the open itself would mint the
2079        // redirect target.
2080        let ghost_target = dir.path().join("ghost.db");
2081        let dangling = dir.path().join("other.db.events.db");
2082        std::os::unix::fs::symlink(&ghost_target, &dangling).unwrap();
2083        assert!(direct_backend_for(&dangling).is_err());
2084        assert!(
2085            !ghost_target.exists(),
2086            "refusal must not create the redirect target"
2087        );
2088        // Control: a regular path proceeds and the backend opens.
2089        let regular = dir.path().join("plain.db.events.db");
2090        direct_backend_for(&regular).expect("regular sidecar path must open");
2091        assert!(regular.exists());
2092    }
2093
2094    #[cfg(unix)]
2095    #[test]
2096    fn deep_symlink_chains_within_the_kernel_bound_derive_the_target_sidecar() {
2097        // Linux resolves up to 40 links per lookup; a 35-hop dangling chain
2098        // therefore opens fine, so it must ALSO derive the target's sidecar
2099        // — a resolver stopping short would split audit records between the
2100        // alias and target spellings of one database.
2101        let dir = tempfile::tempdir().unwrap();
2102        let real = dir.path().join("real.db");
2103        let mut prev = real.clone();
2104        for i in 0..35 {
2105            let link = dir.path().join(format!("hop{i}.db"));
2106            std::os::unix::fs::symlink(&prev, &link).unwrap();
2107            prev = link;
2108        }
2109        assert_eq!(
2110            events_db_path_beside(&prev),
2111            events_db_path_beside(&real),
2112            "a 35-hop dangling chain must derive the target's sidecar"
2113        );
2114        // Control: an unrelated dangling path still derives its own name.
2115        assert_ne!(
2116            events_db_path_beside(&dir.path().join("unrelated.db")),
2117            events_db_path_beside(&real)
2118        );
2119    }
2120
2121    #[cfg(unix)]
2122    #[test]
2123    fn untrusted_parent_directory_is_refused() {
2124        use std::os::unix::fs::PermissionsExt;
2125        // SQLite's open is path-based: the only sound defense against a
2126        // component swapped after validation is refusing directories other
2127        // local users can write. A group/other-writable parent is refused
2128        // before any open; an owner-only parent proceeds.
2129        let dir = tempfile::tempdir().unwrap();
2130        let open_dir = dir.path().join("shared");
2131        std::fs::create_dir(&open_dir).unwrap();
2132        std::fs::set_permissions(&open_dir, std::fs::Permissions::from_mode(0o777)).unwrap();
2133        let refused = direct_backend_for(&open_dir.join("khive.db.events.db"));
2134        match refused {
2135            Ok(_) => panic!("a world-writable events directory must be refused"),
2136            Err(e) => assert!(
2137                e.to_string().contains("untrusted directory"),
2138                "refusal must name the directory trust rule: {e}"
2139            ),
2140        }
2141        // Control: an owner-only sibling directory passes the same gate.
2142        let safe_dir = dir.path().join("owned");
2143        std::fs::create_dir(&safe_dir).unwrap();
2144        std::fs::set_permissions(&safe_dir, std::fs::Permissions::from_mode(0o700)).unwrap();
2145        direct_backend_for(&safe_dir.join("khive.db.events.db"))
2146            .expect("an owner-only events directory must open");
2147    }
2148
2149    #[cfg(unix)]
2150    #[test]
2151    fn harden_refuses_a_sidecar_symlink_at_use_time() {
2152        use std::os::unix::fs::PermissionsExt;
2153        // The hardening step must be coupled to its validation: it opens
2154        // each target with O_NOFOLLOW and chmods the returned handle, so a
2155        // symlink present AT USE TIME is refused by the open itself — no
2156        // lstat-then-chmod window — and the link's target keeps its mode.
2157        let dir = tempfile::tempdir().unwrap();
2158        let victim = dir.path().join("victim.txt");
2159        std::fs::write(&victim, b"v").unwrap();
2160        std::fs::set_permissions(&victim, std::fs::Permissions::from_mode(0o644)).unwrap();
2161        let db = dir.path().join("db.events.db");
2162        std::fs::write(&db, b"").unwrap();
2163        let mut wal = db.as_os_str().to_os_string();
2164        wal.push("-wal");
2165        std::os::unix::fs::symlink(&victim, PathBuf::from(&wal)).unwrap();
2166        let result = harden_events_db_sidecars(&db);
2167        assert!(
2168            result.is_err(),
2169            "a symlinked -wal must be refused at hardening time"
2170        );
2171        let mode = victim.metadata().unwrap().permissions().mode() & 0o7777;
2172        assert_eq!(mode, 0o644, "the link's target must keep its mode");
2173    }
2174
2175    #[cfg(unix)]
2176    #[test]
2177    fn lock_symlink_is_refused_and_target_untouched() {
2178        use std::os::unix::fs::PermissionsExt;
2179        // The daemon lock open is pinned with O_NOFOLLOW and chmods its own
2180        // handle: a symlink planted at the lock name must refuse the guard,
2181        // and the link's target must keep its inode content and mode. A
2182        // plain lock in the same directory must still acquire — the control
2183        // that proves the refusal is the symlink, not a broken guard.
2184        let dir = tempfile::tempdir().unwrap();
2185        let victim = dir.path().join("victim.txt");
2186        std::fs::write(&victim, b"v").unwrap();
2187        std::fs::set_permissions(&victim, std::fs::Permissions::from_mode(0o644)).unwrap();
2188        let socket = dir.path().join("events.sock");
2189        std::os::unix::fs::symlink(&victim, socket.with_extension("lock")).unwrap();
2190        assert!(
2191            try_acquire_events_daemon_guard(&socket).is_none(),
2192            "a symlinked lock entry must refuse the guard"
2193        );
2194        let mode = victim.metadata().unwrap().permissions().mode() & 0o7777;
2195        assert_eq!(mode, 0o644, "the symlink's target must keep its mode");
2196        assert_eq!(std::fs::read(&victim).unwrap(), b"v");
2197        let clean = dir.path().join("clean.sock");
2198        assert!(
2199            try_acquire_events_daemon_guard(&clean).is_some(),
2200            "a plain lock path in the same directory must still acquire"
2201        );
2202    }
2203
2204    #[cfg(unix)]
2205    #[test]
2206    fn planted_wal_symlink_is_refused_before_open() {
2207        // SQLite creates `-wal` without O_EXCL, so a planted `-wal` symlink
2208        // would redirect WAL writes; admission checks the sidecar suffixes
2209        // before the database is ever created or opened.
2210        let dir = tempfile::tempdir().unwrap();
2211        let victim = dir.path().join("victim.txt");
2212        std::fs::write(&victim, b"w").unwrap();
2213        let sidecar = dir.path().join("db.events.db");
2214        let mut wal = sidecar.as_os_str().to_os_string();
2215        wal.push("-wal");
2216        std::os::unix::fs::symlink(&victim, PathBuf::from(&wal)).unwrap();
2217        assert!(direct_backend_for(&sidecar).is_err());
2218        assert!(
2219            !sidecar.exists(),
2220            "refusal must precede creation of the events database"
2221        );
2222    }
2223
2224    #[test]
2225    fn socket_derives_beside_the_sidecar() {
2226        let db = events_db_path_beside(Path::new("/data/khive.db"));
2227        let socket = events_socket_path_beside(&db);
2228        assert!(socket.ends_with("khive.db.events.sock"), "got {socket:?}");
2229    }
2230
2231    #[test]
2232    fn bare_relative_main_db_yields_an_absolute_sidecar() {
2233        // A bare file name has `Some("")` for a parent; every downstream
2234        // consumer (lock parenting, socket-dir trust validation) needs a
2235        // real directory, so the sidecar path must come back anchored.
2236        let path = events_db_path_beside(Path::new("khive.db"));
2237        assert!(path.is_absolute(), "got relative {path:?}");
2238        assert!(path.ends_with("khive.db.events.db"), "got {path:?}");
2239    }
2240
2241    #[cfg(unix)]
2242    #[tokio::test]
2243    async fn over_cap_query_page_limit_is_refused_before_materialization() {
2244        // `PageRequest.limit` arrives client-controlled off the wire; the
2245        // daemon must refuse an over-cap page with a typed, actionable error
2246        // instead of materializing and serializing it — and must never
2247        // silently clamp, or the short page reads as end-of-data.
2248        let backend = Arc::new(StorageBackend::memory().unwrap());
2249        let stores: NamespaceStores =
2250            Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
2251        let request = |limit: u32| EventsRequest::QueryEvents {
2252            protocol_version: EVENTS_PROTOCOL_VERSION,
2253            namespace: "local".to_string(),
2254            filter: EventFilter::default(),
2255            page: PageRequest { offset: 0, limit },
2256        };
2257        let refused =
2258            dispatch_events_request(request(MAX_QUERY_EVENTS_PAGE_ROWS + 1), &backend, &stores)
2259                .await;
2260        match refused {
2261            EventsResponse::Error {
2262                message,
2263                retryable,
2264                writer_task_state,
2265            } => {
2266                assert!(
2267                    message.contains(&MAX_QUERY_EVENTS_PAGE_ROWS.to_string()),
2268                    "refusal must name the cap: {message}"
2269                );
2270                assert!(!retryable, "an over-cap page is not transient");
2271                assert!(writer_task_state.is_none());
2272            }
2273            other => panic!("over-cap query must be refused, got {other:?}"),
2274        }
2275        // Control: a request AT the cap passes admission and reaches the
2276        // store — an empty page, not a refusal.
2277        let at_cap =
2278            dispatch_events_request(request(MAX_QUERY_EVENTS_PAGE_ROWS), &backend, &stores).await;
2279        assert!(
2280            matches!(at_cap, EventsResponse::Pageful { .. }),
2281            "at-cap query must reach the store, got {at_cap:?}"
2282        );
2283    }
2284
2285    #[cfg(unix)]
2286    #[test]
2287    fn side_effects_unknown_state_crosses_the_wire_as_writer_task_state() {
2288        let response = storage_error_response(&StorageError::WriterTaskTerminated {
2289            request_state: khive_storage::WriterTaskRequestState::SideEffectsUnknown,
2290        });
2291        assert!(
2292            matches!(
2293                response,
2294                EventsResponse::Error {
2295                    writer_task_state: Some(WireWriterTaskState::SideEffectsUnknown),
2296                    ..
2297                }
2298            ),
2299            "an unknown-commit-state termination must carry its state on the wire"
2300        );
2301        // Any other refusal must NOT carry a writer state.
2302        let busy = storage_error_response(&StorageError::WriterTaskBusy { timeout_ms: 5 });
2303        assert!(matches!(
2304            busy,
2305            EventsResponse::Error {
2306                writer_task_state: None,
2307                ..
2308            }
2309        ));
2310    }
2311
2312    #[test]
2313    fn error_frames_without_writer_state_still_parse() {
2314        // `writer_task_state` is `#[serde(default)]`: an Error frame for a
2315        // non-writer failure (parse error, version refusal) omits it and the
2316        // client must read `None`, not fail the parse.
2317        let bytes = br#"{"kind":"error","message":"boom","retryable":true}"#;
2318        let parsed: EventsResponse = serde_json::from_slice(bytes).expect("stateless frame parses");
2319        assert!(matches!(
2320            parsed,
2321            EventsResponse::Error {
2322                retryable: true,
2323                writer_task_state: None,
2324                ..
2325            }
2326        ));
2327    }
2328
2329    fn split_retry_event(verb: &str) -> Event {
2330        Event::new(
2331            "test",
2332            verb,
2333            khive_types::EventKind::RecallExecuted,
2334            khive_types::SubstrateKind::Note,
2335            "agent:test",
2336        )
2337    }
2338
2339    fn store_pair(dir: &Path) -> (Arc<dyn EventStore>, Arc<dyn EventStore>) {
2340        let legacy = direct_backend_for(&dir.join("legacy.db"))
2341            .expect("legacy backend")
2342            .events_for_namespace("test")
2343            .expect("legacy store");
2344        let lane = direct_backend_for(&dir.join("lane.db"))
2345            .expect("lane backend")
2346            .events_for_namespace("test")
2347            .expect("lane store");
2348        (legacy, lane)
2349    }
2350
2351    /// A retried audit batch whose first attempt landed on the legacy store
2352    /// before the cutover must be answered by the legacy store's comparison,
2353    /// never re-inserted into the lane — otherwise merged reads and counts
2354    /// double-count the id.
2355    #[tokio::test]
2356    async fn idempotent_retry_of_legacy_resident_rows_does_not_duplicate() {
2357        use khive_storage::event::EventAppendDisposition;
2358
2359        let dir = tempfile::tempdir().unwrap();
2360        let (legacy, lane) = store_pair(dir.path());
2361
2362        let resident = split_retry_event("recall");
2363        legacy
2364            .append_events_idempotent(vec![resident.clone()])
2365            .await
2366            .expect("pre-cutover landing");
2367
2368        let split = SplitEventStore::new(Arc::clone(&legacy), Arc::clone(&lane));
2369        let fresh = split_retry_event("search");
2370        let result = split
2371            .append_events_idempotent(vec![resident.clone(), fresh.clone()])
2372            .await
2373            .expect("mixed retry batch");
2374        assert_eq!(
2375            result.rows,
2376            vec![
2377                EventAppendDisposition::AlreadyPresentIdentical,
2378                EventAppendDisposition::Inserted,
2379            ],
2380            "input order must be preserved across the two sub-batches"
2381        );
2382        assert_eq!(
2383            lane.count_events(EventFilter::default()).await.unwrap(),
2384            1,
2385            "the legacy-resident row must not reach the lane"
2386        );
2387        assert_eq!(
2388            split.count_events(EventFilter::default()).await.unwrap(),
2389            2,
2390            "merged count must not double-count the retried id"
2391        );
2392
2393        // A conflicting retry of the resident row reports the conflict from
2394        // the legacy comparison and inserts nowhere.
2395        let mut mutated = resident.clone();
2396        mutated.verb = "other".to_string();
2397        let conflict = split
2398            .append_events_idempotent(vec![mutated])
2399            .await
2400            .expect("conflicting retry");
2401        assert_eq!(
2402            conflict.rows,
2403            vec![EventAppendDisposition::IdentityConflict]
2404        );
2405        assert_eq!(split.count_events(EventFilter::default()).await.unwrap(), 2);
2406    }
2407
2408    /// The merged plane refuses to materialize an unbounded offset prefix;
2409    /// the refusal names the cursor remedy, and the bound itself is usable.
2410    #[tokio::test]
2411    async fn merged_offset_window_is_bounded() {
2412        let dir = tempfile::tempdir().unwrap();
2413        let (legacy, lane) = store_pair(dir.path());
2414        let split = SplitEventStore::new(legacy, lane);
2415
2416        let err = split
2417            .query_events(
2418                EventFilter::default(),
2419                PageRequest {
2420                    offset: SplitEventStore::MAX_MERGED_WINDOW_ROWS,
2421                    limit: 1,
2422                },
2423            )
2424            .await
2425            .expect_err("a window past the bound must be refused");
2426        assert!(
2427            matches!(err, StorageError::InvalidInput { .. }),
2428            "got {err:?}"
2429        );
2430        assert!(
2431            err.to_string().contains("before"),
2432            "the refusal must name the cursor remedy: {err}"
2433        );
2434
2435        let at_bound = split
2436            .query_events(
2437                EventFilter::default(),
2438                PageRequest {
2439                    offset: SplitEventStore::MAX_MERGED_WINDOW_ROWS - 1,
2440                    limit: 1,
2441                },
2442            )
2443            .await
2444            .expect("a window at the bound is admitted");
2445        assert!(at_bound.items.is_empty());
2446    }
2447
2448    #[cfg(unix)]
2449    #[tokio::test]
2450    async fn stateless_non_retryable_error_maps_terminal_not_writer_terminated() {
2451        // The terminal arm of the client mapping, pinned deliberately: an
2452        // Error frame with `retryable: false` and NO `writer_task_state` is
2453        // a non-writer failure (parse error, version refusal) and must map
2454        // to terminal `InvalidInput` — never to `WriterTaskTerminated`
2455        // (there is no state to reconstruct) and never to the retryable
2456        // `Pool` arm (retrying a version refusal cannot succeed).
2457        let client = EventsSplitClient::new(std::path::PathBuf::from("/tmp/never-bound.sock"))
2458            .expect("client builds");
2459        let store = ForwardingEventStore::new("test", client);
2460        let bytes = br#"{"kind":"error","message":"refused","retryable":false}"#;
2461        let parsed: EventsResponse = serde_json::from_slice(bytes).expect("frame parses");
2462        assert!(matches!(
2463            store.unexpected("append", parsed),
2464            StorageError::InvalidInput { .. }
2465        ));
2466        // Control: the same non-retryable frame WITH a writer state must
2467        // take the reconstruction arm instead — proving the terminal arm
2468        // above is selected by the field's absence, not by `retryable`.
2469        let with_state = br#"{"kind":"error","message":"died","retryable":false,"writer_task_state":"side_effects_unknown"}"#;
2470        let parsed: EventsResponse =
2471            serde_json::from_slice(with_state).expect("stateful frame parses");
2472        assert!(matches!(
2473            store.unexpected("append", parsed),
2474            StorageError::WriterTaskTerminated { .. }
2475        ));
2476    }
2477
2478    #[cfg(unix)]
2479    #[tokio::test]
2480    async fn writer_task_states_cross_the_wire_verbatim() {
2481        // The ADR-133 retry classifier decides per-state, so every
2482        // WriterTaskTerminated state must survive server → wire → client
2483        // reconstruction exactly. Before this field existed, NotStarted and
2484        // TransactionRolledBack crossed as a generic non-retryable refusal
2485        // and came back as terminal InvalidInput — a broken retry contract.
2486        use khive_storage::WriterTaskRequestState as S;
2487        let dir = tempfile::tempdir().unwrap();
2488        let client =
2489            EventsSplitClient::new(dir.path().join("never-bound.sock")).expect("client builds");
2490        let store = ForwardingEventStore::new("test", client);
2491        for state in [
2492            S::NotStarted,
2493            S::TransactionRolledBack,
2494            S::SideEffectsUnknown,
2495        ] {
2496            let response = storage_error_response(&StorageError::WriterTaskTerminated {
2497                request_state: state,
2498            });
2499            // Round-trip through serde like the socket does.
2500            let bytes = serde_json::to_vec(&response).unwrap();
2501            let parsed: EventsResponse = serde_json::from_slice(&bytes).unwrap();
2502            let err = store.unexpected("append_events_idempotent", parsed);
2503            assert!(
2504                matches!(
2505                    err,
2506                    StorageError::WriterTaskTerminated { request_state } if request_state == state
2507                ),
2508                "state {state:?} did not survive the socket: got {err:?}"
2509            );
2510        }
2511        // Control: a non-writer-task refusal must NOT carry the field.
2512        let busy = storage_error_response(&StorageError::WriterTaskBusy { timeout_ms: 5 });
2513        assert!(matches!(
2514            busy,
2515            EventsResponse::Error {
2516                writer_task_state: None,
2517                ..
2518            }
2519        ));
2520    }
2521
2522    #[cfg(unix)]
2523    #[test]
2524    fn direct_backend_hardens_preexisting_db_and_sidecars() {
2525        use std::os::unix::fs::PermissionsExt;
2526        let dir = tempfile::tempdir().unwrap();
2527        let db = dir.path().join("pre-existing.events.db");
2528        let wal = dir.path().join("pre-existing.events.db-wal");
2529        std::fs::write(&db, b"").unwrap();
2530        std::fs::write(&wal, b"").unwrap();
2531        for path in [&db, &wal] {
2532            std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o644)).unwrap();
2533        }
2534        // Control: the loose mode is really in place before the open.
2535        assert_eq!(
2536            std::fs::metadata(&db).unwrap().permissions().mode() & 0o777,
2537            0o644
2538        );
2539        direct_backend_for(&db).expect("writable open succeeds");
2540        for path in [&db, &wal] {
2541            assert_eq!(
2542                std::fs::metadata(path).unwrap().permissions().mode() & 0o777,
2543                0o600,
2544                "pre-existing {} must be tightened to owner-only",
2545                path.display()
2546            );
2547        }
2548    }
2549
2550    #[cfg(unix)]
2551    #[test]
2552    fn namespace_store_cache_is_bounded_and_trim_normalized() {
2553        let backend = StorageBackend::memory().unwrap();
2554        let stores: NamespaceStores =
2555            Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
2556        // cap=1: the first namespace caches; the second evicts it and takes
2557        // the slot — bounded, and never condemned to per-request rebuilds.
2558        namespace_store_with_cap(&backend, &stores, "alpha", 1).expect("first store");
2559        namespace_store_with_cap(&backend, &stores, "beta", 1).expect("second store evicts");
2560        {
2561            let map = stores.lock().unwrap();
2562            assert_eq!(map.len(), 1, "cache must not grow past its cap");
2563            assert!(
2564                map.contains_key("beta"),
2565                "the newest namespace must be admitted at the cap"
2566            );
2567        }
2568        // Trimmed spelling of a cached namespace hits the same entry rather
2569        // than minting a duplicate handle (the backend trims namespaces).
2570        namespace_store_with_cap(&backend, &stores, "beta", 1).expect("cached beta");
2571        namespace_store_with_cap(&backend, &stores, "  beta  ", 1).expect("trimmed spelling");
2572        {
2573            let map = stores.lock().unwrap();
2574            assert_eq!(
2575                map.len(),
2576                1,
2577                "spellings of one namespace must share one entry"
2578            );
2579            assert!(map.contains_key("beta"), "trimmed key is the cache key");
2580        }
2581    }
2582
2583    #[cfg(unix)]
2584    #[tokio::test]
2585    async fn frame_budget_admits_before_allocating_and_releases_after() {
2586        let (mut client, mut server) = UnixStream::pair().expect("socketpair");
2587        let budget = Arc::new(tokio::sync::Semaphore::new(1024));
2588        let payload = vec![7u8; 100];
2589        write_frame(&mut client, &payload).await.expect("write");
2590        let (bytes, permit) = read_frame_budgeted(&mut server, &budget)
2591            .await
2592            .expect("budgeted read");
2593        assert_eq!(bytes, payload);
2594        // The permit holds exactly the frame's bytes against the budget…
2595        assert_eq!(budget.available_permits(), 1024 - 100);
2596        // …and releases them when the request lifecycle ends.
2597        drop(permit);
2598        assert_eq!(budget.available_permits(), 1024);
2599        // A frame past the per-frame cap is refused before admission: the
2600        // budget is untouched, so an oversized declaration cannot starve it.
2601        let mut oversized = Vec::from(((crate::daemon::MAX_FRAME_BYTES + 1) as u32).to_be_bytes());
2602        oversized.extend_from_slice(&[0u8; 8]);
2603        use tokio::io::AsyncWriteExt;
2604        client.write_all(&oversized).await.expect("raw prefix");
2605        let err = read_frame_budgeted(&mut server, &budget)
2606            .await
2607            .expect_err("oversized declaration must refuse");
2608        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
2609        assert_eq!(
2610            budget.available_permits(),
2611            1024,
2612            "refusal must not consume budget"
2613        );
2614    }
2615
2616    #[cfg(unix)]
2617    #[tokio::test]
2618    async fn dispatch_rejects_invalid_wire_namespace() {
2619        let backend = Arc::new(StorageBackend::memory().unwrap());
2620        let stores: NamespaceStores =
2621            Arc::new(std::sync::Mutex::new(std::collections::HashMap::new()));
2622        // An attacker-shaped namespace: far past the 256-byte Namespace
2623        // bound. Must be refused as a typed non-retryable error before it
2624        // can become a cache key or a store.
2625        let huge = "n".repeat(64 * 1024);
2626        let response = dispatch_events_request(
2627            EventsRequest::CountEvents {
2628                protocol_version: EVENTS_PROTOCOL_VERSION,
2629                namespace: huge,
2630                filter: Default::default(),
2631            },
2632            &backend,
2633            &stores,
2634        )
2635        .await;
2636        assert!(
2637            matches!(
2638                &response,
2639                EventsResponse::Error {
2640                    retryable: false,
2641                    writer_task_state: None,
2642                    ..
2643                }
2644            ),
2645            "oversized namespace must be a typed refusal, got {response:?}"
2646        );
2647        assert_eq!(
2648            stores.lock().unwrap().len(),
2649            0,
2650            "a rejected namespace must never enter the cache"
2651        );
2652        // Control: a valid namespace on the same dispatch path succeeds.
2653        let ok = dispatch_events_request(
2654            EventsRequest::CountEvents {
2655                protocol_version: EVENTS_PROTOCOL_VERSION,
2656                namespace: "local".to_string(),
2657                filter: Default::default(),
2658            },
2659            &backend,
2660            &stores,
2661        )
2662        .await;
2663        assert!(
2664            matches!(ok, EventsResponse::Count { .. }),
2665            "valid namespace must dispatch, got {ok:?}"
2666        );
2667    }
2668
2669    #[cfg(unix)]
2670    #[tokio::test]
2671    async fn shutdown_counts_and_logs_queued_and_in_flight_batches_as_dropped() {
2672        // ADR-170 requires shutdown drops to be visible via counters/logs.
2673        // Both `run_forwarder` shutdown arms must therefore count every
2674        // batch it loses: the one parked mid-delivery against a hung peer,
2675        // and every batch still sitting in the queue behind it. Drives
2676        // `run_forwarder` directly with a local `CancellationToken` (rather
2677        // than through `EventsSplitClient`, which wires the process-wide
2678        // `daemon_shutdown_token()` singleton) so cancelling shutdown here
2679        // cannot leak into other tests sharing the process.
2680        let dir = tempfile::tempdir().unwrap();
2681        let socket = dir.path().join("hung-shutdown.sock");
2682        let listener = tokio::net::UnixListener::bind(&socket).unwrap();
2683        // Accept and hold the connection open without ever reading or
2684        // replying, so the first delivery blocks in `deliver_batch`'s
2685        // read_frame until shutdown cancels it.
2686        let _server = tokio::spawn(async move {
2687            let mut held = Vec::new();
2688            loop {
2689                if let Ok((stream, _)) = listener.accept().await {
2690                    held.push(stream);
2691                }
2692            }
2693        });
2694
2695        let (tx, rx) = tokio::sync::mpsc::channel::<(String, Vec<Event>)>(8);
2696        let counters = Arc::new(ForwardingCounters::default());
2697        let outage_logged = Arc::new(AtomicBool::new(false));
2698        let shutdown = tokio_util::sync::CancellationToken::new();
2699
2700        let forwarder = tokio::spawn(run_forwarder(
2701            socket,
2702            rx,
2703            Arc::clone(&counters),
2704            outage_logged,
2705            Duration::from_secs(30),
2706            shutdown.clone(),
2707        ));
2708
2709        fn probe_event(tag: &str) -> Event {
2710            Event::new(
2711                "test",
2712                tag,
2713                khive_types::EventKind::Audit,
2714                khive_types::SubstrateKind::Event,
2715                "tester",
2716            )
2717        }
2718
2719        tx.try_send(("test".to_string(), vec![probe_event("in-flight")]))
2720            .expect("queue has room for the in-flight batch");
2721        // No externally observable "delivery started" signal exists short of
2722        // instrumenting the forwarder; a short sleep reliably lands inside
2723        // the delivery `select!` arm before shutdown cancels it, given the
2724        // 30s delivery timeout has no chance to fire first.
2725        tokio::time::sleep(Duration::from_millis(200)).await;
2726
2727        tx.try_send((
2728            "test".to_string(),
2729            vec![probe_event("queued-1"), probe_event("queued-2")],
2730        ))
2731        .expect("queue has room for the first queued batch");
2732        tx.try_send(("test".to_string(), vec![probe_event("queued-3")]))
2733            .expect("queue has room for the second queued batch");
2734
2735        shutdown.cancel();
2736
2737        tokio::time::timeout(Duration::from_secs(5), forwarder)
2738            .await
2739            .expect("forwarder must exit promptly on shutdown")
2740            .expect("forwarder task must not panic");
2741
2742        assert_eq!(
2743            counters.dropped_batches.load(Ordering::Relaxed),
2744            3,
2745            "the in-flight batch and both queued batches must all be counted as dropped"
2746        );
2747        assert_eq!(
2748            counters.dropped_events.load(Ordering::Relaxed),
2749            4,
2750            "1 in-flight + 2 + 1 queued events must all be counted as dropped"
2751        );
2752        assert_eq!(
2753            counters.forwarded_batches.load(Ordering::Relaxed),
2754            0,
2755            "a hung peer never acknowledges anything in this test"
2756        );
2757    }
2758
2759    #[cfg(unix)]
2760    #[tokio::test]
2761    async fn shutdown_during_backoff_drains_queued_batches() {
2762        // A failed delivery sends the forwarder into `FORWARDER_BACKOFF`
2763        // before it loops back to `rx.recv()`. Shutdown arriving while it
2764        // sits in that backoff sleep must drain and count whatever is left
2765        // in the queue, not just break out silently. Points at a socket path
2766        // with no listener bound so the very first delivery attempt fails
2767        // fast (connect refused) rather than needing a hung peer.
2768        let dir = tempfile::tempdir().unwrap();
2769        let socket = dir.path().join("no-listener.sock");
2770
2771        let (tx, rx) = tokio::sync::mpsc::channel::<(String, Vec<Event>)>(8);
2772        let counters = Arc::new(ForwardingCounters::default());
2773        let outage_logged = Arc::new(AtomicBool::new(false));
2774        let shutdown = tokio_util::sync::CancellationToken::new();
2775
2776        let forwarder = tokio::spawn(run_forwarder(
2777            socket,
2778            rx,
2779            Arc::clone(&counters),
2780            outage_logged,
2781            Duration::from_secs(30),
2782            shutdown.clone(),
2783        ));
2784
2785        fn probe_event(tag: &str) -> Event {
2786            Event::new(
2787                "test",
2788                tag,
2789                khive_types::EventKind::Audit,
2790                khive_types::SubstrateKind::Event,
2791                "tester",
2792            )
2793        }
2794
2795        tx.try_send(("test".to_string(), vec![probe_event("failed-delivery")]))
2796            .expect("queue has room for the first batch");
2797
2798        // Wait for the failed-delivery drop to land, which proves the
2799        // forwarder has moved on to the `FORWARDER_BACKOFF` sleep.
2800        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2801        loop {
2802            if counters.dropped_batches.load(Ordering::Relaxed) >= 1 {
2803                break;
2804            }
2805            assert!(
2806                tokio::time::Instant::now() < deadline,
2807                "forwarder never dropped the first (unreachable-daemon) batch"
2808            );
2809            tokio::time::sleep(Duration::from_millis(20)).await;
2810        }
2811
2812        tx.try_send((
2813            "test".to_string(),
2814            vec![
2815                probe_event("queued-during-backoff-1"),
2816                probe_event("queued-during-backoff-2"),
2817            ],
2818        ))
2819        .expect("queue has room for the batch queued during backoff");
2820
2821        shutdown.cancel();
2822
2823        tokio::time::timeout(Duration::from_secs(5), forwarder)
2824            .await
2825            .expect("forwarder must exit promptly on shutdown, not wait out the full backoff")
2826            .expect("forwarder task must not panic");
2827
2828        assert_eq!(
2829            counters.dropped_batches.load(Ordering::Relaxed),
2830            2,
2831            "the failed-delivery batch and the batch queued during backoff must both be counted as dropped"
2832        );
2833        assert_eq!(
2834            counters.dropped_events.load(Ordering::Relaxed),
2835            3,
2836            "1 failed-delivery + 2 queued-during-backoff events must all be counted as dropped"
2837        );
2838        assert_eq!(
2839            counters.forwarded_batches.load(Ordering::Relaxed),
2840            0,
2841            "no listener is bound, so nothing can ever be acknowledged"
2842        );
2843    }
2844
2845    #[cfg(unix)]
2846    #[tokio::test]
2847    async fn forwarder_abandons_hung_delivery() {
2848        // A daemon that accepts the connection and then never responds must
2849        // not park the forwarder forever: the delivery deadline fires, the
2850        // batch is dropped and counted, and the task stays live.
2851        let dir = tempfile::tempdir().unwrap();
2852        let socket = dir.path().join("hung.sock");
2853        let listener = tokio::net::UnixListener::bind(&socket).unwrap();
2854        // Accept and hold connections open without ever reading or replying.
2855        let _server = tokio::spawn(async move {
2856            let mut held = Vec::new();
2857            loop {
2858                if let Ok((stream, _)) = listener.accept().await {
2859                    held.push(stream);
2860                }
2861            }
2862        });
2863        let client = EventsSplitClient::new_with_queue_depth_and_delivery_timeout(
2864            socket.clone(),
2865            4,
2866            Duration::from_millis(100),
2867        )
2868        .expect("client builds");
2869        let event = Event::new(
2870            "test",
2871            "noop",
2872            khive_types::EventKind::Audit,
2873            khive_types::SubstrateKind::Event,
2874            "tester",
2875        );
2876        client.enqueue("test", vec![event]);
2877        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
2878        loop {
2879            if client.metrics().dropped_batches >= 1 {
2880                break;
2881            }
2882            assert!(
2883                tokio::time::Instant::now() < deadline,
2884                "forwarder never abandoned the hung delivery: {:?}",
2885                client.metrics()
2886            );
2887            tokio::time::sleep(Duration::from_millis(20)).await;
2888        }
2889    }
2890
2891    #[cfg(unix)]
2892    #[test]
2893    fn wire_retryability_follows_the_storage_classifier() {
2894        let busy = storage_error_response(&StorageError::WriterTaskBusy { timeout_ms: 5 });
2895        assert!(
2896            matches!(
2897                busy,
2898                EventsResponse::Error {
2899                    retryable: true,
2900                    ..
2901                }
2902            ),
2903            "transient writer contention must stay retryable across the socket"
2904        );
2905        // A terminated writer task is not retryable per the storage layer's
2906        // own classifier; the wire response must agree rather than widen it.
2907        let terminated = storage_error_response(&StorageError::WriterTaskTerminated {
2908            request_state: khive_storage::WriterTaskRequestState::SideEffectsUnknown,
2909        });
2910        assert!(
2911            matches!(
2912                terminated,
2913                EventsResponse::Error {
2914                    retryable: false,
2915                    ..
2916                }
2917            ),
2918            "a terminated writer must not be reported transient"
2919        );
2920    }
2921
2922    use khive_storage::event::EventAppendDisposition;
2923    use khive_types::{EventKind, SubstrateKind};
2924
2925    fn test_event(namespace: &str) -> Event {
2926        Event::new(
2927            namespace,
2928            "test.verb",
2929            EventKind::Audit,
2930            SubstrateKind::Event,
2931            "actor:test",
2932        )
2933    }
2934
2935    /// Boot a real events daemon on temp paths, poll until its socket accepts.
2936    #[cfg(unix)]
2937    async fn boot_daemon(dir: &tempfile::TempDir) -> (PathBuf, PathBuf) {
2938        let db = dir.path().join("events.db");
2939        let socket = dir.path().join("events.sock");
2940        let (db_clone, socket_clone) = (db.clone(), socket.clone());
2941        tokio::spawn(async move {
2942            let _ = run_events_daemon(&db_clone, &socket_clone).await;
2943        });
2944        for _ in 0..100 {
2945            if UnixStream::connect(&socket).await.is_ok() {
2946                return (db, socket);
2947            }
2948            tokio::time::sleep(Duration::from_millis(20)).await;
2949        }
2950        panic!("events daemon did not come up on {}", socket.display());
2951    }
2952
2953    /// The split store's routing contract: plain appends land ONLY in the
2954    /// legacy store (whose raw-SQL consumers depend on finding them there),
2955    /// the idempotent audit lane lands ONLY in the events lane, and reads
2956    /// merge both sides into one event plane.
2957    #[tokio::test]
2958    async fn split_store_routes_plain_to_legacy_idempotent_to_lane_and_merges_reads() {
2959        let dir = tempfile::tempdir().expect("tempdir");
2960        let legacy_backend =
2961            direct_backend_for(&dir.path().join("legacy.db")).expect("legacy backend");
2962        let lane_backend = direct_backend_for(&dir.path().join("lane.db")).expect("lane backend");
2963        let legacy = legacy_backend
2964            .events_for_namespace("local")
2965            .expect("legacy store");
2966        let lane = lane_backend
2967            .events_for_namespace("local")
2968            .expect("lane store");
2969        let split = SplitEventStore::new(Arc::clone(&legacy), Arc::clone(&lane));
2970
2971        let plain = test_event("local");
2972        let plain_id = plain.id;
2973        split.append_event(plain).await.expect("plain append");
2974
2975        let audit = test_event("local");
2976        let audit_id = audit.id;
2977        let result = split
2978            .append_events_idempotent(vec![audit])
2979            .await
2980            .expect("idempotent append");
2981        assert_eq!(result.rows, vec![EventAppendDisposition::Inserted]);
2982
2983        // Routing: each row is in exactly its own side.
2984        assert!(legacy
2985            .get_event(plain_id)
2986            .await
2987            .expect("legacy get")
2988            .is_some());
2989        assert!(lane.get_event(plain_id).await.expect("lane get").is_none());
2990        assert!(legacy
2991            .get_event(audit_id)
2992            .await
2993            .expect("legacy get")
2994            .is_none());
2995        assert!(lane.get_event(audit_id).await.expect("lane get").is_some());
2996
2997        // Merged reads observe one event plane whichever side holds the row.
2998        assert!(split
2999            .get_event(plain_id)
3000            .await
3001            .expect("split get")
3002            .is_some());
3003        assert!(split
3004            .get_event(audit_id)
3005            .await
3006            .expect("split get")
3007            .is_some());
3008        assert_eq!(
3009            split
3010                .count_events(EventFilter::default())
3011                .await
3012                .expect("split count"),
3013            2
3014        );
3015        let page = split
3016            .query_events(
3017                EventFilter::default(),
3018                PageRequest {
3019                    offset: 0,
3020                    limit: 10,
3021                },
3022            )
3023            .await
3024            .expect("split query");
3025        let ids: Vec<Uuid> = page.items.iter().map(|e| e.id).collect();
3026        assert!(ids.contains(&plain_id) && ids.contains(&audit_id));
3027
3028        // Windowing across the merge: page size 1 at offsets 0 and 1 yields
3029        // the two rows exactly once each.
3030        let mut seen = Vec::new();
3031        for offset in 0..2 {
3032            let page = split
3033                .query_events(EventFilter::default(), PageRequest { offset, limit: 1 })
3034                .await
3035                .expect("windowed query");
3036            assert_eq!(page.items.len(), 1);
3037            seen.push(page.items[0].id);
3038        }
3039        seen.sort();
3040        let mut expected = vec![plain_id, audit_id];
3041        expected.sort();
3042        assert_eq!(seen, expected);
3043    }
3044
3045    /// Mechanized co-residency guard: the amendment's load-bearing claim is
3046    /// that every class a raw-SQL consumer of the legacy `events` table
3047    /// depends on still LANDS in that table. Two of the three enumerated
3048    /// consumers (kg projection guard, graph-query union) fail silently if
3049    /// the classification drifts, so the classification is guarded here by
3050    /// the consumers' own access pattern — a raw SQL read against the legacy
3051    /// backend — rather than by enumeration. Arm 1 reddens if plain appends
3052    /// (the provenance/domain class) ever stop reaching the legacy table.
3053    /// Arm 2 is the boundary's other face and doubles as the positive
3054    /// control for the instrument: the same query DOES find lane-routed rows
3055    /// on the lane's own backend, so an empty arm-1 result could not be a
3056    /// broken query.
3057    #[tokio::test]
3058    async fn raw_sql_consumers_of_the_legacy_events_table_still_see_plain_appends() {
3059        use khive_storage::types::{SqlStatement, SqlValue};
3060
3061        let dir = tempfile::tempdir().expect("tempdir");
3062        let legacy_backend =
3063            direct_backend_for(&dir.path().join("legacy-guard.db")).expect("legacy backend");
3064        let lane_backend =
3065            direct_backend_for(&dir.path().join("lane-guard.db")).expect("lane backend");
3066        let split = SplitEventStore::new(
3067            legacy_backend
3068                .events_for_namespace("local")
3069                .expect("legacy store"),
3070            lane_backend
3071                .events_for_namespace("local")
3072                .expect("lane store"),
3073        );
3074
3075        // The schedule pack's creator-provenance write is a plain append; the
3076        // audit flusher's write is an idempotent batch. Route one of each.
3077        let provenance = test_event("local");
3078        let provenance_id = provenance.id;
3079        split.append_event(provenance).await.expect("plain append");
3080        let audit = test_event("local");
3081        let audit_id = audit.id;
3082        split
3083            .append_events_idempotent(vec![audit])
3084            .await
3085            .expect("idempotent append");
3086
3087        let count_by_raw_sql = |backend: Arc<StorageBackend>, id: Uuid| async move {
3088            let mut reader = backend.sql().reader().await.expect("sql reader");
3089            let rows = reader
3090                .query_all(SqlStatement {
3091                    sql: "SELECT actor FROM events WHERE id = ?1".to_string(),
3092                    params: vec![SqlValue::Text(id.to_string())],
3093                    label: None,
3094                })
3095                .await
3096                .expect("raw events query");
3097            rows.len()
3098        };
3099
3100        // Arm 1: the co-residency contract. A raw-SQL consumer of the legacy
3101        // table finds the plain-append row there.
3102        assert_eq!(
3103            count_by_raw_sql(Arc::clone(&legacy_backend), provenance_id).await,
3104            1,
3105            "plain appends must stay visible to raw-SQL consumers of the legacy events table"
3106        );
3107        // Arm 2a: the moved class is genuinely gone from the legacy table —
3108        // this is the silent-failure face of the boundary, held visible.
3109        assert_eq!(
3110            count_by_raw_sql(Arc::clone(&legacy_backend), audit_id).await,
3111            0,
3112            "audit-lane rows must not land in the legacy events table"
3113        );
3114        // Arm 2b: positive control for the instrument — the identical query
3115        // finds the moved row on the lane backend, so arm 2a's zero (and any
3116        // future arm-1 zero) is a routing fact, not a dead query.
3117        assert_eq!(
3118            count_by_raw_sql(Arc::clone(&lane_backend), audit_id).await,
3119            1,
3120            "the raw query must prove it can find rows where they actually live"
3121        );
3122    }
3123
3124    /// End to end over the real socket: the idempotent lane returns true
3125    /// dispositions, and the row is readable back through the same store.
3126    #[cfg(unix)]
3127    #[tokio::test]
3128    async fn idempotent_append_round_trips_through_the_daemon() {
3129        let dir = tempfile::tempdir().expect("tempdir");
3130        let (_db, socket) = boot_daemon(&dir).await;
3131        let client = EventsSplitClient::new(socket).expect("client");
3132        let store = ForwardingEventStore::new("local", client);
3133
3134        let event = test_event("local");
3135        let id = event.id;
3136        let result = store
3137            .append_events_idempotent(vec![event.clone()])
3138            .await
3139            .expect("idempotent append over socket");
3140        assert_eq!(result.rows, vec![EventAppendDisposition::Inserted]);
3141
3142        // Retry of the identical row reports AlreadyPresentIdentical — the
3143        // daemon ran the real idempotent path, not a blind re-insert.
3144        let retry = store
3145            .append_events_idempotent(vec![event])
3146            .await
3147            .expect("idempotent retry");
3148        assert_eq!(
3149            retry.rows,
3150            vec![EventAppendDisposition::AlreadyPresentIdentical]
3151        );
3152
3153        let fetched = store.get_event(id).await.expect("get over socket");
3154        assert_eq!(fetched.map(|e| e.id), Some(id));
3155        let count = store
3156            .count_events(EventFilter::default())
3157            .await
3158            .expect("count over socket");
3159        assert_eq!(count, 1);
3160    }
3161
3162    /// The fire-and-forget lane delivers to the daemon store: enqueue returns
3163    /// immediately and the row becomes visible to a subsequent count.
3164    #[cfg(unix)]
3165    #[tokio::test]
3166    async fn fire_and_forget_append_lands_in_the_daemon_store() {
3167        let dir = tempfile::tempdir().expect("tempdir");
3168        let (_db, socket) = boot_daemon(&dir).await;
3169        let client = EventsSplitClient::new(socket).expect("client");
3170        let store = ForwardingEventStore::new("local", Arc::clone(&client));
3171
3172        store
3173            .append_event(test_event("local"))
3174            .await
3175            .expect("append_event is fire-and-forget");
3176
3177        // Poll for BOTH effects: the row landing daemon-side and the
3178        // forwarder's own counter — the row can be visible a beat before the
3179        // forwarder task is rescheduled to record the delivery.
3180        let mut count = 0;
3181        let mut metrics = client.metrics();
3182        for _ in 0..100 {
3183            count = store
3184                .count_events(EventFilter::default())
3185                .await
3186                .expect("count over socket");
3187            metrics = client.metrics();
3188            if count == 1 && metrics.forwarded_events >= 1 {
3189                break;
3190            }
3191            tokio::time::sleep(Duration::from_millis(20)).await;
3192        }
3193        assert_eq!(count, 1, "forwarded event must land in the daemon store");
3194        assert_eq!(metrics.dropped_events, 0);
3195        assert!(metrics.forwarded_events >= 1, "delivery must be counted");
3196    }
3197
3198    /// R4 overflow arm: with the daemon dead, appends beyond the queue bound
3199    /// are DROPPED and counted while every append still returns Ok — the
3200    /// domain path proceeds. The drop counter is the loss-tolerance contract.
3201    #[cfg(unix)]
3202    #[tokio::test]
3203    async fn queue_overflow_drops_and_counts_but_never_errors() {
3204        let dir = tempfile::tempdir().expect("tempdir");
3205        let dead_socket = dir.path().join("nobody-home.sock");
3206        let client = EventsSplitClient::new_with_queue_depth(dead_socket, 2).expect("client");
3207        let store = ForwardingEventStore::new("local", Arc::clone(&client));
3208
3209        for _ in 0..20 {
3210            store
3211                .append_event(test_event("local"))
3212                .await
3213                .expect("append_event must not error under overflow");
3214        }
3215        let metrics = client.metrics();
3216        assert!(
3217            metrics.dropped_events > 0,
3218            "overflow must register in the drop counter, got {metrics:?}"
3219        );
3220    }
3221
3222    /// R4 dead-socket arm: synchronous lanes fail with a typed retryable
3223    /// storage error (never hang, never a generic panic), and the offline
3224    /// preflight validator keeps working with zero I/O.
3225    #[cfg(unix)]
3226    #[tokio::test]
3227    async fn dead_socket_reads_fail_typed_and_preflight_stays_local() {
3228        let dir = tempfile::tempdir().expect("tempdir");
3229        let dead_socket = dir.path().join("nobody-home.sock");
3230        let client = EventsSplitClient::new(dead_socket).expect("client");
3231        let store = ForwardingEventStore::new("local", client);
3232
3233        let error = store
3234            .get_event(Uuid::new_v4())
3235            .await
3236            .expect_err("read against a dead socket must fail");
3237        assert!(
3238            matches!(error, StorageError::Pool { .. }),
3239            "expected the typed unreachable error, got {error:?}"
3240        );
3241
3242        // The audit-batch seam contract: preflight is local and functional
3243        // with the daemon down.
3244        assert!(store.supports_idempotent_audit_batch());
3245        store
3246            .preflight_event(&test_event("local"))
3247            .expect("offline preflight validates a well-formed event");
3248    }
3249
3250    /// Version-skew arm: a frame carrying an unknown protocol version gets a
3251    /// typed non-retryable refusal, not a deserialization failure.
3252    #[cfg(unix)]
3253    #[tokio::test]
3254    async fn protocol_version_skew_is_a_typed_refusal() {
3255        let dir = tempfile::tempdir().expect("tempdir");
3256        let (_db, socket) = boot_daemon(&dir).await;
3257
3258        let mut stream = UnixStream::connect(&socket).await.expect("connect");
3259        let request = EventsRequest::CountEvents {
3260            protocol_version: EVENTS_PROTOCOL_VERSION + 1,
3261            namespace: "local".into(),
3262            filter: EventFilter::default(),
3263        };
3264        let payload = serde_json::to_vec(&request).expect("serialize");
3265        write_frame(&mut stream, &payload).await.expect("write");
3266        let bytes = read_frame(&mut stream).await.expect("read");
3267        let response: EventsResponse = serde_json::from_slice(&bytes).expect("parse");
3268        match response {
3269            EventsResponse::Error {
3270                message, retryable, ..
3271            } => {
3272                assert!(!retryable, "version skew is not retryable");
3273                assert!(message.contains("protocol version"), "message: {message}");
3274            }
3275            other => panic!("expected a typed refusal, got {other:?}"),
3276        }
3277    }
3278
3279    /// The per-socket daemon guard is exclusive while held and reusable after
3280    /// release — the mechanism that keeps a supervisor respawn race down to
3281    /// one surviving daemon.
3282    #[cfg(unix)]
3283    #[test]
3284    fn events_daemon_guard_is_exclusive_then_reusable() {
3285        let dir = tempfile::tempdir().expect("tempdir");
3286        let socket = dir.path().join("events.sock");
3287        let first = try_acquire_events_daemon_guard(&socket).expect("first acquire");
3288        assert!(
3289            try_acquire_events_daemon_guard(&socket).is_none(),
3290            "second acquire must fail while the first guard is held"
3291        );
3292        drop(first);
3293        assert!(
3294            try_acquire_events_daemon_guard(&socket).is_some(),
3295            "acquire must succeed again after the guard is released"
3296        );
3297    }
3298
3299    /// A read-only file-backed runtime with the split configured must never
3300    /// create or schema-initialize an events database. Missing events db →
3301    /// `events()` serves the legacy store alone and leaves the filesystem
3302    /// untouched; a pre-existing events db (minted by an earlier writable
3303    /// host) is opened read-only and its rows merge into reads.
3304    /// (unix-gated for the sidecar-freeze harness step only.)
3305    #[cfg(unix)]
3306    #[tokio::test]
3307    async fn read_only_runtime_never_creates_an_events_db() {
3308        use crate::{KhiveRuntime, Namespace, RuntimeConfig};
3309
3310        let dir = tempfile::tempdir().expect("tempdir");
3311        let main_db = dir.path().join("main.db");
3312        // Materialize + migrate the main database with a writable runtime.
3313        drop(
3314            KhiveRuntime::new(RuntimeConfig {
3315                db_path: Some(main_db.clone()),
3316                ..RuntimeConfig::no_embeddings()
3317            })
3318            .expect("create main db"),
3319        );
3320        let events_db = events_db_path_beside(&main_db);
3321        assert!(!events_db.exists(), "precondition: no events db yet");
3322
3323        // Freeze the WAL sidecars the writable runtime left behind — the
3324        // read-only opener refuses a snapshot with a writable -shm.
3325        {
3326            use std::os::unix::fs::PermissionsExt;
3327            for suffix in ["-wal", "-shm"] {
3328                let mut name = main_db.file_name().expect("db file name").to_os_string();
3329                name.push(suffix);
3330                let sidecar = main_db.parent().expect("db parent dir").join(name);
3331                if sidecar.exists() {
3332                    let mut permissions = std::fs::metadata(&sidecar)
3333                        .expect("sidecar metadata")
3334                        .permissions();
3335                    permissions.set_mode(0o444);
3336                    std::fs::set_permissions(&sidecar, permissions).expect("freeze sidecar");
3337                }
3338            }
3339        }
3340
3341        let split_config = |db: PathBuf| RuntimeConfig {
3342            db_path: Some(main_db.clone()),
3343            events_split: Some(EventsSplitConfig {
3344                db_path: db,
3345                socket_path: None,
3346            }),
3347            ..RuntimeConfig::no_embeddings()
3348        };
3349
3350        // Arm 1: read-only runtime, no events db on disk. Reads work through
3351        // the legacy store and nothing is minted.
3352        let ro =
3353            KhiveRuntime::new_readonly(split_config(events_db.clone())).expect("read-only runtime");
3354        let token = ro.authorize(Namespace::local()).expect("token");
3355        let store = ro.events(&token).expect("events store");
3356        let count = store
3357            .count_events(EventFilter::default())
3358            .await
3359            .expect("count through legacy-only plane");
3360        assert_eq!(count, 0);
3361        assert!(
3362            !events_db.exists(),
3363            "a read-only runtime must not mint the events database"
3364        );
3365
3366        // Arm 2: a writable host mints the events db and lands a lane row;
3367        // the read-only runtime then opens it read-only and merges the row.
3368        // The writer is scoped (not the process-global registry) and its WAL
3369        // sidecars frozen after drop — the read-only opener rightly refuses a
3370        // snapshot with a live writer, and that refusal is not this test's
3371        // subject.
3372        {
3373            let lane_backend = StorageBackend::sqlite(&events_db).expect("writable lane");
3374            lane_backend
3375                .events_for_namespace("local")
3376                .expect("lane store")
3377                .append_event(test_event("local"))
3378                .await
3379                .expect("seed lane row");
3380        }
3381        {
3382            use std::os::unix::fs::PermissionsExt;
3383            for suffix in ["-wal", "-shm"] {
3384                let mut name = events_db.file_name().expect("db file name").to_os_string();
3385                name.push(suffix);
3386                let sidecar = events_db.parent().expect("db parent dir").join(name);
3387                if sidecar.exists() {
3388                    let mut permissions = std::fs::metadata(&sidecar)
3389                        .expect("sidecar metadata")
3390                        .permissions();
3391                    permissions.set_mode(0o444);
3392                    std::fs::set_permissions(&sidecar, permissions).expect("freeze sidecar");
3393                }
3394            }
3395        }
3396        let store = ro.events(&token).expect("events store with lane present");
3397        let count = store
3398            .count_events(EventFilter::default())
3399            .await
3400            .expect("merged count");
3401        assert_eq!(count, 1, "the pre-existing lane row must merge into reads");
3402    }
3403
3404    /// Embedded (direct) mode: the process-global backend writes and reads
3405    /// `events.db` without any daemon.
3406    #[tokio::test]
3407    async fn direct_mode_appends_and_reads_without_a_daemon() {
3408        let dir = tempfile::tempdir().expect("tempdir");
3409        let db = dir.path().join("events.db");
3410        let backend = direct_backend_for(&db).expect("direct backend");
3411        let store = backend.events_for_namespace("local").expect("store");
3412        store
3413            .append_event(test_event("local"))
3414            .await
3415            .expect("direct append");
3416        let count = store
3417            .count_events(EventFilter::default())
3418            .await
3419            .expect("count");
3420        assert_eq!(count, 1);
3421    }
3422}