Skip to main content

khive_runtime/events_split/
stores.rs

1use super::events_count_read_budget::CountReadBudget;
2#[cfg(unix)]
3use super::events_count_read_budget::{self, restore_count_read_failure};
4#[cfg(unix)]
5use super::forwarder::EventsSplitClient;
6#[cfg(unix)]
7use super::paths::EVENTS_PROTOCOL_VERSION;
8#[cfg(unix)]
9use super::protocol::{EventsRequest, EventsResponse, WireWriterTaskFailure};
10use async_trait::async_trait;
11use khive_storage::event::{
12    EventGroupBy, EventPageQuery, EventPageWindow, IdempotentEventBatchResult,
13};
14use khive_storage::{
15    BatchWriteSummary, Event, EventFilter, EventStore, Page, PageRequest, StorageError,
16    StorageResult,
17};
18use std::collections::BTreeMap;
19use std::sync::Arc;
20use uuid::Uuid;
21
22/// [`EventStore`] implementation the runtime hands out when the events split
23/// runs in daemon mode. Appends are fire-and-forget through the client's
24/// bounded queue; the ADR-133 idempotent lane and all reads are synchronous
25/// round-trips to the events daemon.
26#[cfg(unix)]
27#[derive(Debug)]
28pub struct ForwardingEventStore {
29    pub(super) namespace: String,
30    pub(super) client: Arc<EventsSplitClient>,
31}
32
33#[cfg(unix)]
34impl ForwardingEventStore {
35    pub fn new(namespace: impl Into<String>, client: Arc<EventsSplitClient>) -> Self {
36        Self {
37            namespace: namespace.into(),
38            client,
39        }
40    }
41
42    pub(super) fn unexpected_count(
43        &self,
44        op: &'static str,
45        response: EventsResponse,
46    ) -> StorageError {
47        match restore_count_read_failure(response) {
48            Err(error) => error,
49            Ok(other) => self.unexpected(op, other),
50        }
51    }
52
53    pub(super) fn unexpected(&self, op: &'static str, response: EventsResponse) -> StorageError {
54        match response {
55            EventsResponse::Error {
56                message,
57                retryable,
58                writer_task_failure,
59                sqlite_write_failure,
60                read_failure,
61            } => {
62                if read_failure.is_some() {
63                    return events_count_read_budget::malformed_read_failure(
64                        "non-count response carries read failure evidence",
65                    );
66                }
67                // Reconstruct the same legacy policy first; evidence must not
68                // widen retries or turn a known rollback into task termination.
69                let source = if retryable {
70                    StorageError::Pool {
71                        operation: op.into(),
72                        message,
73                    }
74                } else {
75                    StorageError::InvalidInput {
76                        capability: khive_storage::StorageCapability::Events,
77                        operation: op.into(),
78                        message,
79                    }
80                };
81                let source = match sqlite_write_failure {
82                    Some(failure) => source.with_sqlite_write_failure(failure),
83                    None => source,
84                };
85                match writer_task_failure {
86                    Some(WireWriterTaskFailure::RequestFailed {
87                        request_state,
88                        evidence,
89                    }) => {
90                        use super::protocol::{
91                            WireWriterFailureEvidence as E, WireWriterOwnerCause as C,
92                            WireWriterTaskState as S,
93                        };
94                        let reconstructed = match evidence {
95                            None => StorageError::WriterTaskRequestFailed {
96                                request_state: request_state.into(),
97                                source: Box::new(source),
98                            },
99                            Some(E::WriteOutcomeUnknown)
100                                if request_state == S::SideEffectsUnknown && !retryable =>
101                            {
102                                StorageError::WriteOutcomeUnknown {
103                                    source: Box::new(source),
104                                }
105                            }
106                            Some(E::OwnerUnavailable {
107                                operation,
108                                cause:
109                                    C::Pool {
110                                        operation: source_operation,
111                                        message,
112                                    },
113                            }) if request_state == S::NotStarted && retryable => {
114                                StorageError::WriterOwnerUnavailable {
115                                    operation: operation.into(),
116                                    source: Box::new(StorageError::Pool {
117                                        operation: source_operation.into(),
118                                        message,
119                                    }),
120                                }
121                            }
122                            Some(E::OwnerUnavailable {
123                                operation,
124                                cause: C::NoRuntime,
125                            }) if request_state == S::NotStarted && !retryable => {
126                                StorageError::WriterOwnerUnavailable {
127                                    operation: operation.into(),
128                                    source: Box::new(StorageError::WriterTaskNoRuntime),
129                                }
130                            }
131                            Some(E::RouteViolation { operation })
132                                if request_state == S::NotStarted && !retryable =>
133                            {
134                                StorageError::WriterRouteViolation {
135                                    operation: operation.into(),
136                                }
137                            }
138                            Some(_) => return StorageError::Serialization {
139                                capability: khive_storage::StorageCapability::Events,
140                                message:
141                                    "events daemon returned contradictory writer failure evidence"
142                                        .into(),
143                            },
144                        };
145                        match sqlite_write_failure {
146                            Some(failure) => reconstructed.with_sqlite_write_failure(failure),
147                            None => reconstructed,
148                        }
149                    }
150                    Some(WireWriterTaskFailure::TaskTerminated {
151                        request_state,
152                        sqlite_full_codes,
153                    }) => {
154                        let terminal = StorageError::WriterTaskTerminated {
155                            request_state: request_state.into(),
156                            sqlite_full_codes,
157                        };
158                        match sqlite_write_failure {
159                            Some(failure) => terminal.with_sqlite_write_failure(failure),
160                            None => terminal,
161                        }
162                    }
163                    None => source,
164                }
165            }
166            other => StorageError::Serialization {
167                capability: khive_storage::StorageCapability::Events,
168                message: format!("events daemon returned mismatched response for {op}: {other:?}"),
169            },
170        }
171    }
172}
173
174#[cfg(unix)]
175#[async_trait]
176impl EventStore for ForwardingEventStore {
177    async fn append_event(&self, event: Event) -> StorageResult<()> {
178        self.client.enqueue(&self.namespace, vec![event]);
179        Ok(())
180    }
181
182    async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
183        let attempted = events.len() as u64;
184        self.client.enqueue(&self.namespace, events);
185        // Fire-and-forget: the hand-off succeeded or was counted as a drop;
186        // either way the caller's contract is "accepted for forwarding".
187        Ok(BatchWriteSummary {
188            attempted,
189            affected: attempted,
190            ..BatchWriteSummary::default()
191        })
192    }
193
194    async fn get_event(&self, id: Uuid) -> StorageResult<Option<Event>> {
195        let request = EventsRequest::GetEvent {
196            protocol_version: EVENTS_PROTOCOL_VERSION,
197            namespace: self.namespace.clone(),
198            id,
199        };
200        match self.client.round_trip(&request).await? {
201            EventsResponse::Event { event } => Ok(event),
202            other => Err(self.unexpected("get_event", other)),
203        }
204    }
205
206    async fn query_events(
207        &self,
208        filter: EventFilter,
209        page: PageRequest,
210    ) -> StorageResult<Page<Event>> {
211        let budget = CountReadBudget::capture();
212        let request = EventsRequest::QueryEvents {
213            protocol_version: EVENTS_PROTOCOL_VERSION,
214            namespace: self.namespace.clone(),
215            filter,
216            page,
217        };
218        match self.client.round_trip_query(&request, &budget).await? {
219            EventsResponse::Pageful { page } => Ok(page),
220            other => Err(self.unexpected("query_events", other)),
221        }
222    }
223
224    async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
225        let budget = CountReadBudget::capture();
226        let request = EventsRequest::CountEvents {
227            protocol_version: EVENTS_PROTOCOL_VERSION,
228            namespace: self.namespace.clone(),
229            operator_read_budget_secs: Some(budget.operator_read_budget_secs),
230            filter,
231        };
232        match self.client.round_trip_count(&request, &budget).await? {
233            EventsResponse::Count { count } => Ok(count),
234            other => Err(self.unexpected_count("count_events", other)),
235        }
236    }
237
238    async fn count_events_grouped(
239        &self,
240        filter: EventFilter,
241        group_by: EventGroupBy,
242    ) -> StorageResult<BTreeMap<String, u64>> {
243        let budget = CountReadBudget::capture();
244        let request = EventsRequest::CountEventsGrouped {
245            protocol_version: EVENTS_PROTOCOL_VERSION,
246            namespace: self.namespace.clone(),
247            operator_read_budget_secs: Some(budget.operator_read_budget_secs),
248            filter,
249            group_by,
250        };
251        match self.client.round_trip_count(&request, &budget).await? {
252            EventsResponse::GroupedCount { counts } => Ok(counts),
253            other => Err(self.unexpected_count("count_events_grouped", other)),
254        }
255    }
256
257    async fn query_event_page(&self, query: EventPageQuery) -> StorageResult<EventPageWindow> {
258        crate::event_page::validate_page_query(&query)?;
259        let request = EventsRequest::QueryEventPage {
260            protocol_version: EVENTS_PROTOCOL_VERSION,
261            namespace: self.namespace.clone(),
262            query: query.clone(),
263        };
264        match self.client.round_trip(&request).await? {
265            EventsResponse::EventPageWindow { window } => {
266                crate::event_page::validate_window(&query, Some(&self.namespace), &window)?;
267                Ok(window)
268            }
269            error @ EventsResponse::Error { .. } => Err(self.unexpected("query_event_page", error)),
270            _ => Err(crate::event_page::page_error(
271                "events page response kind mismatch",
272            )),
273        }
274    }
275
276    fn preflight_event(&self, event: &Event) -> StorageResult<()> {
277        // Same validation code path as the daemon-side store, zero I/O.
278        self.client.preflight_store.preflight_event(event)
279    }
280
281    async fn append_events_idempotent(
282        &self,
283        events: Vec<Event>,
284    ) -> StorageResult<IdempotentEventBatchResult> {
285        let request = EventsRequest::AppendEventsIdempotent {
286            protocol_version: EVENTS_PROTOCOL_VERSION,
287            namespace: self.namespace.clone(),
288            events,
289        };
290        match self.client.round_trip(&request).await? {
291            EventsResponse::Idempotent { result } => Ok(result),
292            other => Err(self.unexpected("append_events_idempotent", other)),
293        }
294    }
295
296    fn supports_idempotent_audit_batch(&self) -> bool {
297        true
298    }
299}
300
301/// The store the runtime hands out when the events split is configured:
302/// routes by APPEND CLASS rather than moving the whole event plane.
303///
304/// - The ADR-133 idempotent audit-batch lane — the measured bulk of event
305///   write volume (verb-dispatch audit plus the config-lock rows that ride
306///   the same flusher) — goes to the events lane (the events database, forwarded or
307///   direct).
308/// - Plain appends stay on the legacy store, because the legacy `events`
309///   table has raw-SQL consumers whose correctness depends on finding those
310///   rows there: the schedule drain's creator-provenance fence, the kg
311///   projection worker's guarded event INSERT (transactional with main-db
312///   state), and GraphQuery's cross-substrate UNION. Those events are
313///   low-volume domain facts; the split's contention relief does not need
314///   them moved, and moving them breaks the consumers by construction.
315/// - Reads merge both stores so trait-level consumers
316///   (`brain.event_counts`, event getters) observe one event plane.
317pub struct SplitEventStore {
318    pub(super) legacy: Arc<dyn EventStore>,
319    pub(super) lane: Arc<dyn EventStore>,
320}
321
322impl std::fmt::Debug for SplitEventStore {
323    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
324        f.debug_struct("SplitEventStore").finish_non_exhaustive()
325    }
326}
327
328impl SplitEventStore {
329    /// Bound on `offset + limit` for a merged `query_events` window. Offset
330    /// pagination over two stores materializes the whole prefix in memory
331    /// (see `query_events`), so an unbounded offset would let a single
332    /// request buffer both stores wholesale. The bound comfortably admits
333    /// the largest legitimate bounded window in the tree
334    /// (`brain.event_counts`' 50k page); deep walks page with a `before`
335    /// cursor at `offset: 0`, which never grows the materialized prefix.
336    pub const MAX_MERGED_WINDOW_ROWS: u64 = 100_000;
337
338    pub fn new(legacy: Arc<dyn EventStore>, lane: Arc<dyn EventStore>) -> Self {
339        Self { legacy, lane }
340    }
341}
342
343#[async_trait]
344impl EventStore for SplitEventStore {
345    async fn append_event(&self, event: Event) -> StorageResult<()> {
346        self.legacy.append_event(event).await
347    }
348
349    async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
350        self.legacy.append_events(events).await
351    }
352
353    async fn get_event(&self, id: Uuid) -> StorageResult<Option<Event>> {
354        // Domain events (legacy) are the likelier and cheaper hit; fall back
355        // to the lane for audit-batch rows.
356        if let Some(event) = self.legacy.get_event(id).await? {
357            return Ok(Some(event));
358        }
359        self.lane.get_event(id).await
360    }
361
362    async fn query_events(
363        &self,
364        filter: EventFilter,
365        page: PageRequest,
366    ) -> StorageResult<Page<Event>> {
367        let budget = CountReadBudget::capture();
368        budget
369            .scope_operation("query_events", async {
370                // Offset pagination cannot be split across two stores: fetch each
371                // store's prefix covering the requested window, merge in the stores'
372                // shared order (created_at DESC, id DESC), then window in memory.
373                // Cost is O(offset + limit) rows materialized from each store — the
374                // floor for offset semantics over two sources, since the split point
375                // is unknowable without both prefixes. The sort below is a single
376                // merge pass in practice (std's stable sort is adaptive on the
377                // concatenation of two sorted runs). That materialization is why the
378                // window is bounded below: without a bound, one authenticated
379                // request with a pathological offset would make both stores buffer
380                // and sort every matching row. Deep walks page with a `before`
381                // cursor (strict `created_at <` bound in `EventFilter`) at
382                // `offset: 0`, which stays inside the bound at any depth.
383                let window = page.offset.saturating_add(u64::from(page.limit));
384                if window > Self::MAX_MERGED_WINDOW_ROWS {
385                    return Err(StorageError::InvalidInput {
386                        capability: khive_storage::StorageCapability::Events,
387                        operation: "query_events".into(),
388                        message: format!(
389                    "offset+limit ({window}) exceeds the merged event plane's window bound \
390                     of {}; page deep windows with a `before` cursor at offset 0, or narrow \
391                     the filter",
392                    Self::MAX_MERGED_WINDOW_ROWS
393                ),
394                    });
395                }
396                let prefix = PageRequest {
397                    offset: 0,
398                    limit: window.min(u64::from(u32::MAX)) as u32,
399                };
400                let legacy = self
401                    .legacy
402                    .query_events(filter.clone(), prefix.clone())
403                    .await
404                    .map_err(|error| budget.attribute(error))?;
405                let lane = self.lane.query_events(filter, prefix).await?;
406                let total = match (legacy.total, lane.total) {
407                    (Some(a), Some(b)) => Some(a + b),
408                    _ => None,
409                };
410                let mut items = legacy.items;
411                items.extend(lane.items);
412                items.sort_by(|a, b| {
413                    b.created_at
414                        .cmp(&a.created_at)
415                        .then_with(|| b.id.cmp(&a.id))
416                });
417                let items = items
418                    .into_iter()
419                    .skip(page.offset as usize)
420                    .take(page.limit as usize)
421                    .collect();
422                Ok(Page { items, total })
423            })
424            .await
425    }
426
427    async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
428        let budget = CountReadBudget::capture();
429        budget
430            .scope(async {
431                let legacy = self
432                    .legacy
433                    .count_events(filter.clone())
434                    .await
435                    .map_err(|error| budget.attribute(error))?;
436                let lane = self.lane.count_events(filter).await?;
437                Ok(legacy + lane)
438            })
439            .await
440    }
441
442    async fn count_events_grouped(
443        &self,
444        filter: EventFilter,
445        group_by: EventGroupBy,
446    ) -> StorageResult<BTreeMap<String, u64>> {
447        let budget = CountReadBudget::capture();
448        budget
449            .scope(async {
450                let mut counts = self
451                    .legacy
452                    .count_events_grouped(filter.clone(), group_by)
453                    .await
454                    .map_err(|error| budget.attribute(error))?;
455                for (key, count) in self.lane.count_events_grouped(filter, group_by).await? {
456                    let total = counts.entry(key).or_default();
457                    *total =
458                        total
459                            .checked_add(count)
460                            .ok_or_else(|| StorageError::InvalidInput {
461                                capability: khive_storage::StorageCapability::Events,
462                                operation: "count_events_grouped".into(),
463                                message: "grouped event count overflow".into(),
464                            })?;
465                }
466                Ok(counts)
467            })
468            .await
469    }
470
471    async fn query_event_page(&self, query: EventPageQuery) -> StorageResult<EventPageWindow> {
472        crate::event_page::split_page(self.legacy.as_ref(), self.lane.as_ref(), query).await
473    }
474
475    fn preflight_event(&self, event: &Event) -> StorageResult<()> {
476        // Validation is store-independent; the lane's implementation is the
477        // local zero-I/O validator in forwarding mode.
478        self.lane.preflight_event(event)
479    }
480
481    async fn append_events_idempotent(
482        &self,
483        events: Vec<Event>,
484    ) -> StorageResult<IdempotentEventBatchResult> {
485        // A retried batch may have landed — fully or partially — on the
486        // legacy store before the split cutover, and merged reads stay
487        // duplicate-free only while an id lives in exactly one store. Probe
488        // the legacy store for the batch's ids and route resident rows back
489        // through the legacy store's own idempotent machinery, which compares
490        // every persisted column without re-inserting; only genuinely new
491        // rows reach the lane. Without the probe, a legacy-resident id would
492        // be inserted a second time into the lane and every merged query and
493        // count would double-count it.
494        if events.is_empty() {
495            return self.lane.append_events_idempotent(events).await;
496        }
497        let ids: Vec<Uuid> = events.iter().map(|event| event.id).collect();
498        let probe_limit = u32::try_from(ids.len()).map_err(|_| StorageError::InvalidInput {
499            capability: khive_storage::StorageCapability::Events,
500            operation: "append_events_idempotent".into(),
501            message: format!(
502                "batch of {} rows exceeds the legacy probe window",
503                ids.len()
504            ),
505        })?;
506        let existing = self
507            .legacy
508            .query_events(
509                EventFilter {
510                    ids,
511                    ..EventFilter::default()
512                },
513                PageRequest {
514                    offset: 0,
515                    limit: probe_limit,
516                },
517            )
518            .await?;
519        let legacy_resident: std::collections::HashSet<Uuid> =
520            existing.items.iter().map(|event| event.id).collect();
521        if legacy_resident.is_empty() {
522            return self.lane.append_events_idempotent(events).await;
523        }
524        let mut legacy_rows = Vec::new();
525        let mut lane_rows = Vec::new();
526        let mut routed_to_legacy = Vec::with_capacity(events.len());
527        for event in events {
528            if legacy_resident.contains(&event.id) {
529                routed_to_legacy.push(true);
530                legacy_rows.push(event);
531            } else {
532                routed_to_legacy.push(false);
533                lane_rows.push(event);
534            }
535        }
536        let legacy_expected = legacy_rows.len();
537        let lane_expected = lane_rows.len();
538        let legacy_result = self.legacy.append_events_idempotent(legacy_rows).await?;
539        let lane_result = if lane_expected == 0 {
540            IdempotentEventBatchResult { rows: Vec::new() }
541        } else {
542            self.lane.append_events_idempotent(lane_rows).await?
543        };
544        if legacy_result.rows.len() != legacy_expected || lane_result.rows.len() != lane_expected {
545            return Err(StorageError::Driver {
546                capability: khive_storage::StorageCapability::Events,
547                operation: "append_events_idempotent".into(),
548                source: format!(
549                    "idempotent sub-batch result length mismatch: legacy {}/{}, lane {}/{}",
550                    legacy_result.rows.len(),
551                    legacy_expected,
552                    lane_result.rows.len(),
553                    lane_expected
554                )
555                .into(),
556            });
557        }
558        let mut legacy_iter = legacy_result.rows.into_iter();
559        let mut lane_iter = lane_result.rows.into_iter();
560        let rows = routed_to_legacy
561            .into_iter()
562            .map(|to_legacy| {
563                if to_legacy {
564                    legacy_iter.next().expect("length checked above")
565                } else {
566                    lane_iter.next().expect("length checked above")
567                }
568            })
569            .collect();
570        Ok(IdempotentEventBatchResult { rows })
571    }
572
573    fn supports_idempotent_audit_batch(&self) -> bool {
574        // The legacy-resident probe above routes retried rows through the
575        // legacy store's idempotent path, so the split supports the audit
576        // batch only when BOTH stores do.
577        self.lane.supports_idempotent_audit_batch() && self.legacy.supports_idempotent_audit_batch()
578    }
579}