khive_runtime/audit_batch.rs
1//! ADR-133 Slice 1: the audit-batch seam.
2//!
3//! Incidental audit writes (gate denials, dispatch outcomes, config-lock
4//! rows, git.digest receipts, and pure-observability rows like
5//! `RecallExecuted`) no longer take one writer-task acquisition per row on
6//! the request hot path. Concurrent submissions arriving while a generation
7//! is committing share the *next* generation instead of each taking their
8//! own writer acquisition, so N concurrent producers collapse to one
9//! [`khive_storage::EventStore::append_events_idempotent`] call per
10//! generation rather than N.
11//!
12//! [`AuditBatch`] owns this seam. A lazily-spawned supervisor task drains
13//! pending rows into generations and drives each through the store; the
14//! supervisor's own `JoinHandle` is retained (never discarded) so an
15//! abnormal exit — panic, cancellation, a lost child join, or a driver that
16//! returns `Ok` while state is not terminally consistent — is observed and
17//! converted into a `Failed` transition with all accepted waiters resolved,
18//! per owner ruling R1/R4 (`.khive/OWNER_RULING_adr133_gate.md`).
19
20#[cfg(any(test, feature = "fault-injection"))]
21use std::sync::atomic::Ordering;
22use std::sync::Arc;
23use std::time::Duration;
24
25use parking_lot::Mutex;
26use tokio::sync::oneshot;
27use tokio::task::JoinHandle;
28
29use khive_storage::event::EventAppendDisposition;
30use khive_storage::{Event, EventStore, StorageError, WriterTaskRequestState};
31
32/// Coarse durability classification for an [`AuditProducer`]. See
33/// [`classify`] for the exhaustive mapping.
34#[derive(Debug, Clone, Copy, PartialEq, Eq)]
35pub(crate) enum AuditProductionClass {
36 /// The dispatch this row audits carries an obligation: the caller-visible
37 /// outcome must not silently diverge from what was durably recorded.
38 DispatchObligation,
39 /// The row is a best-effort observability signal; a non-commit degrades
40 /// gracefully rather than blocking or failing the dispatch it audits.
41 PureObservability,
42}
43
44/// Every call site that can submit a row through [`AuditBatchControl`].
45/// Adding a variant here without extending the crate-private `classify`
46/// function's match is a compile error — there is no wildcard arm.
47#[derive(Debug, Clone, Copy, PartialEq, Eq)]
48pub enum AuditProducer {
49 /// The gate denied a dispatch; the denial itself is audited.
50 GateDenied,
51 /// The gate backend was unreachable and the dispatch failed closed; the
52 /// refusal is audited best-effort: the typed `GateUnavailable` refusal
53 /// already fails the dispatch, so a lost row degrades diagnostics only.
54 GateUnavailable,
55 /// A pack dispatch returned a successful result.
56 DispatchSucceeded,
57 /// A pack dispatch returned an error result.
58 DispatchFailed,
59 /// A redirected KG read consulted the gate on its effective entity id.
60 EffectiveTargetCheck,
61 /// The gate allowed a verb no pack owns.
62 UnknownVerb,
63 /// The strict `git.digest` success receipt (schema v2).
64 GitDigestReceipt,
65 /// A drained process-lifetime `OnceLock` config-lock row.
66 ConfigLocked,
67 /// A `memory.recall` execution's pure-observability audit row.
68 /// Submitted by the recall handler through its owning registry's batch.
69 RecallExecuted,
70}
71
72/// Classify an [`AuditProducer`] into its [`AuditProductionClass`]. One
73/// exhaustive match, no wildcard arm.
74pub(crate) const fn classify(producer: AuditProducer) -> AuditProductionClass {
75 match producer {
76 AuditProducer::GateDenied
77 | AuditProducer::DispatchSucceeded
78 | AuditProducer::DispatchFailed
79 | AuditProducer::EffectiveTargetCheck
80 | AuditProducer::UnknownVerb
81 | AuditProducer::GitDigestReceipt => AuditProductionClass::DispatchObligation,
82 AuditProducer::ConfigLocked
83 | AuditProducer::RecallExecuted
84 | AuditProducer::GateUnavailable => AuditProductionClass::PureObservability,
85 }
86}
87
88/// Exhaustive terminal reasons an [`AuditBatchControl::submit`],
89/// [`AuditBatchControl::quiesce`], or [`AuditBatchControl::close_and_drain`]
90/// call can resolve to. Never mapped through a wildcard arm anywhere in this
91/// module (R4).
92#[derive(Debug, Clone, Copy, PartialEq, Eq)]
93pub enum AuditTerminalReason {
94 /// `EventStore::preflight_event` rejected the row before it was ever
95 /// enqueued. The row was never counted as submitted.
96 PreflightRejected,
97 /// The batch is `Closing` or `Closed`; new admission is refused.
98 AdmissionClosed,
99 /// `AuditBatchConfig::max_pending_rows` was reached before this row was
100 /// enqueued. The row was never counted as submitted and never shared a
101 /// generation with anyone — safe to retry, and doing so applies the
102 /// obligation at most once.
103 QueueAdmissionExhausted,
104 /// The row was already enqueued (counted in `submitted_rows`) when this
105 /// caller's `AuditBatchConfig::admission_deadline` elapsed waiting for
106 /// its generation's outcome. Unlike [`Self::QueueAdmissionExhausted`],
107 /// the row was not refused: by the moment the deadline fires it may
108 /// still be sitting in `state.pending`, or the driver may have already
109 /// drained it into an in-flight generation — either way it remains
110 /// enqueued and unresolved, and is committed (or terminally failed) by
111 /// the generation driver independently of this caller's timeout, so the
112 /// caller cannot tell from this reason alone whether the row eventually
113 /// committed, or even which of those two states it was in when the
114 /// deadline elapsed. Retrying is only safe for an idempotent caller —
115 /// the prior submission may still land. When this reason degrades an
116 /// admission-degrade-safe read's own audit obligation, it is counted
117 /// separately from [`Self::QueueAdmissionExhausted`] — see
118 /// `pack::audit_admission_unresolved_obligation_count` — precisely
119 /// because a row counted here may still commit, unlike one refused
120 /// before enqueue.
121 AdmissionDeadlineExpired,
122 /// Reached only through `AuditBatch::submit_until_resolved`: the row's
123 /// [`Self::AdmissionDeadlineExpired`] wait had already elapsed, and this
124 /// caller's own `AuditBatchConfig::resolution_deadline` then also
125 /// elapsed still waiting for the row's real generation outcome. The
126 /// caller reaches this only after its domain effect has already
127 /// committed, so the effect is never retried and the row is never
128 /// re-enqueued — but unlike an ordinary commit, the caller now learns
129 /// the effect committed while the audit outcome itself is unresolved.
130 /// Same as [`Self::AdmissionDeadlineExpired`], the row is left exactly
131 /// where the driver holds it (`state.pending`, or already mid-generation)
132 /// for the driver to resolve independently — this reason performs no
133 /// removal. Kept distinct from [`Self::AdmissionDeadlineExpired`] so a
134 /// caller, and diagnostics reading this reason, can tell a merely-slow
135 /// admission wait apart from a resolution wait that gave up entirely.
136 ResolutionDeadlineExpired,
137 /// A row shared this generation's id with a previously stored row whose
138 /// columns or observation projection did not match exactly.
139 IdentityConflict,
140 /// `classify_store_error` judged the store's error non-retryable, and the
141 /// generation stopped on that attempt. Usually that is the first attempt,
142 /// but not necessarily: a generation whose earlier attempts failed
143 /// retryably and whose next one returns a non-retryable error reports
144 /// this reason too, because the attempt that decided the outcome is the
145 /// non-retryable one. Distinct from [`Self::RetryExhausted`], which the
146 /// classifier judged safe to retry on every attempt and which failed
147 /// anyway once they ran out — this reason carries no such hope. Whatever
148 /// the store returned on the deciding attempt, it is not the kind of
149 /// failure `AuditBatchConfig::max_commit_attempts` exists to ride out,
150 /// so a caller or an automated retry policy reading this reason should
151 /// not schedule a bare retry of the same call and should instead treat
152 /// it as a storage fault needing attention.
153 StoreFailure,
154 /// The store returned a `classify_store_error`-retryable error on every
155 /// one of `AuditBatchConfig::max_commit_attempts` attempts for this
156 /// generation, and the last attempt still failed retryable. Kept
157 /// distinct from [`Self::StoreFailure`] — the same way
158 /// [`Self::ResolutionDeadlineExpired`] is kept distinct from
159 /// [`Self::AdmissionDeadlineExpired`] — so a caller, and diagnostics
160 /// reading this reason, can tell a store call the classifier judged
161 /// hopeless apart from one that kept failing a condition (write-queue or
162 /// writer-task pressure, pool or timeout) the classifier judged
163 /// transient. The underlying condition may still be
164 /// transient at the moment attempts run out (a daemon restart, pool
165 /// pressure outlasting the configured backoff), so an operator or an
166 /// automated retry policy sitting above this batch can choose to wait
167 /// longer and try again rather than treating it identically to
168 /// [`Self::StoreFailure`]. This reason changes no tolerance, deadline,
169 /// retry count, or backoff on its own — it only names which of the two
170 /// causes produced the generation's failure.
171 RetryExhausted,
172 /// The configured `EventStore` backend does not implement
173 /// `append_events_idempotent`.
174 IdempotencyUnsupported,
175 /// The generation driver task panicked, or the supervisor awaiting it
176 /// unwound while armed.
177 DriverPanicked,
178 /// The generation driver task was cancelled/aborted, or the supervisor
179 /// awaiting it was dropped mid-await (shutdown abort) while armed.
180 DriverCancelled,
181 /// The child driver's `JoinHandle` was lost — dropped without ever being
182 /// inspected — so its outcome could not be classified.
183 DriverJoinLost,
184 /// The driver returned `Ok` but locked batch state was not proved
185 /// terminally consistent afterward (`in_flight` still set, or the
186 /// generation's rows were never resolved).
187 DriverExitedInconsistent,
188 /// The driver's own bound on a single generation's
189 /// `EventStore::append_events_idempotent()` call
190 /// (`supervisor_loop`'s `driver_append_deadline`) elapsed while that
191 /// call was still in flight. Unlike [`Self::AdmissionDeadlineExpired`]
192 /// and [`Self::ResolutionDeadlineExpired`], which bound only how long a
193 /// *caller* keeps waiting while the row stays wherever the driver holds
194 /// it, this bounds the driver's own hold: every waiter on the abandoned
195 /// generation is resolved with this reason and the generation is
196 /// removed from the driver's in-flight state, so a stalled store call
197 /// cannot pin `pending` at `max_pending_rows` and starve all later
198 /// admission (khive#2331). The underlying append is not cancelled — it
199 /// may not be safely cancellable mid-flight — so it is left to run to
200 /// completion in the background and its eventual result, whatever it
201 /// is, is discarded; no waiter is still listening for it. The caller's
202 /// domain effect (if any already committed before this row was
203 /// enqueued) is never retried, and the row is never re-enqueued.
204 DriverAppendAbandoned,
205 /// Before spawning a generation's child, the driver found
206 /// `AuditBatchConfig::max_abandoned_appends` detached appends already
207 /// outstanding from prior [`Self::DriverAppendAbandoned`] generations
208 /// (khive#2331). The store is treated as wedged: this generation's rows
209 /// are shed without ever attempting an append — no child task is
210 /// spawned, no store call is made, and every waiter resolves with this
211 /// reason immediately, well inside `driver_append_deadline`. Same
212 /// non-retry contract as [`Self::DriverAppendAbandoned`]: any
213 /// already-committed domain effect is not retried and the row is never
214 /// re-enqueued, and pure-observability producers record degradation.
215 /// The driver reattempts an append on the next generation as soon as an
216 /// outstanding append returns (commit or failure) and the count drops
217 /// back below the cap — recovery needs no timer of its own.
218 StoreWedged,
219}
220
221/// One row accepted for batching: the immutable event identity plus the
222/// producer that minted it, used for classification and, on failure,
223/// degradation accounting.
224pub struct PreparedAuditRow {
225 pub event: Event,
226 pub producer: AuditProducer,
227}
228
229/// What [`AuditBatchControl::submit`] resolves to on a non-error outcome.
230#[derive(Debug, Clone, Copy, PartialEq, Eq)]
231pub enum AuditCommitOutcome {
232 /// The row was freshly inserted.
233 Committed,
234 /// A prior row with the same identity already matched exactly (internal
235 /// retry replaying the same producer-minted identity).
236 AlreadyPresentIdentical,
237}
238
239/// Production-visible snapshot of [`AuditBatch::health_metrics`]. See there
240/// for field semantics. `khive_db::diagnostics::RuntimeAuditBatchMetrics` carries
241/// these three fields plus `admission_refused_obligations`,
242/// `retry_exhausted_obligations`, and `admission_unresolved_obligations`,
243/// which are sourced from process-wide
244/// counters outside `AuditBatch` rather than from this struct — see
245/// `VerbRegistry::audit_batch_metrics`.
246#[derive(Debug, Clone, Copy, PartialEq, Eq)]
247pub struct AuditBatchHealthMetrics {
248 pub flush_failures: u64,
249 pub degraded_rows: u64,
250 pub degraded: bool,
251 /// Detached [`AuditTerminalReason::DriverAppendAbandoned`] appends that
252 /// eventually returned a commit, after their generation had already
253 /// resolved every waiter with `DriverAppendAbandoned`. Lets an operator
254 /// see that a store recorded as wedged later drained.
255 pub late_append_commits: u64,
256 /// Detached [`AuditTerminalReason::DriverAppendAbandoned`] appends that
257 /// eventually returned a failure (store error, panic, or cancellation),
258 /// after their generation had already resolved every waiter with
259 /// `DriverAppendAbandoned`.
260 pub late_append_failures: u64,
261}
262
263/// Tunables for the batch seam. Defaults are conservative; every field is
264/// exercised by at least one mechanism test.
265#[derive(Debug, Clone)]
266pub struct AuditBatchConfig {
267 pub max_pending_rows: std::num::NonZeroUsize,
268 pub max_rows_per_generation: std::num::NonZeroUsize,
269 pub max_commit_attempts: std::num::NonZeroU8,
270 pub retry_backoff: Duration,
271 pub admission_deadline: Duration,
272 /// Caps how much longer `AuditBatch::submit_until_resolved` keeps
273 /// waiting on a row's real generation outcome once `admission_deadline`
274 /// has already elapsed on it. Without this bound a generation stuck on a
275 /// stalled `EventStore::append_events_idempotent` call retains the
276 /// completed write's caller, its request slot, and its audit-lane waiter
277 /// forever, exhausting both request and audit capacity (khive#2331).
278 /// Must be at least `admission_deadline` — validated (debug-only) in
279 /// [`AuditBatch::new`]. Defaults to 6x `admission_deadline`.
280 ///
281 /// Also the single source value the driver's own per-generation append
282 /// bound (`supervisor_loop`'s `driver_append_deadline`) is derived from
283 /// — that bound exists so a stalled append cannot keep the driver from
284 /// ever draining `pending` again, which is what actually exhausts
285 /// admission for every later caller, not just the one already waiting
286 /// on the stuck row. See [`AuditTerminalReason::DriverAppendAbandoned`].
287 pub resolution_deadline: Duration,
288 /// Caps how many [`AuditTerminalReason::DriverAppendAbandoned`] appends
289 /// may be outstanding (handed to a detached task, still running) at
290 /// once. Each outstanding append retains up to
291 /// `max_rows_per_generation` events until it finally returns, so the
292 /// retained-buffer bound this places on a wedged store is
293 /// `max_abandoned_appends * max_rows_per_generation` rows — without it,
294 /// a store whose append never returns mints one such task per
295 /// `driver_append_deadline` with no cap, and both the live task count
296 /// and the retained event batches inside them grow until the process
297 /// exits (khive#2331). Once the cap is reached, `supervisor_loop` sheds
298 /// further generations with [`AuditTerminalReason::StoreWedged`] instead
299 /// of attempting another append; it resumes attempting appends as soon
300 /// as an outstanding one returns and the count drops back below the
301 /// cap. Defaults to 4.
302 pub max_abandoned_appends: std::num::NonZeroUsize,
303}
304
305impl Default for AuditBatchConfig {
306 fn default() -> Self {
307 let admission_deadline = Duration::from_secs(5);
308 Self {
309 max_pending_rows: std::num::NonZeroUsize::new(4096).unwrap(),
310 max_rows_per_generation: std::num::NonZeroUsize::new(256).unwrap(),
311 max_commit_attempts: std::num::NonZeroU8::new(3).unwrap(),
312 retry_backoff: Duration::from_millis(20),
313 admission_deadline,
314 resolution_deadline: admission_deadline * 6,
315 max_abandoned_appends: std::num::NonZeroUsize::new(4).unwrap(),
316 }
317 }
318}
319
320#[async_trait::async_trait]
321pub trait AuditBatchControl: Send + Sync {
322 async fn submit(
323 &self,
324 row: PreparedAuditRow,
325 ) -> Result<AuditCommitOutcome, AuditTerminalReason>;
326 async fn quiesce(&self) -> Result<(), AuditTerminalReason>;
327 async fn close_and_drain(&self) -> Result<(), AuditTerminalReason>;
328}
329
330#[derive(Debug, Clone, PartialEq, Eq)]
331enum Lifecycle {
332 Open,
333 Closing,
334 Closed,
335 Failed(AuditTerminalReason),
336}
337
338struct Waiting {
339 event: Event,
340 producer: AuditProducer,
341 responder: oneshot::Sender<Result<AuditCommitOutcome, AuditTerminalReason>>,
342}
343
344/// A committed/failed generation's accounting. Detailed history is retained when
345/// compiled for tests or with `test-internals`; ordinary builds keep cumulative counters.
346#[derive(Debug, Clone, PartialEq, Eq)]
347pub struct AuditGenerationSnapshot {
348 pub generation_id: u64,
349 pub submitted_rows: u64,
350 pub committed_rows: u64,
351 pub store_batch_calls: u64,
352 pub terminal_reason: Option<AuditTerminalReason>,
353}
354
355struct State {
356 lifecycle: Lifecycle,
357 pending: Vec<Waiting>,
358 in_flight_generation: Option<u64>,
359 driver_active: bool,
360 physical_reservations: usize,
361 next_generation_id: u64,
362 submitted_rows: u64,
363 committed_rows: u64,
364 store_batch_calls: u64,
365 #[cfg(any(test, feature = "test-internals"))]
366 generations: Vec<AuditGenerationSnapshot>,
367 flush_failures: u64,
368 degraded_rows: u64,
369 degraded: bool,
370 /// Detached `DriverAppendAbandoned` appends currently still running,
371 /// bounded by `AuditBatchConfig::max_abandoned_appends`. Incremented
372 /// when `supervisor_loop` abandons a generation and hands its child to a
373 /// detached task; decremented by that task once the child finally
374 /// returns.
375 outstanding_abandoned_appends: usize,
376 late_append_commits: u64,
377 late_append_failures: u64,
378}
379
380impl State {
381 fn new() -> Self {
382 Self {
383 lifecycle: Lifecycle::Open,
384 pending: Vec::new(),
385 in_flight_generation: None,
386 driver_active: false,
387 physical_reservations: 0,
388 next_generation_id: 0,
389 submitted_rows: 0,
390 committed_rows: 0,
391 store_batch_calls: 0,
392 #[cfg(any(test, feature = "test-internals"))]
393 generations: Vec::new(),
394 flush_failures: 0,
395 degraded_rows: 0,
396 degraded: false,
397 outstanding_abandoned_appends: 0,
398 late_append_commits: 0,
399 late_append_failures: 0,
400 }
401 }
402
403 /// No-op outside `test`/`test-internals`; operators read [`AuditBatch::health_metrics`].
404 fn record_generation(&mut self, generation: AuditGenerationSnapshot) {
405 #[cfg(any(test, feature = "test-internals"))]
406 self.generations.push(generation);
407 #[cfg(not(any(test, feature = "test-internals")))]
408 let _ = generation;
409 }
410
411 fn is_idle(&self) -> bool {
412 self.pending.is_empty() && self.in_flight_generation.is_none() && !self.driver_active
413 }
414}
415
416struct Inner {
417 state: Mutex<State>,
418 // Completion witnesses only: physical shutdown never aborts admitted work.
419 physical_tasks: Mutex<Vec<tokio::task::AbortHandle>>,
420}
421
422struct PhysicalReservation(Arc<Inner>);
423
424impl Drop for PhysicalReservation {
425 fn drop(&mut self) {
426 self.0.state.lock().physical_reservations -= 1;
427 }
428}
429
430impl Inner {
431 fn reserve_physical(self: &Arc<Self>) -> PhysicalReservation {
432 self.state.lock().physical_reservations += 1;
433 PhysicalReservation(Arc::clone(self))
434 }
435
436 fn spawn_physical<F, T>(
437 self: &Arc<Self>,
438 registration: PhysicalReservation,
439 future: F,
440 ) -> JoinHandle<T>
441 where
442 F: std::future::Future<Output = T> + Send + 'static,
443 T: Send + 'static,
444 {
445 let running = self.reserve_physical();
446 let handle = tokio::spawn(async move {
447 let result = future.await;
448 drop(running);
449 result
450 });
451 {
452 let mut tasks = self.physical_tasks.lock();
453 tasks.retain(|task| !task.is_finished());
454 tasks.push(handle.abort_handle());
455 }
456 // A very short task can finish before its witness is published. Keep
457 // registration reserved until publication to close that zero-work gap.
458 drop(registration);
459 handle
460 }
461
462 fn physically_finished(&self) -> bool {
463 self.state.lock().physical_reservations == 0
464 && self
465 .physical_tasks
466 .lock()
467 .iter()
468 .all(|task| task.is_finished())
469 }
470
471 /// Wins once: the first abnormal driver/supervisor exit sets `Failed`
472 /// and drains every accepted waiter (this generation's, plus anything
473 /// still queued for a future one) with the same typed reason. A later
474 /// abnormal path observes the existing terminal state and does not
475 /// double-count (R1).
476 fn fail_driver(&self, reason: AuditTerminalReason, in_flight_waiters: Vec<Waiting>) {
477 let mut state = self.state.lock();
478 let already_failed = matches!(state.lifecycle, Lifecycle::Failed(_));
479 if !already_failed {
480 state.lifecycle = Lifecycle::Failed(reason);
481 }
482 state.driver_active = false;
483 state.in_flight_generation = None;
484 let drained: Vec<Waiting> = std::mem::take(&mut state.pending);
485 if !already_failed {
486 state.flush_failures += 1;
487 state.degraded = true;
488 let generation_id = state.next_generation_id;
489 state.record_generation(AuditGenerationSnapshot {
490 generation_id,
491 submitted_rows: (in_flight_waiters.len() + drained.len()) as u64,
492 committed_rows: 0,
493 store_batch_calls: 0,
494 terminal_reason: Some(reason),
495 });
496 state.next_generation_id += 1;
497 }
498 drop(state);
499 for waiting in in_flight_waiters.into_iter().chain(drained) {
500 record_degradation_if_pure(self, waiting.producer);
501 let _ = waiting.responder.send(Err(reason));
502 }
503 }
504
505 fn record_degradation(&self) {
506 let mut state = self.state.lock();
507 state.degraded_rows += 1;
508 state.degraded = true;
509 }
510}
511
512fn record_degradation_if_pure(inner: &Inner, producer: AuditProducer) {
513 if classify(producer) == AuditProductionClass::PureObservability {
514 inner.record_degradation();
515 }
516}
517
518/// Armed for the lifetime of one generation's supervision. If dropped while
519/// still armed — the supervisor's own frame unwinding (panic) or being
520/// dropped mid-await (cancellation/shutdown abort) — the guard fails the
521/// generation before returning control to whatever tore it down, so the
522/// caller never observes a background-task-count restoration that raced
523/// ahead of the failure broadcast (R1).
524struct SupervisorGuard<'a> {
525 inner: &'a Arc<Inner>,
526 waiting: Option<Vec<Waiting>>,
527}
528
529impl<'a> SupervisorGuard<'a> {
530 fn armed(inner: &'a Arc<Inner>, waiting: Vec<Waiting>) -> Self {
531 Self {
532 inner,
533 waiting: Some(waiting),
534 }
535 }
536
537 /// Clean disarm: the caller has fully classified the outcome and taken
538 /// ownership of the waiters to resolve them itself.
539 fn disarm(mut self) -> Vec<Waiting> {
540 self.waiting.take().unwrap_or_default()
541 }
542
543 /// Explicit failure classification reached without unwinding/cancelling
544 /// this frame (e.g. a `JoinError` was returned normally). Equivalent to
545 /// what `Drop` does when armed, but callable inline.
546 fn fail(mut self, reason: AuditTerminalReason) {
547 if let Some(waiting) = self.waiting.take() {
548 self.inner.fail_driver(reason, waiting);
549 }
550 }
551}
552
553impl Drop for SupervisorGuard<'_> {
554 fn drop(&mut self) {
555 let Some(waiting) = self.waiting.take() else {
556 return;
557 };
558 let reason = if std::thread::panicking() {
559 AuditTerminalReason::DriverPanicked
560 } else {
561 AuditTerminalReason::DriverCancelled
562 };
563 self.inner.fail_driver(reason, waiting);
564 }
565}
566
567enum RetryDecision {
568 Retry,
569 Terminal(AuditTerminalReason),
570}
571
572fn classify_store_error(err: &StorageError) -> RetryDecision {
573 match err.without_sqlite_write_stage() {
574 // This replay is safe only because the batch append is idempotent.
575 StorageError::WriteOutcomeUnknown { .. } => RetryDecision::Retry,
576 StorageError::WriterOwnerUnavailable { source, .. } => classify_store_error(source),
577 StorageError::WriteQueueFull { .. } | StorageError::WriterTaskBusy { .. } => {
578 RetryDecision::Retry
579 }
580 StorageError::CapacityUnavailable {
581 phase: khive_storage::CapacityUnavailablePhase::LockContended,
582 ..
583 } => RetryDecision::Retry,
584 // Transient availability conditions, not judgments on the batch. The
585 // events-daemon forwarding lane (ADR-170) reports an unreachable or
586 // stalled daemon as `Pool`/`Timeout`; the direct SQL path reports
587 // acquisition pressure the same way. Both are exactly what the
588 // configured bounded retries exist for — treating them as terminal
589 // would abandon a generation on the first blip of a daemon restart.
590 StorageError::Pool { .. } | StorageError::Timeout { .. } => RetryDecision::Retry,
591 // The writer task wraps any request operation that failed and was rolled
592 // back, whatever the cause, so the wrapper alone does not say whether a
593 // retry can succeed: a missing column fails the same way on every
594 // attempt. Judge the cause it carries. A request left in an unknown
595 // state is replayed regardless, because the append is idempotent.
596 StorageError::WriterTaskRequestFailed {
597 request_state,
598 source,
599 } => match request_state {
600 WriterTaskRequestState::NotStarted | WriterTaskRequestState::TransactionRolledBack => {
601 if is_sqlite_busy_or_locked(source) {
602 RetryDecision::Retry
603 } else {
604 classify_store_error(source)
605 }
606 }
607 WriterTaskRequestState::SideEffectsUnknown => RetryDecision::Retry,
608 },
609 StorageError::WriterTaskTerminated { request_state, .. } => match request_state {
610 WriterTaskRequestState::NotStarted | WriterTaskRequestState::TransactionRolledBack => {
611 RetryDecision::Retry
612 }
613 WriterTaskRequestState::SideEffectsUnknown => RetryDecision::Retry,
614 },
615 StorageError::Unsupported { operation, .. }
616 if operation.as_ref() == "append_events_idempotent" =>
617 {
618 RetryDecision::Terminal(AuditTerminalReason::IdempotencyUnsupported)
619 }
620 _ => RetryDecision::Terminal(AuditTerminalReason::StoreFailure),
621 }
622}
623
624/// Whether a failed writer request's preserved cause is SQLite contention. The
625/// writer keeps the request body's driver error as the cause, and a driver
626/// error is not one of the typed transient variants `classify_store_error`
627/// retries on its own.
628fn is_sqlite_busy_or_locked(err: &StorageError) -> bool {
629 let StorageError::Driver { source, .. } = err.without_sqlite_write_stage() else {
630 return false;
631 };
632 matches!(
633 source
634 .downcast_ref::<rusqlite::Error>()
635 .and_then(|error| error.sqlite_error_code()),
636 Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked)
637 )
638}
639
640enum GenerationResult {
641 Committed(Vec<EventAppendDisposition>),
642 Failed(AuditTerminalReason),
643 /// Fault-injection only: proves the supervisor's post-`Ok` consistency
644 /// check actually runs.
645 #[cfg_attr(not(any(test, feature = "fault-injection")), allow(dead_code))]
646 FakedInconsistent,
647}
648
649/// The driver's own bound on how long `supervisor_loop` waits for one
650/// generation's `run_generation` child task before abandoning it
651/// (khive#2331; see [`AuditTerminalReason::DriverAppendAbandoned`]).
652///
653/// Derived from `resolution_deadline` rather than a second config field, at
654/// 3x it. That margin is load-bearing, not arbitrary: `AuditBatch::new`
655/// already enforces `resolution_deadline >= admission_deadline`, so the
656/// worst-case *caller*-side wait for a `submit_until_resolved` row —
657/// `admission_deadline` then `resolution_deadline` in sequence — is always
658/// `<= 2 * resolution_deadline`. Bounding the driver at `3 *
659/// resolution_deadline` guarantees it can never resolve a still-waiting
660/// caller's row with `DriverAppendAbandoned` ahead of that caller's own,
661/// more specific `ResolutionDeadlineExpired` reason.
662///
663/// This bound alone only stops the driver from holding one stalled
664/// generation forever — a store whose append never returns still mints one
665/// detached append per `driver_append_deadline` with nothing capping how
666/// many run at once. `AuditBatchConfig::max_abandoned_appends` closes that:
667/// once that many detached appends are outstanding, further generations are
668/// shed with [`AuditTerminalReason::StoreWedged`] instead of attempting
669/// another append. Combined, the two bounds guarantee at most
670/// `max_abandoned_appends` live detached appends at any time, at most that
671/// many retained event batches, and a shed generation costs no store work
672/// at all — it resolves inside this function's caller without ever calling
673/// `run_generation`.
674fn driver_append_deadline(config: &AuditBatchConfig) -> Duration {
675 config.resolution_deadline.saturating_mul(3)
676}
677
678async fn run_generation(
679 store: Arc<dyn EventStore>,
680 events: Vec<Event>,
681 config: Arc<AuditBatchConfig>,
682) -> GenerationResult {
683 #[cfg(any(test, feature = "fault-injection"))]
684 if fault::CHILD_PANIC.swap(false, Ordering::SeqCst) {
685 panic!("adr133 fault injection: audit_batch child_panic");
686 }
687 #[cfg(any(test, feature = "fault-injection"))]
688 if fault::INCONSISTENT_EXIT.swap(false, Ordering::SeqCst) {
689 return GenerationResult::FakedInconsistent;
690 }
691
692 let mut attempt: u8 = 0;
693 loop {
694 attempt += 1;
695 match store.append_events_idempotent(events.clone()).await {
696 Ok(result) => return GenerationResult::Committed(result.rows),
697 Err(err) => {
698 let reason = match classify_store_error(&err) {
699 RetryDecision::Retry if attempt < config.max_commit_attempts.get() => {
700 tokio::time::sleep(config.retry_backoff).await;
701 continue;
702 }
703 RetryDecision::Retry => AuditTerminalReason::RetryExhausted,
704 RetryDecision::Terminal(reason) => reason,
705 };
706 tracing::warn!(
707 error = %err,
708 attempts = attempt,
709 ?reason,
710 "audit generation failed; its rows were not committed"
711 );
712 return GenerationResult::Failed(reason);
713 }
714 }
715 }
716}
717
718/// The batch owner. Constructed once per configured `EventStore`; every
719/// dispatch-audit call site routes its row through [`AuditBatch::submit`]
720/// instead of taking its own writer-task acquisition.
721pub struct AuditBatch {
722 inner: Arc<Inner>,
723 store: Arc<dyn EventStore>,
724 config: Arc<AuditBatchConfig>,
725 supervisor: Mutex<Option<JoinHandle<()>>>,
726}
727
728impl AuditBatch {
729 pub fn new(store: Arc<dyn EventStore>, config: AuditBatchConfig) -> Arc<Self> {
730 debug_assert!(
731 config.resolution_deadline >= config.admission_deadline,
732 "resolution_deadline ({:?}) must be at least admission_deadline ({:?})",
733 config.resolution_deadline,
734 config.admission_deadline
735 );
736 Arc::new(Self {
737 inner: Arc::new(Inner {
738 state: Mutex::new(State::new()),
739 physical_tasks: Mutex::new(Vec::new()),
740 }),
741 store,
742 config: Arc::new(config),
743 supervisor: Mutex::new(None),
744 })
745 }
746
747 /// Process-lifetime audit-batch health counters, for the registry/
748 /// runtime owner to feed into `db_diagnostics` (D8's operator surface).
749 /// Unlike `test_internals::AuditBatchSnapshot::metrics_snapshot`,
750 /// which is test-only (the module is cfg-gated and invisible to the
751 /// default doc build, so an intra-doc link cannot resolve), this is
752 /// always available.
753 pub fn health_metrics(&self) -> AuditBatchHealthMetrics {
754 let state = self.inner.state.lock();
755 AuditBatchHealthMetrics {
756 flush_failures: state.flush_failures,
757 degraded_rows: state.degraded_rows,
758 degraded: state.degraded,
759 late_append_commits: state.late_append_commits,
760 late_append_failures: state.late_append_failures,
761 }
762 }
763
764 fn spawn_supervisor_if_idle(&self, registration: PhysicalReservation) {
765 let inner = self.inner.clone();
766 let store = self.store.clone();
767 let config = self.config.clone();
768 let handle = self.inner.spawn_physical(registration, async move {
769 supervisor_loop(inner, store, config).await;
770 });
771 *self.supervisor.lock() = Some(handle);
772 }
773
774 /// Stop admission and wait until all accepted append tasks and their
775 /// supervisors/reapers have physically finished. Unlike `close_and_drain`,
776 /// a driver-abandoned append still holds this barrier until it returns.
777 ///
778 /// Task completion witnesses stay on the batch, so cancelling this wait
779 /// neither detaches a taken join nor loses the barrier for a later call.
780 /// No admitted write is aborted. This may wait indefinitely for a store
781 /// that never returns; callers may cancel their wait without cancelling it.
782 /// The batch's own store Arc remains owned until the batch is dropped.
783 pub(crate) async fn close_and_wait_for_physical_completion(
784 &self,
785 ) -> Result<(), AuditTerminalReason> {
786 {
787 let mut state = self.inner.state.lock();
788 if matches!(state.lifecycle, Lifecycle::Open) {
789 state.lifecycle = Lifecycle::Closing;
790 }
791 }
792 let result = AuditBatchControl::quiesce(self).await;
793 while !self.inner.physically_finished() {
794 tokio::time::sleep(Duration::from_millis(2)).await;
795 }
796 {
797 let mut state = self.inner.state.lock();
798 if result.is_ok() && matches!(state.lifecycle, Lifecycle::Closing) {
799 state.lifecycle = Lifecycle::Closed;
800 }
801 }
802 result
803 }
804
805 /// Enqueue one row and keep waiting for its real generation outcome past
806 /// the ordinary admission wait deadline, up to `resolution_deadline`.
807 ///
808 /// This is the narrow khive#2256 seam for a successful operation whose
809 /// domain effect has already committed. Returning
810 /// [`AuditTerminalReason::AdmissionDeadlineExpired`] there would report a
811 /// false operation failure and invite an unsafe retry while the same
812 /// audit row remains enqueued. Pre-enqueue refusal and genuine terminal
813 /// generation failures still return normally. The post-admission wait is
814 /// itself bounded by `resolution_deadline`
815 /// ([`AuditTerminalReason::ResolutionDeadlineExpired`]) so a stalled
816 /// store cannot retain this caller, its request slot, and its audit-lane
817 /// waiter forever (khive#2331).
818 ///
819 /// The two deadlines therefore mean different things to the caller, and
820 /// callers key on the difference. Admission expiry means the row is
821 /// enqueued and this seam keeps waiting; the generation commits it
822 /// independently, so a dispatch that reaches it reports its committed
823 /// result and counts the row as unresolved. Resolution expiry means the
824 /// caller waited for the real outcome and never received one: the commit
825 /// is unconfirmed, not proven absent. That is returned as a terminal
826 /// reason and the dispatch propagates it as a structured
827 /// committed-outcome error carrying the domain result, with no retryable
828 /// context, for every verb rather than only for the receipt verb. Never
829 /// replay the handler or re-enqueue the row on it; the original driver
830 /// work continues on its own.
831 pub(crate) async fn submit_until_resolved(
832 &self,
833 row: PreparedAuditRow,
834 ) -> Result<AuditCommitOutcome, AuditTerminalReason> {
835 self.submit_with_wait_policy(row, true).await
836 }
837
838 async fn submit_with_wait_policy(
839 &self,
840 row: PreparedAuditRow,
841 wait_until_resolved: bool,
842 ) -> Result<AuditCommitOutcome, AuditTerminalReason> {
843 // Pre-enqueue validation (invariant 3): a malformed row is rejected
844 // before it can share a generation with anyone else's.
845 if self.store.preflight_event(&row.event).is_err() {
846 return Err(AuditTerminalReason::PreflightRejected);
847 }
848
849 let producer = row.producer;
850 let (tx, mut rx) = oneshot::channel();
851 let supervisor_registration = {
852 let mut state = self.inner.state.lock();
853 match state.lifecycle {
854 Lifecycle::Closed | Lifecycle::Closing => {
855 return Err(AuditTerminalReason::AdmissionClosed)
856 }
857 Lifecycle::Failed(reason) => return Err(reason),
858 Lifecycle::Open => {}
859 }
860 if state.pending.len() >= self.config.max_pending_rows.get() {
861 return Err(AuditTerminalReason::QueueAdmissionExhausted);
862 }
863 state.pending.push(Waiting {
864 event: row.event,
865 producer,
866 responder: tx,
867 });
868 state.submitted_rows += 1;
869 let need_spawn = !state.driver_active;
870 if need_spawn {
871 state.driver_active = true;
872 // Publish this obligation under the admission lock, before a
873 // concurrent physical close can observe terminal/idle state.
874 state.physical_reservations += 1;
875 Some(PhysicalReservation(Arc::clone(&self.inner)))
876 } else {
877 None
878 }
879 };
880 if let Some(registration) = supervisor_registration {
881 self.spawn_supervisor_if_idle(registration);
882 }
883
884 if wait_until_resolved {
885 match tokio::time::timeout(self.config.admission_deadline, &mut rx).await {
886 Ok(Ok(result)) => return result,
887 Ok(Err(_recv_error)) => return Err(AuditTerminalReason::DriverJoinLost),
888 Err(_elapsed) => tracing::warn!(
889 ?producer,
890 "strict audit obligation remains enqueued after the admission wait deadline; \
891 waiting up to the resolution deadline for its real terminal outcome"
892 ),
893 }
894 return match tokio::time::timeout(self.config.resolution_deadline, rx).await {
895 Ok(Ok(result)) => result,
896 Ok(Err(_recv_error)) => Err(AuditTerminalReason::DriverJoinLost),
897 // Same non-removal contract as the `AdmissionDeadlineExpired`
898 // arm below: the row is left exactly where the driver holds
899 // it. The caller's domain effect already committed by the
900 // time it reached this wait, so it is never retried here;
901 // the audit outcome itself is what remains unresolved.
902 Err(_elapsed) => {
903 tracing::warn!(
904 ?producer,
905 "strict audit obligation remains unresolved after the resolution \
906 deadline; giving up on this caller's wait — the committed effect is \
907 not retried and the row is not re-enqueued"
908 );
909 Err(AuditTerminalReason::ResolutionDeadlineExpired)
910 }
911 };
912 }
913
914 match tokio::time::timeout(self.config.admission_deadline, rx).await {
915 Ok(Ok(result)) => result,
916 Ok(Err(_recv_error)) => Err(AuditTerminalReason::DriverJoinLost),
917 // The row was already pushed onto `state.pending` above (and
918 // `submitted_rows` incremented) before this wait began — this is
919 // a deadline elapsing on an enqueued row, not a queue-full
920 // refusal, so it gets its own terminal reason (khive#2117,
921 // khive#2208). The row is left in place for the driver to drain;
922 // this arm performs no removal.
923 Err(_elapsed) => Err(AuditTerminalReason::AdmissionDeadlineExpired),
924 }
925 }
926}
927
928async fn supervisor_loop(
929 inner: Arc<Inner>,
930 store: Arc<dyn EventStore>,
931 config: Arc<AuditBatchConfig>,
932) {
933 loop {
934 let (waiting, wedged_generation_id) = {
935 let mut state = inner.state.lock();
936 if matches!(state.lifecycle, Lifecycle::Failed(_)) {
937 state.driver_active = false;
938 break;
939 }
940 if state.pending.is_empty() {
941 state.driver_active = false;
942 break;
943 }
944 let take_n = state
945 .pending
946 .len()
947 .min(config.max_rows_per_generation.get());
948 let waiting: Vec<Waiting> = state.pending.drain(..take_n).collect();
949 let generation_id = state.next_generation_id;
950 state.next_generation_id += 1;
951 // khive#2331: a store whose append never returns must not mint
952 // an unbounded number of detached appends. Before committing to
953 // spawning this generation's child, check whether the cap is
954 // already saturated by prior abandoned generations still
955 // running in the background — if so, this generation is shed
956 // below instead of ever calling the store.
957 if state.outstanding_abandoned_appends >= config.max_abandoned_appends.get() {
958 (waiting, Some(generation_id))
959 } else {
960 state.in_flight_generation = Some(generation_id);
961 (waiting, None)
962 }
963 };
964
965 if let Some(generation_id) = wedged_generation_id {
966 tracing::warn!(
967 generation_id,
968 max_abandoned_appends = config.max_abandoned_appends.get(),
969 "audit store treated as wedged: max_abandoned_appends detached appends are \
970 already outstanding; shedding this generation without attempting an append"
971 );
972 let submitted = waiting.len() as u64;
973 {
974 let mut state = inner.state.lock();
975 state.flush_failures += 1;
976 state.record_generation(AuditGenerationSnapshot {
977 generation_id,
978 submitted_rows: submitted,
979 committed_rows: 0,
980 store_batch_calls: 0,
981 terminal_reason: Some(AuditTerminalReason::StoreWedged),
982 });
983 }
984 for w in waiting {
985 record_degradation_if_pure(&inner, w.producer);
986 let _ = w.responder.send(Err(AuditTerminalReason::StoreWedged));
987 }
988 continue;
989 }
990
991 #[cfg(any(test, feature = "fault-injection"))]
992 if fault::SUPERVISOR_PANIC.swap(false, Ordering::SeqCst) {
993 let _guard = SupervisorGuard::armed(&inner, waiting);
994 panic!("adr133 fault injection: audit_batch supervisor_panic");
995 }
996 #[cfg(any(test, feature = "fault-injection"))]
997 if fault::SUPERVISOR_SLEEP_BEFORE_SPAWN.swap(false, Ordering::SeqCst) {
998 let guard = SupervisorGuard::armed(&inner, waiting);
999 tokio::time::sleep(Duration::from_secs(3600)).await;
1000 drop(guard);
1001 continue;
1002 }
1003
1004 let guard = SupervisorGuard::armed(&inner, waiting);
1005 let events: Vec<Event> = guard
1006 .waiting
1007 .as_ref()
1008 .expect("guard freshly armed")
1009 .iter()
1010 .map(|w| w.event.clone())
1011 .collect();
1012
1013 let mut child: JoinHandle<GenerationResult> = inner.spawn_physical(
1014 inner.reserve_physical(),
1015 run_generation(store.clone(), events, config.clone()),
1016 );
1017
1018 #[cfg(any(test, feature = "fault-injection"))]
1019 if fault::CHILD_CANCEL.swap(false, Ordering::SeqCst) {
1020 child.abort();
1021 }
1022 #[cfg(any(test, feature = "fault-injection"))]
1023 if fault::JOIN_LOST.swap(false, Ordering::SeqCst) {
1024 drop(child);
1025 guard.fail(AuditTerminalReason::DriverJoinLost);
1026 continue;
1027 }
1028
1029 let append_deadline = driver_append_deadline(&config);
1030 let join_result = match tokio::time::timeout(append_deadline, &mut child).await {
1031 Ok(join_result) => join_result,
1032 Err(_elapsed) => {
1033 // The append is still in flight and may not be safely
1034 // cancellable mid-write (a blocking storage call cannot be
1035 // forced to stop), so `child` is not aborted here — it is
1036 // handed to a detached reaper that drives it to completion
1037 // and discards whatever it eventually returns. Every waiter
1038 // on this generation is resolved now, and the driver loops
1039 // back to `pending` immediately instead of staying pinned on
1040 // this one stalled call, which is what actually starves
1041 // later admission (khive#2331).
1042 tracing::warn!(
1043 ?append_deadline,
1044 "audit generation append exceeded the driver's own bound; abandoning \
1045 this generation so admission capacity recovers for later callers"
1046 );
1047 let waiting = guard.disarm();
1048 let submitted = waiting.len() as u64;
1049 {
1050 let mut state = inner.state.lock();
1051 state.in_flight_generation = None;
1052 state.flush_failures += 1;
1053 state.outstanding_abandoned_appends += 1;
1054 let generation_id = state.next_generation_id.saturating_sub(1);
1055 state.record_generation(AuditGenerationSnapshot {
1056 generation_id,
1057 submitted_rows: submitted,
1058 committed_rows: 0,
1059 store_batch_calls: 0,
1060 terminal_reason: Some(AuditTerminalReason::DriverAppendAbandoned),
1061 });
1062 }
1063 for w in waiting {
1064 record_degradation_if_pure(&inner, w.producer);
1065 let _ = w
1066 .responder
1067 .send(Err(AuditTerminalReason::DriverAppendAbandoned));
1068 }
1069 let reaper_inner = inner.clone();
1070 inner.spawn_physical(inner.reserve_physical(), async move {
1071 let result = child.await;
1072 let mut state = reaper_inner.state.lock();
1073 state.outstanding_abandoned_appends =
1074 state.outstanding_abandoned_appends.saturating_sub(1);
1075 match result {
1076 Ok(GenerationResult::Committed(_)) => state.late_append_commits += 1,
1077 Ok(GenerationResult::Failed(_) | GenerationResult::FakedInconsistent)
1078 | Err(_) => state.late_append_failures += 1,
1079 }
1080 });
1081 continue;
1082 }
1083 };
1084 match join_result {
1085 Err(join_err) => {
1086 let reason = if join_err.is_panic() {
1087 AuditTerminalReason::DriverPanicked
1088 } else {
1089 AuditTerminalReason::DriverCancelled
1090 };
1091 guard.fail(reason);
1092 }
1093 Ok(GenerationResult::FakedInconsistent) => {
1094 guard.fail(AuditTerminalReason::DriverExitedInconsistent);
1095 }
1096 Ok(GenerationResult::Failed(reason)) => {
1097 let waiting = guard.disarm();
1098 let submitted = waiting.len() as u64;
1099 {
1100 let mut state = inner.state.lock();
1101 state.in_flight_generation = None;
1102 state.store_batch_calls += 1;
1103 state.flush_failures += 1;
1104 state.degraded = true;
1105 let generation_id = state.next_generation_id.saturating_sub(1);
1106 state.record_generation(AuditGenerationSnapshot {
1107 generation_id,
1108 submitted_rows: submitted,
1109 committed_rows: 0,
1110 store_batch_calls: 1,
1111 terminal_reason: Some(reason),
1112 });
1113 }
1114 for w in waiting {
1115 record_degradation_if_pure(&inner, w.producer);
1116 let _ = w.responder.send(Err(reason));
1117 }
1118 }
1119 Ok(GenerationResult::Committed(dispositions)) => {
1120 let waiting = guard.disarm();
1121 if dispositions.len() != waiting.len() {
1122 // Defensive: the store must preserve input order/length.
1123 // Treat a mismatch as the driver having exited in a
1124 // state that cannot be reconciled with the accepted
1125 // waiters.
1126 {
1127 let mut state = inner.state.lock();
1128 state.in_flight_generation = None;
1129 }
1130 inner.fail_driver(AuditTerminalReason::DriverExitedInconsistent, waiting);
1131 continue;
1132 }
1133 let mut committed_n = 0u64;
1134 for d in &dispositions {
1135 if !matches!(d, EventAppendDisposition::IdentityConflict) {
1136 committed_n += 1;
1137 }
1138 }
1139 let submitted = dispositions.len() as u64;
1140 {
1141 let mut state = inner.state.lock();
1142 state.in_flight_generation = None;
1143 state.store_batch_calls += 1;
1144 state.committed_rows += committed_n;
1145 let generation_id = state.next_generation_id.saturating_sub(1);
1146 state.record_generation(AuditGenerationSnapshot {
1147 generation_id,
1148 submitted_rows: submitted,
1149 committed_rows: committed_n,
1150 store_batch_calls: 1,
1151 terminal_reason: None,
1152 });
1153 }
1154 for (w, disposition) in waiting.into_iter().zip(dispositions) {
1155 let result = match disposition {
1156 EventAppendDisposition::Inserted => Ok(AuditCommitOutcome::Committed),
1157 EventAppendDisposition::AlreadyPresentIdentical => {
1158 Ok(AuditCommitOutcome::AlreadyPresentIdentical)
1159 }
1160 EventAppendDisposition::IdentityConflict => {
1161 record_degradation_if_pure(&inner, w.producer);
1162 Err(AuditTerminalReason::IdentityConflict)
1163 }
1164 };
1165 let _ = w.responder.send(result);
1166 }
1167 }
1168 }
1169 }
1170}
1171
1172#[async_trait::async_trait]
1173impl AuditBatchControl for AuditBatch {
1174 async fn submit(
1175 &self,
1176 row: PreparedAuditRow,
1177 ) -> Result<AuditCommitOutcome, AuditTerminalReason> {
1178 self.submit_with_wait_policy(row, false).await
1179 }
1180
1181 async fn quiesce(&self) -> Result<(), AuditTerminalReason> {
1182 loop {
1183 let (lifecycle, idle) = {
1184 let state = self.inner.state.lock();
1185 (state.lifecycle.clone(), state.is_idle())
1186 };
1187 match lifecycle {
1188 Lifecycle::Failed(reason) => return Err(reason),
1189 Lifecycle::Closed => return Ok(()),
1190 _ if idle => return Ok(()),
1191 _ => tokio::time::sleep(Duration::from_millis(2)).await,
1192 }
1193 }
1194 }
1195
1196 async fn close_and_drain(&self) -> Result<(), AuditTerminalReason> {
1197 {
1198 let mut state = self.inner.state.lock();
1199 if matches!(state.lifecycle, Lifecycle::Open) {
1200 state.lifecycle = Lifecycle::Closing;
1201 }
1202 }
1203 let result = AuditBatchControl::quiesce(self).await;
1204 {
1205 let mut state = self.inner.state.lock();
1206 if result.is_ok() && matches!(state.lifecycle, Lifecycle::Closing) {
1207 state.lifecycle = Lifecycle::Closed;
1208 }
1209 }
1210 let handle = self.supervisor.lock().take();
1211 if let Some(handle) = handle {
1212 let _ = handle.await;
1213 }
1214 result
1215 }
1216}
1217
1218#[cfg(any(test, feature = "fault-injection"))]
1219mod fault {
1220 use std::sync::atomic::AtomicBool;
1221
1222 pub(super) static CHILD_PANIC: AtomicBool = AtomicBool::new(false);
1223 pub(super) static CHILD_CANCEL: AtomicBool = AtomicBool::new(false);
1224 pub(super) static SUPERVISOR_PANIC: AtomicBool = AtomicBool::new(false);
1225 pub(super) static SUPERVISOR_SLEEP_BEFORE_SPAWN: AtomicBool = AtomicBool::new(false);
1226 pub(super) static JOIN_LOST: AtomicBool = AtomicBool::new(false);
1227 pub(super) static INCONSISTENT_EXIT: AtomicBool = AtomicBool::new(false);
1228}
1229
1230/// Deterministic fault-injection arms consumed by exactly one subsequent
1231/// generation each. Test-only surface, gated identically to the rest of this
1232/// crate's `fault-injection` fixtures (see `crate::operations`).
1233#[cfg(any(test, feature = "fault-injection"))]
1234pub mod fault_injection {
1235 use std::sync::atomic::Ordering;
1236
1237 pub fn arm_child_panic() {
1238 super::fault::CHILD_PANIC.store(true, Ordering::SeqCst);
1239 }
1240 pub fn arm_child_cancel() {
1241 super::fault::CHILD_CANCEL.store(true, Ordering::SeqCst);
1242 }
1243 pub fn arm_supervisor_panic() {
1244 super::fault::SUPERVISOR_PANIC.store(true, Ordering::SeqCst);
1245 }
1246 pub fn arm_supervisor_sleep_before_spawn() {
1247 super::fault::SUPERVISOR_SLEEP_BEFORE_SPAWN.store(true, Ordering::SeqCst);
1248 }
1249 pub fn arm_join_lost() {
1250 super::fault::JOIN_LOST.store(true, Ordering::SeqCst);
1251 }
1252 pub fn arm_inconsistent_exit() {
1253 super::fault::INCONSISTENT_EXIT.store(true, Ordering::SeqCst);
1254 }
1255}
1256
1257// ── Test-attribution surface (R2) ────────────────────────────────────────
1258//
1259// Additive only: production attribution still lands in the public
1260// `db_diagnostics` counters via `metrics_snapshot`-shaped data supplied at
1261// that seam. This surface never substitutes for it.
1262#[cfg(any(test, feature = "test-internals"))]
1263mod test_internals {
1264 use super::*;
1265
1266 /// A point-in-time view of the batch's counters and generation history,
1267 /// used by mechanism tests to compute a delta across a measured
1268 /// operation.
1269 #[derive(Debug, Clone)]
1270 pub struct AuditBatchSnapshot {
1271 pub pending_rows: usize,
1272 pub in_flight_generation: Option<u64>,
1273 pub driver_active: bool,
1274 pub next_generation_id: u64,
1275 pub submitted_rows: u64,
1276 pub committed_rows: u64,
1277 pub store_batch_calls: u64,
1278 pub per_generation: Vec<AuditGenerationSnapshot>,
1279 /// Live count of detached `DriverAppendAbandoned` appends still
1280 /// running, bounded by `AuditBatchConfig::max_abandoned_appends`.
1281 pub outstanding_abandoned_appends: usize,
1282 }
1283
1284 impl AuditBatchSnapshot {
1285 pub fn is_idle(&self) -> bool {
1286 self.pending_rows == 0 && self.in_flight_generation.is_none() && !self.driver_active
1287 }
1288 }
1289
1290 #[derive(Debug, Clone, Copy)]
1291 pub struct AuditBatchMetricsSnapshot {
1292 pub flush_failures: u64,
1293 pub degraded_rows: u64,
1294 pub degraded: bool,
1295 pub late_append_commits: u64,
1296 pub late_append_failures: u64,
1297 }
1298
1299 #[derive(Debug, Clone, PartialEq)]
1300 pub struct AuditBatchDelta {
1301 pub submitted_rows: u64,
1302 pub committed_rows: u64,
1303 pub store_batch_calls: u64,
1304 pub per_generation: Vec<AuditGenerationSnapshot>,
1305 }
1306
1307 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1308 pub enum AuditSnapshotError {
1309 CounterRegressed,
1310 GenerationHistoryRegressed,
1311 }
1312
1313 impl AuditBatch {
1314 pub fn test_snapshot(&self) -> AuditBatchSnapshot {
1315 let state = self.inner.state.lock();
1316 AuditBatchSnapshot {
1317 pending_rows: state.pending.len(),
1318 in_flight_generation: state.in_flight_generation,
1319 driver_active: state.driver_active,
1320 next_generation_id: state.next_generation_id,
1321 submitted_rows: state.submitted_rows,
1322 committed_rows: state.committed_rows,
1323 store_batch_calls: state.store_batch_calls,
1324 per_generation: state.generations.clone(),
1325 outstanding_abandoned_appends: state.outstanding_abandoned_appends,
1326 }
1327 }
1328
1329 pub fn metrics_snapshot(&self) -> AuditBatchMetricsSnapshot {
1330 let state = self.inner.state.lock();
1331 AuditBatchMetricsSnapshot {
1332 flush_failures: state.flush_failures,
1333 degraded_rows: state.degraded_rows,
1334 degraded: state.degraded,
1335 late_append_commits: state.late_append_commits,
1336 late_append_failures: state.late_append_failures,
1337 }
1338 }
1339
1340 /// Abort the currently-retained supervisor `JoinHandle`, if any,
1341 /// simulating a shutdown abort landing mid-generation. Returns
1342 /// whether a handle was found and aborted. Test-only: exercises the
1343 /// R1 supervisor-cancellation path without reaching into a private
1344 /// field from outside this module.
1345 pub fn test_abort_supervisor(&self) -> bool {
1346 let handle = self.supervisor.lock().take();
1347 match handle {
1348 Some(handle) => {
1349 handle.abort();
1350 true
1351 }
1352 None => false,
1353 }
1354 }
1355 }
1356
1357 /// Checked monotonic subtraction. Rejects a regressed counter or a
1358 /// generation history that is not an append-only extension of `before`.
1359 pub fn audit_delta(
1360 before: &AuditBatchSnapshot,
1361 after: &AuditBatchSnapshot,
1362 ) -> Result<AuditBatchDelta, AuditSnapshotError> {
1363 let submitted_rows = after
1364 .submitted_rows
1365 .checked_sub(before.submitted_rows)
1366 .ok_or(AuditSnapshotError::CounterRegressed)?;
1367 let committed_rows = after
1368 .committed_rows
1369 .checked_sub(before.committed_rows)
1370 .ok_or(AuditSnapshotError::CounterRegressed)?;
1371 let store_batch_calls = after
1372 .store_batch_calls
1373 .checked_sub(before.store_batch_calls)
1374 .ok_or(AuditSnapshotError::CounterRegressed)?;
1375 if after.per_generation.len() < before.per_generation.len() {
1376 return Err(AuditSnapshotError::GenerationHistoryRegressed);
1377 }
1378 if after.per_generation[..before.per_generation.len()] != before.per_generation[..] {
1379 return Err(AuditSnapshotError::GenerationHistoryRegressed);
1380 }
1381 let per_generation = after.per_generation[before.per_generation.len()..].to_vec();
1382 Ok(AuditBatchDelta {
1383 submitted_rows,
1384 committed_rows,
1385 store_batch_calls,
1386 per_generation,
1387 })
1388 }
1389
1390 /// Exhaustive, non-wildcard producer classification (see
1391 /// [`super::AuditProducer`] and [`super::classify`], the real
1392 /// crate-private definitions this mirrors one-for-one). The doctest
1393 /// below proves the general property those definitions rely on: a match
1394 /// over a non-`#[non_exhaustive]` enum that omits a variant, with no
1395 /// wildcard arm to silently absorb it, fails to compile rather than
1396 /// passing an incomplete classification.
1397 ///
1398 /// ```compile_fail
1399 /// enum AuditProducer {
1400 /// GateDenied,
1401 /// DispatchSucceeded,
1402 /// DispatchFailed,
1403 /// UnknownVerb,
1404 /// GitDigestReceipt,
1405 /// ConfigLocked,
1406 /// RecallExecuted,
1407 /// }
1408 ///
1409 /// fn describe(p: AuditProducer) -> &'static str {
1410 /// match p {
1411 /// AuditProducer::GateDenied => "obligation",
1412 /// AuditProducer::DispatchSucceeded => "obligation",
1413 /// AuditProducer::DispatchFailed => "obligation",
1414 /// AuditProducer::UnknownVerb => "obligation",
1415 /// AuditProducer::GitDigestReceipt => "obligation",
1416 /// AuditProducer::ConfigLocked => "observability",
1417 /// // RecallExecuted intentionally omitted: a non-exhaustive
1418 /// // match must fail to compile, proving no variant can
1419 /// // silently fall through an absent wildcard arm.
1420 /// }
1421 /// }
1422 /// ```
1423 #[allow(dead_code)]
1424 struct DoctestAnchor;
1425}
1426
1427#[cfg(any(test, feature = "test-internals"))]
1428pub use test_internals::*;
1429
1430#[cfg(test)]
1431#[path = "audit_batch_physical_close_tests.rs"]
1432mod physical_close_tests;
1433
1434#[cfg(test)]
1435mod commit_reply_tests {
1436 use super::*;
1437 use khive_storage::event::IdempotentEventBatchResult;
1438 use khive_storage::{BatchWriteSummary, EventFilter, Page, PageRequest, StorageResult};
1439 use std::sync::atomic::AtomicUsize;
1440
1441 struct LostReplyStore {
1442 inner: Arc<dyn EventStore>,
1443 calls: AtomicUsize,
1444 }
1445
1446 #[async_trait::async_trait]
1447 impl EventStore for LostReplyStore {
1448 async fn append_event(&self, event: Event) -> StorageResult<()> {
1449 self.inner.append_event(event).await
1450 }
1451 async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
1452 self.inner.append_events(events).await
1453 }
1454 async fn get_event(&self, id: uuid::Uuid) -> StorageResult<Option<Event>> {
1455 self.inner.get_event(id).await
1456 }
1457 async fn query_events(
1458 &self,
1459 filter: EventFilter,
1460 page: PageRequest,
1461 ) -> StorageResult<Page<Event>> {
1462 self.inner.query_events(filter, page).await
1463 }
1464 async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
1465 self.inner.count_events(filter).await
1466 }
1467 fn preflight_event(&self, event: &Event) -> StorageResult<()> {
1468 self.inner.preflight_event(event)
1469 }
1470 fn supports_idempotent_audit_batch(&self) -> bool {
1471 true
1472 }
1473 async fn append_events_idempotent(
1474 &self,
1475 events: Vec<Event>,
1476 ) -> StorageResult<IdempotentEventBatchResult> {
1477 let result = self.inner.append_events_idempotent(events).await?;
1478 if self.calls.fetch_add(1, Ordering::SeqCst) == 0 {
1479 return Err(StorageError::WriteOutcomeUnknown {
1480 source: Box::new(std::io::Error::other("reply lost after stored append")),
1481 });
1482 }
1483 Ok(result)
1484 }
1485 }
1486
1487 #[tokio::test]
1488 async fn lost_commit_reply_reconciles_identical_stored_audit_without_duplicate() {
1489 let backend = khive_db::StorageBackend::memory().expect("memory event store");
1490 let store = Arc::new(LostReplyStore {
1491 inner: backend.events().expect("event capability"),
1492 calls: AtomicUsize::new(0),
1493 });
1494 let event = Event::new(
1495 "local",
1496 "memory.recall",
1497 khive_types::EventKind::Audit,
1498 khive_types::SubstrateKind::Event,
1499 "test:lost-reply",
1500 );
1501 let result = run_generation(
1502 store.clone(),
1503 vec![event.clone()],
1504 Arc::new(AuditBatchConfig {
1505 max_commit_attempts: std::num::NonZeroU8::new(2).unwrap(),
1506 retry_backoff: Duration::ZERO,
1507 ..AuditBatchConfig::default()
1508 }),
1509 )
1510 .await;
1511 let GenerationResult::Committed(rows) = result else {
1512 panic!("idempotent audit replay must reconcile the lost COMMIT reply");
1513 };
1514 assert_eq!(rows, vec![EventAppendDisposition::AlreadyPresentIdentical]);
1515 assert_eq!(store.calls.load(Ordering::SeqCst), 2);
1516 assert_eq!(store.get_event(event.id).await.unwrap(), Some(event));
1517 assert_eq!(
1518 store.count_events(EventFilter::default()).await.unwrap(),
1519 1,
1520 "audit recovery must not duplicate the already stored row"
1521 );
1522 }
1523
1524 #[test]
1525 fn unavailable_owner_retries_pool_but_not_missing_runtime_or_route_violation() {
1526 let unavailable = |source| StorageError::WriterOwnerUnavailable {
1527 operation: "audit append".into(),
1528 source: Box::new(source),
1529 };
1530 assert!(matches!(
1531 classify_store_error(&unavailable(StorageError::Pool {
1532 operation: "owner".into(),
1533 message: "stopped".into(),
1534 })),
1535 RetryDecision::Retry
1536 ));
1537 for error in [
1538 unavailable(StorageError::WriterTaskNoRuntime),
1539 StorageError::WriterRouteViolation {
1540 operation: "audit append".into(),
1541 },
1542 ] {
1543 assert!(
1544 matches!(
1545 classify_store_error(&error),
1546 RetryDecision::Terminal(AuditTerminalReason::StoreFailure)
1547 ),
1548 "configuration/routing refusal is not an idempotent replay candidate"
1549 );
1550 }
1551 }
1552}