khive_runtime/audit_batch.rs
1//! ADR-133 Slice 1: the audit-batch seam.
2//!
3//! Incidental audit writes (gate denials, dispatch outcomes, config-lock
4//! rows, git.digest receipts, and pure-observability rows like
5//! `RecallExecuted`) no longer take one writer-task acquisition per row on
6//! the request hot path. Concurrent submissions arriving while a generation
7//! is committing share the *next* generation instead of each taking their
8//! own writer acquisition, so N concurrent producers collapse to one
9//! [`khive_storage::EventStore::append_events_idempotent`] call per
10//! generation rather than N.
11//!
12//! [`AuditBatch`] owns this seam. A lazily-spawned supervisor task drains
13//! pending rows into generations and drives each through the store; the
14//! supervisor's own `JoinHandle` is retained (never discarded) so an
15//! abnormal exit — panic, cancellation, a lost child join, or a driver that
16//! returns `Ok` while state is not terminally consistent — is observed and
17//! converted into a `Failed` transition with all accepted waiters resolved,
18//! per owner ruling R1/R4 (`.khive/OWNER_RULING_adr133_gate.md`).
19
20#[cfg(any(test, feature = "fault-injection"))]
21use std::sync::atomic::Ordering;
22use std::sync::Arc;
23use std::time::Duration;
24
25use parking_lot::Mutex;
26use tokio::sync::oneshot;
27use tokio::task::JoinHandle;
28
29use khive_storage::event::EventAppendDisposition;
30use khive_storage::{Event, EventStore, StorageError, WriterTaskRequestState};
31
32/// Coarse durability classification for an [`AuditProducer`]. See
33/// [`classify`] for the exhaustive mapping.
34#[derive(Debug, Clone, Copy, PartialEq, Eq)]
35pub(crate) enum AuditProductionClass {
36 /// The dispatch this row audits carries an obligation: the caller-visible
37 /// outcome must not silently diverge from what was durably recorded.
38 DispatchObligation,
39 /// The row is a best-effort observability signal; a non-commit degrades
40 /// gracefully rather than blocking or failing the dispatch it audits.
41 PureObservability,
42}
43
44/// Every call site that can submit a row through [`AuditBatchControl`].
45/// Adding a variant here without extending the crate-private `classify`
46/// function's match is a compile error — there is no wildcard arm.
47#[derive(Debug, Clone, Copy, PartialEq, Eq)]
48pub enum AuditProducer {
49 /// The gate denied a dispatch; the denial itself is audited.
50 GateDenied,
51 /// The gate backend was unreachable and the dispatch failed closed; the
52 /// refusal is audited best-effort: the typed `GateUnavailable` refusal
53 /// already fails the dispatch, so a lost row degrades diagnostics only.
54 GateUnavailable,
55 /// A pack dispatch returned a successful result.
56 DispatchSucceeded,
57 /// A pack dispatch returned an error result.
58 DispatchFailed,
59 /// A redirected KG read consulted the gate on its effective entity id.
60 EffectiveTargetCheck,
61 /// The gate allowed a verb no pack owns.
62 UnknownVerb,
63 /// The strict `git.digest` success receipt (schema v2).
64 GitDigestReceipt,
65 /// A drained process-lifetime `OnceLock` config-lock row.
66 ConfigLocked,
67 /// A `memory.recall` execution's pure-observability audit row.
68 /// Submitted by the recall handler through its owning registry's batch.
69 RecallExecuted,
70}
71
72/// Classify an [`AuditProducer`] into its [`AuditProductionClass`]. One
73/// exhaustive match, no wildcard arm.
74pub(crate) const fn classify(producer: AuditProducer) -> AuditProductionClass {
75 match producer {
76 AuditProducer::GateDenied
77 | AuditProducer::DispatchSucceeded
78 | AuditProducer::DispatchFailed
79 | AuditProducer::EffectiveTargetCheck
80 | AuditProducer::UnknownVerb
81 | AuditProducer::GitDigestReceipt => AuditProductionClass::DispatchObligation,
82 AuditProducer::ConfigLocked
83 | AuditProducer::RecallExecuted
84 | AuditProducer::GateUnavailable => AuditProductionClass::PureObservability,
85 }
86}
87
88/// Exhaustive terminal reasons an [`AuditBatchControl::submit`],
89/// [`AuditBatchControl::quiesce`], or [`AuditBatchControl::close_and_drain`]
90/// call can resolve to. Never mapped through a wildcard arm anywhere in this
91/// module (R4).
92#[derive(Debug, Clone, Copy, PartialEq, Eq)]
93pub enum AuditTerminalReason {
94 /// `EventStore::preflight_event` rejected the row before it was ever
95 /// enqueued. The row was never counted as submitted.
96 PreflightRejected,
97 /// The batch is `Closing` or `Closed`; new admission is refused.
98 AdmissionClosed,
99 /// `AuditBatchConfig::max_pending_rows` was reached before this row was
100 /// enqueued. The row was never counted as submitted and never shared a
101 /// generation with anyone — safe to retry, and doing so applies the
102 /// obligation at most once.
103 QueueAdmissionExhausted,
104 /// The row was already enqueued (counted in `submitted_rows`) when this
105 /// caller's `AuditBatchConfig::admission_deadline` elapsed waiting for
106 /// its generation's outcome. Unlike [`Self::QueueAdmissionExhausted`],
107 /// the row was not refused: by the moment the deadline fires it may
108 /// still be sitting in `state.pending`, or the driver may have already
109 /// drained it into an in-flight generation — either way it remains
110 /// enqueued and unresolved, and is committed (or terminally failed) by
111 /// the generation driver independently of this caller's timeout, so the
112 /// caller cannot tell from this reason alone whether the row eventually
113 /// committed, or even which of those two states it was in when the
114 /// deadline elapsed. Retrying is only safe for an idempotent caller —
115 /// the prior submission may still land. When this reason degrades an
116 /// admission-degrade-safe read's own audit obligation, it is counted
117 /// separately from [`Self::QueueAdmissionExhausted`] — see
118 /// `pack::audit_admission_unresolved_obligation_count` — precisely
119 /// because a row counted here may still commit, unlike one refused
120 /// before enqueue.
121 AdmissionDeadlineExpired,
122 /// Reached only through `AuditBatch::submit_until_resolved`: the row's
123 /// [`Self::AdmissionDeadlineExpired`] wait had already elapsed, and this
124 /// caller's own `AuditBatchConfig::resolution_deadline` then also
125 /// elapsed still waiting for the row's real generation outcome. The
126 /// caller reaches this only after its domain effect has already
127 /// committed, so the effect is never retried and the row is never
128 /// re-enqueued — but unlike an ordinary commit, the caller now learns
129 /// the effect committed while the audit outcome itself is unresolved.
130 /// Same as [`Self::AdmissionDeadlineExpired`], the row is left exactly
131 /// where the driver holds it (`state.pending`, or already mid-generation)
132 /// for the driver to resolve independently — this reason performs no
133 /// removal. Kept distinct from [`Self::AdmissionDeadlineExpired`] so a
134 /// caller, and diagnostics reading this reason, can tell a merely-slow
135 /// admission wait apart from a resolution wait that gave up entirely.
136 ResolutionDeadlineExpired,
137 /// A row shared this generation's id with a previously stored row whose
138 /// columns or observation projection did not match exactly.
139 IdentityConflict,
140 /// `classify_store_error` judged the store's error non-retryable, and the
141 /// generation stopped on that attempt. Usually that is the first attempt,
142 /// but not necessarily: a generation whose earlier attempts failed
143 /// retryably and whose next one returns a non-retryable error reports
144 /// this reason too, because the attempt that decided the outcome is the
145 /// non-retryable one. Distinct from [`Self::RetryExhausted`], which the
146 /// classifier judged safe to retry on every attempt and which failed
147 /// anyway once they ran out — this reason carries no such hope. Whatever
148 /// the store returned on the deciding attempt, it is not the kind of
149 /// failure `AuditBatchConfig::max_commit_attempts` exists to ride out,
150 /// so a caller or an automated retry policy reading this reason should
151 /// not schedule a bare retry of the same call and should instead treat
152 /// it as a storage fault needing attention.
153 StoreFailure,
154 /// The store returned a `classify_store_error`-retryable error on every
155 /// one of `AuditBatchConfig::max_commit_attempts` attempts for this
156 /// generation, and the last attempt still failed retryable. Kept
157 /// distinct from [`Self::StoreFailure`] — the same way
158 /// [`Self::ResolutionDeadlineExpired`] is kept distinct from
159 /// [`Self::AdmissionDeadlineExpired`] — so a caller, and diagnostics
160 /// reading this reason, can tell a store call the classifier judged
161 /// hopeless apart from one that kept failing a condition (write-queue or
162 /// writer-task pressure, pool or timeout) the classifier judged
163 /// transient. The underlying condition may still be
164 /// transient at the moment attempts run out (a daemon restart, pool
165 /// pressure outlasting the configured backoff), so an operator or an
166 /// automated retry policy sitting above this batch can choose to wait
167 /// longer and try again rather than treating it identically to
168 /// [`Self::StoreFailure`]. This reason changes no tolerance, deadline,
169 /// retry count, or backoff on its own — it only names which of the two
170 /// causes produced the generation's failure.
171 RetryExhausted,
172 /// The configured `EventStore` backend does not implement
173 /// `append_events_idempotent`.
174 IdempotencyUnsupported,
175 /// The generation driver task panicked, or the supervisor awaiting it
176 /// unwound while armed.
177 DriverPanicked,
178 /// The generation driver task was cancelled/aborted, or the supervisor
179 /// awaiting it was dropped mid-await (shutdown abort) while armed.
180 DriverCancelled,
181 /// The child driver's `JoinHandle` was lost — dropped without ever being
182 /// inspected — so its outcome could not be classified.
183 DriverJoinLost,
184 /// The driver returned `Ok` but locked batch state was not proved
185 /// terminally consistent afterward (`in_flight` still set, or the
186 /// generation's rows were never resolved).
187 DriverExitedInconsistent,
188 /// The driver's own bound on a single generation's
189 /// `EventStore::append_events_idempotent()` call
190 /// (`supervisor_loop`'s `driver_append_deadline`) elapsed while that
191 /// call was still in flight. Unlike [`Self::AdmissionDeadlineExpired`]
192 /// and [`Self::ResolutionDeadlineExpired`], which bound only how long a
193 /// *caller* keeps waiting while the row stays wherever the driver holds
194 /// it, this bounds the driver's own hold: every waiter on the abandoned
195 /// generation is resolved with this reason and the generation is
196 /// removed from the driver's in-flight state, so a stalled store call
197 /// cannot pin `pending` at `max_pending_rows` and starve all later
198 /// admission (khive#2331). The underlying append is not cancelled — it
199 /// may not be safely cancellable mid-flight — so it is left to run to
200 /// completion in the background and its eventual result, whatever it
201 /// is, is discarded; no waiter is still listening for it. The caller's
202 /// domain effect (if any already committed before this row was
203 /// enqueued) is never retried, and the row is never re-enqueued.
204 DriverAppendAbandoned,
205 /// Before spawning a generation's child, the driver found
206 /// `AuditBatchConfig::max_abandoned_appends` detached appends already
207 /// outstanding from prior [`Self::DriverAppendAbandoned`] generations
208 /// (khive#2331). The store is treated as wedged: this generation's rows
209 /// are shed without ever attempting an append — no child task is
210 /// spawned, no store call is made, and every waiter resolves with this
211 /// reason immediately, well inside `driver_append_deadline`. Same
212 /// non-retry contract as [`Self::DriverAppendAbandoned`]: any
213 /// already-committed domain effect is not retried and the row is never
214 /// re-enqueued, and pure-observability producers record degradation.
215 /// The driver reattempts an append on the next generation as soon as an
216 /// outstanding append returns (commit or failure) and the count drops
217 /// back below the cap — recovery needs no timer of its own.
218 StoreWedged,
219}
220
221/// One row accepted for batching: the immutable event identity plus the
222/// producer that minted it, used for classification and, on failure,
223/// degradation accounting.
224pub struct PreparedAuditRow {
225 pub event: Event,
226 pub producer: AuditProducer,
227}
228
229/// What [`AuditBatchControl::submit`] resolves to on a non-error outcome.
230#[derive(Debug, Clone, Copy, PartialEq, Eq)]
231pub enum AuditCommitOutcome {
232 /// The row was freshly inserted.
233 Committed,
234 /// A prior row with the same identity already matched exactly (internal
235 /// retry replaying the same producer-minted identity).
236 AlreadyPresentIdentical,
237}
238
239/// Production-visible snapshot of [`AuditBatch::health_metrics`]. See there
240/// for field semantics. `khive_db::diagnostics::RuntimeAuditBatchMetrics` carries
241/// these three fields plus `admission_refused_obligations`,
242/// `retry_exhausted_obligations`, and `admission_unresolved_obligations`,
243/// which are sourced from process-wide
244/// counters outside `AuditBatch` rather than from this struct — see
245/// `VerbRegistry::audit_batch_metrics`.
246#[derive(Debug, Clone, Copy, PartialEq, Eq)]
247pub struct AuditBatchHealthMetrics {
248 pub flush_failures: u64,
249 pub degraded_rows: u64,
250 pub degraded: bool,
251 /// Detached [`AuditTerminalReason::DriverAppendAbandoned`] appends that
252 /// eventually returned a commit, after their generation had already
253 /// resolved every waiter with `DriverAppendAbandoned`. Lets an operator
254 /// see that a store recorded as wedged later drained.
255 pub late_append_commits: u64,
256 /// Detached [`AuditTerminalReason::DriverAppendAbandoned`] appends that
257 /// eventually returned a failure (store error, panic, or cancellation),
258 /// after their generation had already resolved every waiter with
259 /// `DriverAppendAbandoned`.
260 pub late_append_failures: u64,
261}
262
263/// Tunables for the batch seam. Defaults are conservative; every field is
264/// exercised by at least one mechanism test.
265#[derive(Debug, Clone)]
266pub struct AuditBatchConfig {
267 pub max_pending_rows: std::num::NonZeroUsize,
268 pub max_rows_per_generation: std::num::NonZeroUsize,
269 pub max_commit_attempts: std::num::NonZeroU8,
270 pub retry_backoff: Duration,
271 pub admission_deadline: Duration,
272 /// Caps how much longer `AuditBatch::submit_until_resolved` keeps
273 /// waiting on a row's real generation outcome once `admission_deadline`
274 /// has already elapsed on it. Without this bound a generation stuck on a
275 /// stalled `EventStore::append_events_idempotent` call retains the
276 /// completed write's caller, its request slot, and its audit-lane waiter
277 /// forever, exhausting both request and audit capacity (khive#2331).
278 /// Must be at least `admission_deadline` — validated (debug-only) in
279 /// [`AuditBatch::new`]. Defaults to 6x `admission_deadline`.
280 ///
281 /// Also the single source value the driver's own per-generation append
282 /// bound (`supervisor_loop`'s `driver_append_deadline`) is derived from
283 /// — that bound exists so a stalled append cannot keep the driver from
284 /// ever draining `pending` again, which is what actually exhausts
285 /// admission for every later caller, not just the one already waiting
286 /// on the stuck row. See [`AuditTerminalReason::DriverAppendAbandoned`].
287 pub resolution_deadline: Duration,
288 /// Caps how many [`AuditTerminalReason::DriverAppendAbandoned`] appends
289 /// may be outstanding (handed to a detached task, still running) at
290 /// once. Each outstanding append retains up to
291 /// `max_rows_per_generation` events until it finally returns, so the
292 /// retained-buffer bound this places on a wedged store is
293 /// `max_abandoned_appends * max_rows_per_generation` rows — without it,
294 /// a store whose append never returns mints one such task per
295 /// `driver_append_deadline` with no cap, and both the live task count
296 /// and the retained event batches inside them grow until the process
297 /// exits (khive#2331). Once the cap is reached, `supervisor_loop` sheds
298 /// further generations with [`AuditTerminalReason::StoreWedged`] instead
299 /// of attempting another append; it resumes attempting appends as soon
300 /// as an outstanding one returns and the count drops back below the
301 /// cap. Defaults to 4.
302 pub max_abandoned_appends: std::num::NonZeroUsize,
303}
304
305impl Default for AuditBatchConfig {
306 fn default() -> Self {
307 let admission_deadline = Duration::from_secs(5);
308 Self {
309 max_pending_rows: std::num::NonZeroUsize::new(4096).unwrap(),
310 max_rows_per_generation: std::num::NonZeroUsize::new(256).unwrap(),
311 max_commit_attempts: std::num::NonZeroU8::new(3).unwrap(),
312 retry_backoff: Duration::from_millis(20),
313 admission_deadline,
314 resolution_deadline: admission_deadline * 6,
315 max_abandoned_appends: std::num::NonZeroUsize::new(4).unwrap(),
316 }
317 }
318}
319
320#[async_trait::async_trait]
321pub trait AuditBatchControl: Send + Sync {
322 async fn submit(
323 &self,
324 row: PreparedAuditRow,
325 ) -> Result<AuditCommitOutcome, AuditTerminalReason>;
326 async fn quiesce(&self) -> Result<(), AuditTerminalReason>;
327 async fn close_and_drain(&self) -> Result<(), AuditTerminalReason>;
328}
329
330#[derive(Debug, Clone, PartialEq, Eq)]
331enum Lifecycle {
332 Open,
333 Closing,
334 Closed,
335 Failed(AuditTerminalReason),
336}
337
338struct Waiting {
339 event: Event,
340 producer: AuditProducer,
341 responder: oneshot::Sender<Result<AuditCommitOutcome, AuditTerminalReason>>,
342}
343
344/// A committed/failed generation's accounting. Detailed history is retained when
345/// compiled for tests or with `test-internals`; ordinary builds keep cumulative counters.
346#[derive(Debug, Clone, PartialEq, Eq)]
347pub struct AuditGenerationSnapshot {
348 pub generation_id: u64,
349 pub submitted_rows: u64,
350 pub committed_rows: u64,
351 pub store_batch_calls: u64,
352 pub terminal_reason: Option<AuditTerminalReason>,
353}
354
355struct State {
356 lifecycle: Lifecycle,
357 pending: Vec<Waiting>,
358 in_flight_generation: Option<u64>,
359 driver_active: bool,
360 physical_reservations: usize,
361 next_generation_id: u64,
362 submitted_rows: u64,
363 committed_rows: u64,
364 store_batch_calls: u64,
365 #[cfg(any(test, feature = "test-internals"))]
366 generations: Vec<AuditGenerationSnapshot>,
367 flush_failures: u64,
368 degraded_rows: u64,
369 degraded: bool,
370 /// Detached `DriverAppendAbandoned` appends currently still running,
371 /// bounded by `AuditBatchConfig::max_abandoned_appends`. Incremented
372 /// when `supervisor_loop` abandons a generation and hands its child to a
373 /// detached task; decremented by that task once the child finally
374 /// returns.
375 outstanding_abandoned_appends: usize,
376 late_append_commits: u64,
377 late_append_failures: u64,
378}
379
380impl State {
381 fn new() -> Self {
382 Self {
383 lifecycle: Lifecycle::Open,
384 pending: Vec::new(),
385 in_flight_generation: None,
386 driver_active: false,
387 physical_reservations: 0,
388 next_generation_id: 0,
389 submitted_rows: 0,
390 committed_rows: 0,
391 store_batch_calls: 0,
392 #[cfg(any(test, feature = "test-internals"))]
393 generations: Vec::new(),
394 flush_failures: 0,
395 degraded_rows: 0,
396 degraded: false,
397 outstanding_abandoned_appends: 0,
398 late_append_commits: 0,
399 late_append_failures: 0,
400 }
401 }
402
403 /// No-op outside `test`/`test-internals`; operators read [`AuditBatch::health_metrics`].
404 fn record_generation(&mut self, generation: AuditGenerationSnapshot) {
405 #[cfg(any(test, feature = "test-internals"))]
406 self.generations.push(generation);
407 #[cfg(not(any(test, feature = "test-internals")))]
408 let _ = generation;
409 }
410
411 fn is_idle(&self) -> bool {
412 self.pending.is_empty() && self.in_flight_generation.is_none() && !self.driver_active
413 }
414}
415
416struct Inner {
417 state: Mutex<State>,
418 // Completion witnesses only: physical shutdown never aborts admitted work.
419 physical_tasks: Mutex<Vec<tokio::task::AbortHandle>>,
420}
421
422struct PhysicalReservation(Arc<Inner>);
423
424impl Drop for PhysicalReservation {
425 fn drop(&mut self) {
426 self.0.state.lock().physical_reservations -= 1;
427 }
428}
429
430impl Inner {
431 fn reserve_physical(self: &Arc<Self>) -> PhysicalReservation {
432 self.state.lock().physical_reservations += 1;
433 PhysicalReservation(Arc::clone(self))
434 }
435
436 fn spawn_physical<F, T>(
437 self: &Arc<Self>,
438 registration: PhysicalReservation,
439 future: F,
440 ) -> JoinHandle<T>
441 where
442 F: std::future::Future<Output = T> + Send + 'static,
443 T: Send + 'static,
444 {
445 let running = self.reserve_physical();
446 let handle = tokio::spawn(async move {
447 let result = future.await;
448 drop(running);
449 result
450 });
451 {
452 let mut tasks = self.physical_tasks.lock();
453 tasks.retain(|task| !task.is_finished());
454 tasks.push(handle.abort_handle());
455 }
456 // A very short task can finish before its witness is published. Keep
457 // registration reserved until publication to close that zero-work gap.
458 drop(registration);
459 handle
460 }
461
462 fn physically_finished(&self) -> bool {
463 self.state.lock().physical_reservations == 0
464 && self
465 .physical_tasks
466 .lock()
467 .iter()
468 .all(|task| task.is_finished())
469 }
470
471 /// Wins once: the first abnormal driver/supervisor exit sets `Failed`
472 /// and drains every accepted waiter (this generation's, plus anything
473 /// still queued for a future one) with the same typed reason. A later
474 /// abnormal path observes the existing terminal state and does not
475 /// double-count (R1).
476 fn fail_driver(&self, reason: AuditTerminalReason, in_flight_waiters: Vec<Waiting>) {
477 let mut state = self.state.lock();
478 let already_failed = matches!(state.lifecycle, Lifecycle::Failed(_));
479 if !already_failed {
480 state.lifecycle = Lifecycle::Failed(reason);
481 }
482 state.driver_active = false;
483 state.in_flight_generation = None;
484 let drained: Vec<Waiting> = std::mem::take(&mut state.pending);
485 if !already_failed {
486 state.flush_failures += 1;
487 state.degraded = true;
488 let generation_id = state.next_generation_id;
489 state.record_generation(AuditGenerationSnapshot {
490 generation_id,
491 submitted_rows: (in_flight_waiters.len() + drained.len()) as u64,
492 committed_rows: 0,
493 store_batch_calls: 0,
494 terminal_reason: Some(reason),
495 });
496 state.next_generation_id += 1;
497 }
498 drop(state);
499 for waiting in in_flight_waiters.into_iter().chain(drained) {
500 record_degradation_if_pure(self, waiting.producer);
501 let _ = waiting.responder.send(Err(reason));
502 }
503 }
504
505 fn record_degradation(&self) {
506 let mut state = self.state.lock();
507 state.degraded_rows += 1;
508 state.degraded = true;
509 }
510}
511
512fn record_degradation_if_pure(inner: &Inner, producer: AuditProducer) {
513 if classify(producer) == AuditProductionClass::PureObservability {
514 inner.record_degradation();
515 }
516}
517
518/// Armed for the lifetime of one generation's supervision. If dropped while
519/// still armed — the supervisor's own frame unwinding (panic) or being
520/// dropped mid-await (cancellation/shutdown abort) — the guard fails the
521/// generation before returning control to whatever tore it down, so the
522/// caller never observes a background-task-count restoration that raced
523/// ahead of the failure broadcast (R1).
524struct SupervisorGuard<'a> {
525 inner: &'a Arc<Inner>,
526 waiting: Option<Vec<Waiting>>,
527}
528
529impl<'a> SupervisorGuard<'a> {
530 fn armed(inner: &'a Arc<Inner>, waiting: Vec<Waiting>) -> Self {
531 Self {
532 inner,
533 waiting: Some(waiting),
534 }
535 }
536
537 /// Clean disarm: the caller has fully classified the outcome and taken
538 /// ownership of the waiters to resolve them itself.
539 fn disarm(mut self) -> Vec<Waiting> {
540 self.waiting.take().unwrap_or_default()
541 }
542
543 /// Explicit failure classification reached without unwinding/cancelling
544 /// this frame (e.g. a `JoinError` was returned normally). Equivalent to
545 /// what `Drop` does when armed, but callable inline.
546 fn fail(mut self, reason: AuditTerminalReason) {
547 if let Some(waiting) = self.waiting.take() {
548 self.inner.fail_driver(reason, waiting);
549 }
550 }
551}
552
553impl Drop for SupervisorGuard<'_> {
554 fn drop(&mut self) {
555 let Some(waiting) = self.waiting.take() else {
556 return;
557 };
558 let reason = if std::thread::panicking() {
559 AuditTerminalReason::DriverPanicked
560 } else {
561 AuditTerminalReason::DriverCancelled
562 };
563 self.inner.fail_driver(reason, waiting);
564 }
565}
566
567enum RetryDecision {
568 Retry,
569 Terminal(AuditTerminalReason),
570}
571
572fn classify_store_error(err: &StorageError) -> RetryDecision {
573 match err.without_sqlite_write_stage() {
574 StorageError::WriteQueueFull { .. } | StorageError::WriterTaskBusy { .. } => {
575 RetryDecision::Retry
576 }
577 StorageError::CapacityUnavailable {
578 phase: khive_storage::CapacityUnavailablePhase::LockContended,
579 ..
580 } => RetryDecision::Retry,
581 // Transient availability conditions, not judgments on the batch. The
582 // events-daemon forwarding lane (ADR-170) reports an unreachable or
583 // stalled daemon as `Pool`/`Timeout`; the direct SQL path reports
584 // acquisition pressure the same way. Both are exactly what the
585 // configured bounded retries exist for — treating them as terminal
586 // would abandon a generation on the first blip of a daemon restart.
587 StorageError::Pool { .. } | StorageError::Timeout { .. } => RetryDecision::Retry,
588 // The writer task wraps any request operation that failed and was rolled
589 // back, whatever the cause, so the wrapper alone does not say whether a
590 // retry can succeed: a missing column fails the same way on every
591 // attempt. Judge the cause it carries. A request left in an unknown
592 // state is replayed regardless, because the append is idempotent.
593 StorageError::WriterTaskRequestFailed {
594 request_state,
595 source,
596 } => match request_state {
597 WriterTaskRequestState::NotStarted | WriterTaskRequestState::TransactionRolledBack => {
598 if is_sqlite_busy_or_locked(source) {
599 RetryDecision::Retry
600 } else {
601 classify_store_error(source)
602 }
603 }
604 WriterTaskRequestState::SideEffectsUnknown => RetryDecision::Retry,
605 },
606 StorageError::WriterTaskTerminated { request_state, .. } => match request_state {
607 WriterTaskRequestState::NotStarted | WriterTaskRequestState::TransactionRolledBack => {
608 RetryDecision::Retry
609 }
610 WriterTaskRequestState::SideEffectsUnknown => RetryDecision::Retry,
611 },
612 StorageError::Unsupported { operation, .. }
613 if operation.as_ref() == "append_events_idempotent" =>
614 {
615 RetryDecision::Terminal(AuditTerminalReason::IdempotencyUnsupported)
616 }
617 _ => RetryDecision::Terminal(AuditTerminalReason::StoreFailure),
618 }
619}
620
621/// Whether a failed writer request's preserved cause is SQLite contention. The
622/// writer keeps the request body's driver error as the cause, and a driver
623/// error is not one of the typed transient variants `classify_store_error`
624/// retries on its own.
625fn is_sqlite_busy_or_locked(err: &StorageError) -> bool {
626 let StorageError::Driver { source, .. } = err.without_sqlite_write_stage() else {
627 return false;
628 };
629 matches!(
630 source
631 .downcast_ref::<rusqlite::Error>()
632 .and_then(|error| error.sqlite_error_code()),
633 Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked)
634 )
635}
636
637enum GenerationResult {
638 Committed(Vec<EventAppendDisposition>),
639 Failed(AuditTerminalReason),
640 /// Fault-injection only: proves the supervisor's post-`Ok` consistency
641 /// check actually runs.
642 #[cfg_attr(not(any(test, feature = "fault-injection")), allow(dead_code))]
643 FakedInconsistent,
644}
645
646/// The driver's own bound on how long `supervisor_loop` waits for one
647/// generation's `run_generation` child task before abandoning it
648/// (khive#2331; see [`AuditTerminalReason::DriverAppendAbandoned`]).
649///
650/// Derived from `resolution_deadline` rather than a second config field, at
651/// 3x it. That margin is load-bearing, not arbitrary: `AuditBatch::new`
652/// already enforces `resolution_deadline >= admission_deadline`, so the
653/// worst-case *caller*-side wait for a `submit_until_resolved` row —
654/// `admission_deadline` then `resolution_deadline` in sequence — is always
655/// `<= 2 * resolution_deadline`. Bounding the driver at `3 *
656/// resolution_deadline` guarantees it can never resolve a still-waiting
657/// caller's row with `DriverAppendAbandoned` ahead of that caller's own,
658/// more specific `ResolutionDeadlineExpired` reason.
659///
660/// This bound alone only stops the driver from holding one stalled
661/// generation forever — a store whose append never returns still mints one
662/// detached append per `driver_append_deadline` with nothing capping how
663/// many run at once. `AuditBatchConfig::max_abandoned_appends` closes that:
664/// once that many detached appends are outstanding, further generations are
665/// shed with [`AuditTerminalReason::StoreWedged`] instead of attempting
666/// another append. Combined, the two bounds guarantee at most
667/// `max_abandoned_appends` live detached appends at any time, at most that
668/// many retained event batches, and a shed generation costs no store work
669/// at all — it resolves inside this function's caller without ever calling
670/// `run_generation`.
671fn driver_append_deadline(config: &AuditBatchConfig) -> Duration {
672 config.resolution_deadline.saturating_mul(3)
673}
674
675async fn run_generation(
676 store: Arc<dyn EventStore>,
677 events: Vec<Event>,
678 config: Arc<AuditBatchConfig>,
679) -> GenerationResult {
680 #[cfg(any(test, feature = "fault-injection"))]
681 if fault::CHILD_PANIC.swap(false, Ordering::SeqCst) {
682 panic!("adr133 fault injection: audit_batch child_panic");
683 }
684 #[cfg(any(test, feature = "fault-injection"))]
685 if fault::INCONSISTENT_EXIT.swap(false, Ordering::SeqCst) {
686 return GenerationResult::FakedInconsistent;
687 }
688
689 let mut attempt: u8 = 0;
690 loop {
691 attempt += 1;
692 match store.append_events_idempotent(events.clone()).await {
693 Ok(result) => return GenerationResult::Committed(result.rows),
694 Err(err) => {
695 let reason = match classify_store_error(&err) {
696 RetryDecision::Retry if attempt < config.max_commit_attempts.get() => {
697 tokio::time::sleep(config.retry_backoff).await;
698 continue;
699 }
700 RetryDecision::Retry => AuditTerminalReason::RetryExhausted,
701 RetryDecision::Terminal(reason) => reason,
702 };
703 tracing::warn!(
704 error = %err,
705 attempts = attempt,
706 ?reason,
707 "audit generation failed; its rows were not committed"
708 );
709 return GenerationResult::Failed(reason);
710 }
711 }
712 }
713}
714
715/// The batch owner. Constructed once per configured `EventStore`; every
716/// dispatch-audit call site routes its row through [`AuditBatch::submit`]
717/// instead of taking its own writer-task acquisition.
718pub struct AuditBatch {
719 inner: Arc<Inner>,
720 store: Arc<dyn EventStore>,
721 config: Arc<AuditBatchConfig>,
722 supervisor: Mutex<Option<JoinHandle<()>>>,
723}
724
725impl AuditBatch {
726 pub fn new(store: Arc<dyn EventStore>, config: AuditBatchConfig) -> Arc<Self> {
727 debug_assert!(
728 config.resolution_deadline >= config.admission_deadline,
729 "resolution_deadline ({:?}) must be at least admission_deadline ({:?})",
730 config.resolution_deadline,
731 config.admission_deadline
732 );
733 Arc::new(Self {
734 inner: Arc::new(Inner {
735 state: Mutex::new(State::new()),
736 physical_tasks: Mutex::new(Vec::new()),
737 }),
738 store,
739 config: Arc::new(config),
740 supervisor: Mutex::new(None),
741 })
742 }
743
744 /// Process-lifetime audit-batch health counters, for the registry/
745 /// runtime owner to feed into `db_diagnostics` (D8's operator surface).
746 /// Unlike `test_internals::AuditBatchSnapshot::metrics_snapshot`,
747 /// which is test-only (the module is cfg-gated and invisible to the
748 /// default doc build, so an intra-doc link cannot resolve), this is
749 /// always available.
750 pub fn health_metrics(&self) -> AuditBatchHealthMetrics {
751 let state = self.inner.state.lock();
752 AuditBatchHealthMetrics {
753 flush_failures: state.flush_failures,
754 degraded_rows: state.degraded_rows,
755 degraded: state.degraded,
756 late_append_commits: state.late_append_commits,
757 late_append_failures: state.late_append_failures,
758 }
759 }
760
761 fn spawn_supervisor_if_idle(&self, registration: PhysicalReservation) {
762 let inner = self.inner.clone();
763 let store = self.store.clone();
764 let config = self.config.clone();
765 let handle = self.inner.spawn_physical(registration, async move {
766 supervisor_loop(inner, store, config).await;
767 });
768 *self.supervisor.lock() = Some(handle);
769 }
770
771 /// Stop admission and wait until all accepted append tasks and their
772 /// supervisors/reapers have physically finished. Unlike `close_and_drain`,
773 /// a driver-abandoned append still holds this barrier until it returns.
774 ///
775 /// Task completion witnesses stay on the batch, so cancelling this wait
776 /// neither detaches a taken join nor loses the barrier for a later call.
777 /// No admitted write is aborted. This may wait indefinitely for a store
778 /// that never returns; callers may cancel their wait without cancelling it.
779 /// The batch's own store Arc remains owned until the batch is dropped.
780 pub(crate) async fn close_and_wait_for_physical_completion(
781 &self,
782 ) -> Result<(), AuditTerminalReason> {
783 {
784 let mut state = self.inner.state.lock();
785 if matches!(state.lifecycle, Lifecycle::Open) {
786 state.lifecycle = Lifecycle::Closing;
787 }
788 }
789 let result = AuditBatchControl::quiesce(self).await;
790 while !self.inner.physically_finished() {
791 tokio::time::sleep(Duration::from_millis(2)).await;
792 }
793 {
794 let mut state = self.inner.state.lock();
795 if result.is_ok() && matches!(state.lifecycle, Lifecycle::Closing) {
796 state.lifecycle = Lifecycle::Closed;
797 }
798 }
799 result
800 }
801
802 /// Enqueue one row and keep waiting for its real generation outcome past
803 /// the ordinary admission wait deadline, up to `resolution_deadline`.
804 ///
805 /// This is the narrow khive#2256 seam for a successful operation whose
806 /// domain effect has already committed. Returning
807 /// [`AuditTerminalReason::AdmissionDeadlineExpired`] there would report a
808 /// false operation failure and invite an unsafe retry while the same
809 /// audit row remains enqueued. Pre-enqueue refusal and genuine terminal
810 /// generation failures still return normally. The post-admission wait is
811 /// itself bounded by `resolution_deadline`
812 /// ([`AuditTerminalReason::ResolutionDeadlineExpired`]) so a stalled
813 /// store cannot retain this caller, its request slot, and its audit-lane
814 /// waiter forever (khive#2331).
815 ///
816 /// The two deadlines therefore mean different things to the caller, and
817 /// callers key on the difference. Admission expiry means the row is
818 /// enqueued and this seam keeps waiting; the generation commits it
819 /// independently, so a dispatch that reaches it reports its committed
820 /// result and counts the row as unresolved. Resolution expiry means the
821 /// caller waited for the real outcome and never received one: the commit
822 /// is unconfirmed, not proven absent. That is returned as a terminal
823 /// reason and the dispatch propagates it as a structured
824 /// committed-outcome error carrying the domain result, with no retryable
825 /// context, for every verb rather than only for the receipt verb. Never
826 /// replay the handler or re-enqueue the row on it; the original driver
827 /// work continues on its own.
828 pub(crate) async fn submit_until_resolved(
829 &self,
830 row: PreparedAuditRow,
831 ) -> Result<AuditCommitOutcome, AuditTerminalReason> {
832 self.submit_with_wait_policy(row, true).await
833 }
834
835 async fn submit_with_wait_policy(
836 &self,
837 row: PreparedAuditRow,
838 wait_until_resolved: bool,
839 ) -> Result<AuditCommitOutcome, AuditTerminalReason> {
840 // Pre-enqueue validation (invariant 3): a malformed row is rejected
841 // before it can share a generation with anyone else's.
842 if self.store.preflight_event(&row.event).is_err() {
843 return Err(AuditTerminalReason::PreflightRejected);
844 }
845
846 let producer = row.producer;
847 let (tx, mut rx) = oneshot::channel();
848 let supervisor_registration = {
849 let mut state = self.inner.state.lock();
850 match state.lifecycle {
851 Lifecycle::Closed | Lifecycle::Closing => {
852 return Err(AuditTerminalReason::AdmissionClosed)
853 }
854 Lifecycle::Failed(reason) => return Err(reason),
855 Lifecycle::Open => {}
856 }
857 if state.pending.len() >= self.config.max_pending_rows.get() {
858 return Err(AuditTerminalReason::QueueAdmissionExhausted);
859 }
860 state.pending.push(Waiting {
861 event: row.event,
862 producer,
863 responder: tx,
864 });
865 state.submitted_rows += 1;
866 let need_spawn = !state.driver_active;
867 if need_spawn {
868 state.driver_active = true;
869 // Publish this obligation under the admission lock, before a
870 // concurrent physical close can observe terminal/idle state.
871 state.physical_reservations += 1;
872 Some(PhysicalReservation(Arc::clone(&self.inner)))
873 } else {
874 None
875 }
876 };
877 if let Some(registration) = supervisor_registration {
878 self.spawn_supervisor_if_idle(registration);
879 }
880
881 if wait_until_resolved {
882 match tokio::time::timeout(self.config.admission_deadline, &mut rx).await {
883 Ok(Ok(result)) => return result,
884 Ok(Err(_recv_error)) => return Err(AuditTerminalReason::DriverJoinLost),
885 Err(_elapsed) => tracing::warn!(
886 ?producer,
887 "strict audit obligation remains enqueued after the admission wait deadline; \
888 waiting up to the resolution deadline for its real terminal outcome"
889 ),
890 }
891 return match tokio::time::timeout(self.config.resolution_deadline, rx).await {
892 Ok(Ok(result)) => result,
893 Ok(Err(_recv_error)) => Err(AuditTerminalReason::DriverJoinLost),
894 // Same non-removal contract as the `AdmissionDeadlineExpired`
895 // arm below: the row is left exactly where the driver holds
896 // it. The caller's domain effect already committed by the
897 // time it reached this wait, so it is never retried here;
898 // the audit outcome itself is what remains unresolved.
899 Err(_elapsed) => {
900 tracing::warn!(
901 ?producer,
902 "strict audit obligation remains unresolved after the resolution \
903 deadline; giving up on this caller's wait — the committed effect is \
904 not retried and the row is not re-enqueued"
905 );
906 Err(AuditTerminalReason::ResolutionDeadlineExpired)
907 }
908 };
909 }
910
911 match tokio::time::timeout(self.config.admission_deadline, rx).await {
912 Ok(Ok(result)) => result,
913 Ok(Err(_recv_error)) => Err(AuditTerminalReason::DriverJoinLost),
914 // The row was already pushed onto `state.pending` above (and
915 // `submitted_rows` incremented) before this wait began — this is
916 // a deadline elapsing on an enqueued row, not a queue-full
917 // refusal, so it gets its own terminal reason (khive#2117,
918 // khive#2208). The row is left in place for the driver to drain;
919 // this arm performs no removal.
920 Err(_elapsed) => Err(AuditTerminalReason::AdmissionDeadlineExpired),
921 }
922 }
923}
924
925async fn supervisor_loop(
926 inner: Arc<Inner>,
927 store: Arc<dyn EventStore>,
928 config: Arc<AuditBatchConfig>,
929) {
930 loop {
931 let (waiting, wedged_generation_id) = {
932 let mut state = inner.state.lock();
933 if matches!(state.lifecycle, Lifecycle::Failed(_)) {
934 state.driver_active = false;
935 break;
936 }
937 if state.pending.is_empty() {
938 state.driver_active = false;
939 break;
940 }
941 let take_n = state
942 .pending
943 .len()
944 .min(config.max_rows_per_generation.get());
945 let waiting: Vec<Waiting> = state.pending.drain(..take_n).collect();
946 let generation_id = state.next_generation_id;
947 state.next_generation_id += 1;
948 // khive#2331: a store whose append never returns must not mint
949 // an unbounded number of detached appends. Before committing to
950 // spawning this generation's child, check whether the cap is
951 // already saturated by prior abandoned generations still
952 // running in the background — if so, this generation is shed
953 // below instead of ever calling the store.
954 if state.outstanding_abandoned_appends >= config.max_abandoned_appends.get() {
955 (waiting, Some(generation_id))
956 } else {
957 state.in_flight_generation = Some(generation_id);
958 (waiting, None)
959 }
960 };
961
962 if let Some(generation_id) = wedged_generation_id {
963 tracing::warn!(
964 generation_id,
965 max_abandoned_appends = config.max_abandoned_appends.get(),
966 "audit store treated as wedged: max_abandoned_appends detached appends are \
967 already outstanding; shedding this generation without attempting an append"
968 );
969 let submitted = waiting.len() as u64;
970 {
971 let mut state = inner.state.lock();
972 state.flush_failures += 1;
973 state.record_generation(AuditGenerationSnapshot {
974 generation_id,
975 submitted_rows: submitted,
976 committed_rows: 0,
977 store_batch_calls: 0,
978 terminal_reason: Some(AuditTerminalReason::StoreWedged),
979 });
980 }
981 for w in waiting {
982 record_degradation_if_pure(&inner, w.producer);
983 let _ = w.responder.send(Err(AuditTerminalReason::StoreWedged));
984 }
985 continue;
986 }
987
988 #[cfg(any(test, feature = "fault-injection"))]
989 if fault::SUPERVISOR_PANIC.swap(false, Ordering::SeqCst) {
990 let _guard = SupervisorGuard::armed(&inner, waiting);
991 panic!("adr133 fault injection: audit_batch supervisor_panic");
992 }
993 #[cfg(any(test, feature = "fault-injection"))]
994 if fault::SUPERVISOR_SLEEP_BEFORE_SPAWN.swap(false, Ordering::SeqCst) {
995 let guard = SupervisorGuard::armed(&inner, waiting);
996 tokio::time::sleep(Duration::from_secs(3600)).await;
997 drop(guard);
998 continue;
999 }
1000
1001 let guard = SupervisorGuard::armed(&inner, waiting);
1002 let events: Vec<Event> = guard
1003 .waiting
1004 .as_ref()
1005 .expect("guard freshly armed")
1006 .iter()
1007 .map(|w| w.event.clone())
1008 .collect();
1009
1010 let mut child: JoinHandle<GenerationResult> = inner.spawn_physical(
1011 inner.reserve_physical(),
1012 run_generation(store.clone(), events, config.clone()),
1013 );
1014
1015 #[cfg(any(test, feature = "fault-injection"))]
1016 if fault::CHILD_CANCEL.swap(false, Ordering::SeqCst) {
1017 child.abort();
1018 }
1019 #[cfg(any(test, feature = "fault-injection"))]
1020 if fault::JOIN_LOST.swap(false, Ordering::SeqCst) {
1021 drop(child);
1022 guard.fail(AuditTerminalReason::DriverJoinLost);
1023 continue;
1024 }
1025
1026 let append_deadline = driver_append_deadline(&config);
1027 let join_result = match tokio::time::timeout(append_deadline, &mut child).await {
1028 Ok(join_result) => join_result,
1029 Err(_elapsed) => {
1030 // The append is still in flight and may not be safely
1031 // cancellable mid-write (a blocking storage call cannot be
1032 // forced to stop), so `child` is not aborted here — it is
1033 // handed to a detached reaper that drives it to completion
1034 // and discards whatever it eventually returns. Every waiter
1035 // on this generation is resolved now, and the driver loops
1036 // back to `pending` immediately instead of staying pinned on
1037 // this one stalled call, which is what actually starves
1038 // later admission (khive#2331).
1039 tracing::warn!(
1040 ?append_deadline,
1041 "audit generation append exceeded the driver's own bound; abandoning \
1042 this generation so admission capacity recovers for later callers"
1043 );
1044 let waiting = guard.disarm();
1045 let submitted = waiting.len() as u64;
1046 {
1047 let mut state = inner.state.lock();
1048 state.in_flight_generation = None;
1049 state.flush_failures += 1;
1050 state.outstanding_abandoned_appends += 1;
1051 let generation_id = state.next_generation_id.saturating_sub(1);
1052 state.record_generation(AuditGenerationSnapshot {
1053 generation_id,
1054 submitted_rows: submitted,
1055 committed_rows: 0,
1056 store_batch_calls: 0,
1057 terminal_reason: Some(AuditTerminalReason::DriverAppendAbandoned),
1058 });
1059 }
1060 for w in waiting {
1061 record_degradation_if_pure(&inner, w.producer);
1062 let _ = w
1063 .responder
1064 .send(Err(AuditTerminalReason::DriverAppendAbandoned));
1065 }
1066 let reaper_inner = inner.clone();
1067 inner.spawn_physical(inner.reserve_physical(), async move {
1068 let result = child.await;
1069 let mut state = reaper_inner.state.lock();
1070 state.outstanding_abandoned_appends =
1071 state.outstanding_abandoned_appends.saturating_sub(1);
1072 match result {
1073 Ok(GenerationResult::Committed(_)) => state.late_append_commits += 1,
1074 Ok(GenerationResult::Failed(_) | GenerationResult::FakedInconsistent)
1075 | Err(_) => state.late_append_failures += 1,
1076 }
1077 });
1078 continue;
1079 }
1080 };
1081 match join_result {
1082 Err(join_err) => {
1083 let reason = if join_err.is_panic() {
1084 AuditTerminalReason::DriverPanicked
1085 } else {
1086 AuditTerminalReason::DriverCancelled
1087 };
1088 guard.fail(reason);
1089 }
1090 Ok(GenerationResult::FakedInconsistent) => {
1091 guard.fail(AuditTerminalReason::DriverExitedInconsistent);
1092 }
1093 Ok(GenerationResult::Failed(reason)) => {
1094 let waiting = guard.disarm();
1095 let submitted = waiting.len() as u64;
1096 {
1097 let mut state = inner.state.lock();
1098 state.in_flight_generation = None;
1099 state.store_batch_calls += 1;
1100 state.flush_failures += 1;
1101 state.degraded = true;
1102 let generation_id = state.next_generation_id.saturating_sub(1);
1103 state.record_generation(AuditGenerationSnapshot {
1104 generation_id,
1105 submitted_rows: submitted,
1106 committed_rows: 0,
1107 store_batch_calls: 1,
1108 terminal_reason: Some(reason),
1109 });
1110 }
1111 for w in waiting {
1112 record_degradation_if_pure(&inner, w.producer);
1113 let _ = w.responder.send(Err(reason));
1114 }
1115 }
1116 Ok(GenerationResult::Committed(dispositions)) => {
1117 let waiting = guard.disarm();
1118 if dispositions.len() != waiting.len() {
1119 // Defensive: the store must preserve input order/length.
1120 // Treat a mismatch as the driver having exited in a
1121 // state that cannot be reconciled with the accepted
1122 // waiters.
1123 {
1124 let mut state = inner.state.lock();
1125 state.in_flight_generation = None;
1126 }
1127 inner.fail_driver(AuditTerminalReason::DriverExitedInconsistent, waiting);
1128 continue;
1129 }
1130 let mut committed_n = 0u64;
1131 for d in &dispositions {
1132 if !matches!(d, EventAppendDisposition::IdentityConflict) {
1133 committed_n += 1;
1134 }
1135 }
1136 let submitted = dispositions.len() as u64;
1137 {
1138 let mut state = inner.state.lock();
1139 state.in_flight_generation = None;
1140 state.store_batch_calls += 1;
1141 state.committed_rows += committed_n;
1142 let generation_id = state.next_generation_id.saturating_sub(1);
1143 state.record_generation(AuditGenerationSnapshot {
1144 generation_id,
1145 submitted_rows: submitted,
1146 committed_rows: committed_n,
1147 store_batch_calls: 1,
1148 terminal_reason: None,
1149 });
1150 }
1151 for (w, disposition) in waiting.into_iter().zip(dispositions) {
1152 let result = match disposition {
1153 EventAppendDisposition::Inserted => Ok(AuditCommitOutcome::Committed),
1154 EventAppendDisposition::AlreadyPresentIdentical => {
1155 Ok(AuditCommitOutcome::AlreadyPresentIdentical)
1156 }
1157 EventAppendDisposition::IdentityConflict => {
1158 record_degradation_if_pure(&inner, w.producer);
1159 Err(AuditTerminalReason::IdentityConflict)
1160 }
1161 };
1162 let _ = w.responder.send(result);
1163 }
1164 }
1165 }
1166 }
1167}
1168
1169#[async_trait::async_trait]
1170impl AuditBatchControl for AuditBatch {
1171 async fn submit(
1172 &self,
1173 row: PreparedAuditRow,
1174 ) -> Result<AuditCommitOutcome, AuditTerminalReason> {
1175 self.submit_with_wait_policy(row, false).await
1176 }
1177
1178 async fn quiesce(&self) -> Result<(), AuditTerminalReason> {
1179 loop {
1180 let (lifecycle, idle) = {
1181 let state = self.inner.state.lock();
1182 (state.lifecycle.clone(), state.is_idle())
1183 };
1184 match lifecycle {
1185 Lifecycle::Failed(reason) => return Err(reason),
1186 Lifecycle::Closed => return Ok(()),
1187 _ if idle => return Ok(()),
1188 _ => tokio::time::sleep(Duration::from_millis(2)).await,
1189 }
1190 }
1191 }
1192
1193 async fn close_and_drain(&self) -> Result<(), AuditTerminalReason> {
1194 {
1195 let mut state = self.inner.state.lock();
1196 if matches!(state.lifecycle, Lifecycle::Open) {
1197 state.lifecycle = Lifecycle::Closing;
1198 }
1199 }
1200 let result = AuditBatchControl::quiesce(self).await;
1201 {
1202 let mut state = self.inner.state.lock();
1203 if result.is_ok() && matches!(state.lifecycle, Lifecycle::Closing) {
1204 state.lifecycle = Lifecycle::Closed;
1205 }
1206 }
1207 let handle = self.supervisor.lock().take();
1208 if let Some(handle) = handle {
1209 let _ = handle.await;
1210 }
1211 result
1212 }
1213}
1214
1215#[cfg(any(test, feature = "fault-injection"))]
1216mod fault {
1217 use std::sync::atomic::AtomicBool;
1218
1219 pub(super) static CHILD_PANIC: AtomicBool = AtomicBool::new(false);
1220 pub(super) static CHILD_CANCEL: AtomicBool = AtomicBool::new(false);
1221 pub(super) static SUPERVISOR_PANIC: AtomicBool = AtomicBool::new(false);
1222 pub(super) static SUPERVISOR_SLEEP_BEFORE_SPAWN: AtomicBool = AtomicBool::new(false);
1223 pub(super) static JOIN_LOST: AtomicBool = AtomicBool::new(false);
1224 pub(super) static INCONSISTENT_EXIT: AtomicBool = AtomicBool::new(false);
1225}
1226
1227/// Deterministic fault-injection arms consumed by exactly one subsequent
1228/// generation each. Test-only surface, gated identically to the rest of this
1229/// crate's `fault-injection` fixtures (see `crate::operations`).
1230#[cfg(any(test, feature = "fault-injection"))]
1231pub mod fault_injection {
1232 use std::sync::atomic::Ordering;
1233
1234 pub fn arm_child_panic() {
1235 super::fault::CHILD_PANIC.store(true, Ordering::SeqCst);
1236 }
1237 pub fn arm_child_cancel() {
1238 super::fault::CHILD_CANCEL.store(true, Ordering::SeqCst);
1239 }
1240 pub fn arm_supervisor_panic() {
1241 super::fault::SUPERVISOR_PANIC.store(true, Ordering::SeqCst);
1242 }
1243 pub fn arm_supervisor_sleep_before_spawn() {
1244 super::fault::SUPERVISOR_SLEEP_BEFORE_SPAWN.store(true, Ordering::SeqCst);
1245 }
1246 pub fn arm_join_lost() {
1247 super::fault::JOIN_LOST.store(true, Ordering::SeqCst);
1248 }
1249 pub fn arm_inconsistent_exit() {
1250 super::fault::INCONSISTENT_EXIT.store(true, Ordering::SeqCst);
1251 }
1252}
1253
1254// ── Test-attribution surface (R2) ────────────────────────────────────────
1255//
1256// Additive only: production attribution still lands in the public
1257// `db_diagnostics` counters via `metrics_snapshot`-shaped data supplied at
1258// that seam. This surface never substitutes for it.
1259#[cfg(any(test, feature = "test-internals"))]
1260mod test_internals {
1261 use super::*;
1262
1263 /// A point-in-time view of the batch's counters and generation history,
1264 /// used by mechanism tests to compute a delta across a measured
1265 /// operation.
1266 #[derive(Debug, Clone)]
1267 pub struct AuditBatchSnapshot {
1268 pub pending_rows: usize,
1269 pub in_flight_generation: Option<u64>,
1270 pub driver_active: bool,
1271 pub next_generation_id: u64,
1272 pub submitted_rows: u64,
1273 pub committed_rows: u64,
1274 pub store_batch_calls: u64,
1275 pub per_generation: Vec<AuditGenerationSnapshot>,
1276 /// Live count of detached `DriverAppendAbandoned` appends still
1277 /// running, bounded by `AuditBatchConfig::max_abandoned_appends`.
1278 pub outstanding_abandoned_appends: usize,
1279 }
1280
1281 impl AuditBatchSnapshot {
1282 pub fn is_idle(&self) -> bool {
1283 self.pending_rows == 0 && self.in_flight_generation.is_none() && !self.driver_active
1284 }
1285 }
1286
1287 #[derive(Debug, Clone, Copy)]
1288 pub struct AuditBatchMetricsSnapshot {
1289 pub flush_failures: u64,
1290 pub degraded_rows: u64,
1291 pub degraded: bool,
1292 pub late_append_commits: u64,
1293 pub late_append_failures: u64,
1294 }
1295
1296 #[derive(Debug, Clone, PartialEq)]
1297 pub struct AuditBatchDelta {
1298 pub submitted_rows: u64,
1299 pub committed_rows: u64,
1300 pub store_batch_calls: u64,
1301 pub per_generation: Vec<AuditGenerationSnapshot>,
1302 }
1303
1304 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1305 pub enum AuditSnapshotError {
1306 CounterRegressed,
1307 GenerationHistoryRegressed,
1308 }
1309
1310 impl AuditBatch {
1311 pub fn test_snapshot(&self) -> AuditBatchSnapshot {
1312 let state = self.inner.state.lock();
1313 AuditBatchSnapshot {
1314 pending_rows: state.pending.len(),
1315 in_flight_generation: state.in_flight_generation,
1316 driver_active: state.driver_active,
1317 next_generation_id: state.next_generation_id,
1318 submitted_rows: state.submitted_rows,
1319 committed_rows: state.committed_rows,
1320 store_batch_calls: state.store_batch_calls,
1321 per_generation: state.generations.clone(),
1322 outstanding_abandoned_appends: state.outstanding_abandoned_appends,
1323 }
1324 }
1325
1326 pub fn metrics_snapshot(&self) -> AuditBatchMetricsSnapshot {
1327 let state = self.inner.state.lock();
1328 AuditBatchMetricsSnapshot {
1329 flush_failures: state.flush_failures,
1330 degraded_rows: state.degraded_rows,
1331 degraded: state.degraded,
1332 late_append_commits: state.late_append_commits,
1333 late_append_failures: state.late_append_failures,
1334 }
1335 }
1336
1337 /// Abort the currently-retained supervisor `JoinHandle`, if any,
1338 /// simulating a shutdown abort landing mid-generation. Returns
1339 /// whether a handle was found and aborted. Test-only: exercises the
1340 /// R1 supervisor-cancellation path without reaching into a private
1341 /// field from outside this module.
1342 pub fn test_abort_supervisor(&self) -> bool {
1343 let handle = self.supervisor.lock().take();
1344 match handle {
1345 Some(handle) => {
1346 handle.abort();
1347 true
1348 }
1349 None => false,
1350 }
1351 }
1352 }
1353
1354 /// Checked monotonic subtraction. Rejects a regressed counter or a
1355 /// generation history that is not an append-only extension of `before`.
1356 pub fn audit_delta(
1357 before: &AuditBatchSnapshot,
1358 after: &AuditBatchSnapshot,
1359 ) -> Result<AuditBatchDelta, AuditSnapshotError> {
1360 let submitted_rows = after
1361 .submitted_rows
1362 .checked_sub(before.submitted_rows)
1363 .ok_or(AuditSnapshotError::CounterRegressed)?;
1364 let committed_rows = after
1365 .committed_rows
1366 .checked_sub(before.committed_rows)
1367 .ok_or(AuditSnapshotError::CounterRegressed)?;
1368 let store_batch_calls = after
1369 .store_batch_calls
1370 .checked_sub(before.store_batch_calls)
1371 .ok_or(AuditSnapshotError::CounterRegressed)?;
1372 if after.per_generation.len() < before.per_generation.len() {
1373 return Err(AuditSnapshotError::GenerationHistoryRegressed);
1374 }
1375 if after.per_generation[..before.per_generation.len()] != before.per_generation[..] {
1376 return Err(AuditSnapshotError::GenerationHistoryRegressed);
1377 }
1378 let per_generation = after.per_generation[before.per_generation.len()..].to_vec();
1379 Ok(AuditBatchDelta {
1380 submitted_rows,
1381 committed_rows,
1382 store_batch_calls,
1383 per_generation,
1384 })
1385 }
1386
1387 /// Exhaustive, non-wildcard producer classification (see
1388 /// [`super::AuditProducer`] and [`super::classify`], the real
1389 /// crate-private definitions this mirrors one-for-one). The doctest
1390 /// below proves the general property those definitions rely on: a match
1391 /// over a non-`#[non_exhaustive]` enum that omits a variant, with no
1392 /// wildcard arm to silently absorb it, fails to compile rather than
1393 /// passing an incomplete classification.
1394 ///
1395 /// ```compile_fail
1396 /// enum AuditProducer {
1397 /// GateDenied,
1398 /// DispatchSucceeded,
1399 /// DispatchFailed,
1400 /// UnknownVerb,
1401 /// GitDigestReceipt,
1402 /// ConfigLocked,
1403 /// RecallExecuted,
1404 /// }
1405 ///
1406 /// fn describe(p: AuditProducer) -> &'static str {
1407 /// match p {
1408 /// AuditProducer::GateDenied => "obligation",
1409 /// AuditProducer::DispatchSucceeded => "obligation",
1410 /// AuditProducer::DispatchFailed => "obligation",
1411 /// AuditProducer::UnknownVerb => "obligation",
1412 /// AuditProducer::GitDigestReceipt => "obligation",
1413 /// AuditProducer::ConfigLocked => "observability",
1414 /// // RecallExecuted intentionally omitted: a non-exhaustive
1415 /// // match must fail to compile, proving no variant can
1416 /// // silently fall through an absent wildcard arm.
1417 /// }
1418 /// }
1419 /// ```
1420 #[allow(dead_code)]
1421 struct DoctestAnchor;
1422}
1423
1424#[cfg(any(test, feature = "test-internals"))]
1425pub use test_internals::*;
1426
1427#[cfg(test)]
1428#[path = "audit_batch_physical_close_tests.rs"]
1429mod physical_close_tests;