Skip to main content

khive_runtime/
audit_batch.rs

1//! ADR-133 Slice 1: the audit-batch seam.
2//!
3//! Incidental audit writes (gate denials, dispatch outcomes, config-lock
4//! rows, git.digest receipts, and pure-observability rows like
5//! `RecallExecuted`) no longer take one writer-task acquisition per row on
6//! the request hot path. Concurrent submissions arriving while a generation
7//! is committing share the *next* generation instead of each taking their
8//! own writer acquisition, so N concurrent producers collapse to one
9//! [`khive_storage::EventStore::append_events_idempotent`] call per
10//! generation rather than N.
11//!
12//! [`AuditBatch`] owns this seam. A lazily-spawned supervisor task drains
13//! pending rows into generations and drives each through the store; the
14//! supervisor's own `JoinHandle` is retained (never discarded) so an
15//! abnormal exit — panic, cancellation, a lost child join, or a driver that
16//! returns `Ok` while state is not terminally consistent — is observed and
17//! converted into a `Failed` transition with all accepted waiters resolved,
18//! per owner ruling R1/R4 (`.khive/OWNER_RULING_adr133_gate.md`).
19
20#[cfg(any(test, feature = "fault-injection"))]
21use std::sync::atomic::Ordering;
22use std::sync::Arc;
23use std::time::Duration;
24
25use parking_lot::Mutex;
26use tokio::sync::oneshot;
27use tokio::task::JoinHandle;
28
29use khive_storage::event::EventAppendDisposition;
30use khive_storage::{Event, EventStore, StorageError, WriterTaskRequestState};
31
32/// Coarse durability classification for an [`AuditProducer`]. See
33/// [`classify`] for the exhaustive mapping.
34#[derive(Debug, Clone, Copy, PartialEq, Eq)]
35pub(crate) enum AuditProductionClass {
36    /// The dispatch this row audits carries an obligation: the caller-visible
37    /// outcome must not silently diverge from what was durably recorded.
38    DispatchObligation,
39    /// The row is a best-effort observability signal; a non-commit degrades
40    /// gracefully rather than blocking or failing the dispatch it audits.
41    PureObservability,
42}
43
44/// Every call site that can submit a row through [`AuditBatchControl`].
45/// Adding a variant here without extending the crate-private `classify`
46/// function's match is a compile error — there is no wildcard arm.
47#[derive(Debug, Clone, Copy, PartialEq, Eq)]
48pub enum AuditProducer {
49    /// The gate denied a dispatch; the denial itself is audited.
50    GateDenied,
51    /// The gate backend was unreachable and the dispatch failed closed; the
52    /// refusal is audited best-effort: the typed `GateUnavailable` refusal
53    /// already fails the dispatch, so a lost row degrades diagnostics only.
54    GateUnavailable,
55    /// A pack dispatch returned a successful result.
56    DispatchSucceeded,
57    /// A pack dispatch returned an error result.
58    DispatchFailed,
59    /// A redirected KG read consulted the gate on its effective entity id.
60    EffectiveTargetCheck,
61    /// The gate allowed a verb no pack owns.
62    UnknownVerb,
63    /// The strict `git.digest` success receipt (schema v2).
64    GitDigestReceipt,
65    /// A drained process-lifetime `OnceLock` config-lock row.
66    ConfigLocked,
67    /// A `memory.recall` execution's pure-observability audit row.
68    /// Submitted by the recall handler through its owning registry's batch.
69    RecallExecuted,
70}
71
72/// Classify an [`AuditProducer`] into its [`AuditProductionClass`]. One
73/// exhaustive match, no wildcard arm.
74pub(crate) const fn classify(producer: AuditProducer) -> AuditProductionClass {
75    match producer {
76        AuditProducer::GateDenied
77        | AuditProducer::DispatchSucceeded
78        | AuditProducer::DispatchFailed
79        | AuditProducer::EffectiveTargetCheck
80        | AuditProducer::UnknownVerb
81        | AuditProducer::GitDigestReceipt => AuditProductionClass::DispatchObligation,
82        AuditProducer::ConfigLocked
83        | AuditProducer::RecallExecuted
84        | AuditProducer::GateUnavailable => AuditProductionClass::PureObservability,
85    }
86}
87
88/// Exhaustive terminal reasons an [`AuditBatchControl::submit`],
89/// [`AuditBatchControl::quiesce`], or [`AuditBatchControl::close_and_drain`]
90/// call can resolve to. Never mapped through a wildcard arm anywhere in this
91/// module (R4).
92#[derive(Debug, Clone, Copy, PartialEq, Eq)]
93pub enum AuditTerminalReason {
94    /// `EventStore::preflight_event` rejected the row before it was ever
95    /// enqueued. The row was never counted as submitted.
96    PreflightRejected,
97    /// The batch is `Closing` or `Closed`; new admission is refused.
98    AdmissionClosed,
99    /// `AuditBatchConfig::max_pending_rows` was reached before this row was
100    /// enqueued. The row was never counted as submitted and never shared a
101    /// generation with anyone — safe to retry, and doing so applies the
102    /// obligation at most once.
103    QueueAdmissionExhausted,
104    /// The row was already enqueued (counted in `submitted_rows`) when this
105    /// caller's `AuditBatchConfig::admission_deadline` elapsed waiting for
106    /// its generation's outcome. Unlike [`Self::QueueAdmissionExhausted`],
107    /// the row was not refused: by the moment the deadline fires it may
108    /// still be sitting in `state.pending`, or the driver may have already
109    /// drained it into an in-flight generation — either way it remains
110    /// enqueued and unresolved, and is committed (or terminally failed) by
111    /// the generation driver independently of this caller's timeout, so the
112    /// caller cannot tell from this reason alone whether the row eventually
113    /// committed, or even which of those two states it was in when the
114    /// deadline elapsed. Retrying is only safe for an idempotent caller —
115    /// the prior submission may still land. When this reason degrades an
116    /// admission-degrade-safe read's own audit obligation, it is counted
117    /// separately from [`Self::QueueAdmissionExhausted`] — see
118    /// `pack::audit_admission_unresolved_obligation_count` — precisely
119    /// because a row counted here may still commit, unlike one refused
120    /// before enqueue.
121    AdmissionDeadlineExpired,
122    /// Reached only through `AuditBatch::submit_until_resolved`: the row's
123    /// [`Self::AdmissionDeadlineExpired`] wait had already elapsed, and this
124    /// caller's own `AuditBatchConfig::resolution_deadline` then also
125    /// elapsed still waiting for the row's real generation outcome. The
126    /// caller reaches this only after its domain effect has already
127    /// committed, so the effect is never retried and the row is never
128    /// re-enqueued — but unlike an ordinary commit, the caller now learns
129    /// the effect committed while the audit outcome itself is unresolved.
130    /// Same as [`Self::AdmissionDeadlineExpired`], the row is left exactly
131    /// where the driver holds it (`state.pending`, or already mid-generation)
132    /// for the driver to resolve independently — this reason performs no
133    /// removal. Kept distinct from [`Self::AdmissionDeadlineExpired`] so a
134    /// caller, and diagnostics reading this reason, can tell a merely-slow
135    /// admission wait apart from a resolution wait that gave up entirely.
136    ResolutionDeadlineExpired,
137    /// A row shared this generation's id with a previously stored row whose
138    /// columns or observation projection did not match exactly.
139    IdentityConflict,
140    /// `classify_store_error` judged the store's error non-retryable, and the
141    /// generation stopped on that attempt. Usually that is the first attempt,
142    /// but not necessarily: a generation whose earlier attempts failed
143    /// retryably and whose next one returns a non-retryable error reports
144    /// this reason too, because the attempt that decided the outcome is the
145    /// non-retryable one. Distinct from [`Self::RetryExhausted`], which the
146    /// classifier judged safe to retry on every attempt and which failed
147    /// anyway once they ran out — this reason carries no such hope. Whatever
148    /// the store returned on the deciding attempt, it is not the kind of
149    /// failure `AuditBatchConfig::max_commit_attempts` exists to ride out,
150    /// so a caller or an automated retry policy reading this reason should
151    /// not schedule a bare retry of the same call and should instead treat
152    /// it as a storage fault needing attention.
153    StoreFailure,
154    /// The store returned a `classify_store_error`-retryable error on every
155    /// one of `AuditBatchConfig::max_commit_attempts` attempts for this
156    /// generation, and the last attempt still failed retryable. Kept
157    /// distinct from [`Self::StoreFailure`] — the same way
158    /// [`Self::ResolutionDeadlineExpired`] is kept distinct from
159    /// [`Self::AdmissionDeadlineExpired`] — so a caller, and diagnostics
160    /// reading this reason, can tell a store call the classifier judged
161    /// hopeless apart from one that kept failing a condition (write-queue or
162    /// writer-task pressure, pool or timeout) the classifier judged
163    /// transient. The underlying condition may still be
164    /// transient at the moment attempts run out (a daemon restart, pool
165    /// pressure outlasting the configured backoff), so an operator or an
166    /// automated retry policy sitting above this batch can choose to wait
167    /// longer and try again rather than treating it identically to
168    /// [`Self::StoreFailure`]. This reason changes no tolerance, deadline,
169    /// retry count, or backoff on its own — it only names which of the two
170    /// causes produced the generation's failure.
171    RetryExhausted,
172    /// The configured `EventStore` backend does not implement
173    /// `append_events_idempotent`.
174    IdempotencyUnsupported,
175    /// The generation driver task panicked, or the supervisor awaiting it
176    /// unwound while armed.
177    DriverPanicked,
178    /// The generation driver task was cancelled/aborted, or the supervisor
179    /// awaiting it was dropped mid-await (shutdown abort) while armed.
180    DriverCancelled,
181    /// The child driver's `JoinHandle` was lost — dropped without ever being
182    /// inspected — so its outcome could not be classified.
183    DriverJoinLost,
184    /// The driver returned `Ok` but locked batch state was not proved
185    /// terminally consistent afterward (`in_flight` still set, or the
186    /// generation's rows were never resolved).
187    DriverExitedInconsistent,
188    /// The driver's own bound on a single generation's
189    /// `EventStore::append_events_idempotent()` call
190    /// (`supervisor_loop`'s `driver_append_deadline`) elapsed while that
191    /// call was still in flight. Unlike [`Self::AdmissionDeadlineExpired`]
192    /// and [`Self::ResolutionDeadlineExpired`], which bound only how long a
193    /// *caller* keeps waiting while the row stays wherever the driver holds
194    /// it, this bounds the driver's own hold: every waiter on the abandoned
195    /// generation is resolved with this reason and the generation is
196    /// removed from the driver's in-flight state, so a stalled store call
197    /// cannot pin `pending` at `max_pending_rows` and starve all later
198    /// admission (khive#2331). The underlying append is not cancelled — it
199    /// may not be safely cancellable mid-flight — so it is left to run to
200    /// completion in the background and its eventual result, whatever it
201    /// is, is discarded; no waiter is still listening for it. The caller's
202    /// domain effect (if any already committed before this row was
203    /// enqueued) is never retried, and the row is never re-enqueued.
204    DriverAppendAbandoned,
205    /// Before spawning a generation's child, the driver found
206    /// `AuditBatchConfig::max_abandoned_appends` detached appends already
207    /// outstanding from prior [`Self::DriverAppendAbandoned`] generations
208    /// (khive#2331). The store is treated as wedged: this generation's rows
209    /// are shed without ever attempting an append — no child task is
210    /// spawned, no store call is made, and every waiter resolves with this
211    /// reason immediately, well inside `driver_append_deadline`. Same
212    /// non-retry contract as [`Self::DriverAppendAbandoned`]: any
213    /// already-committed domain effect is not retried and the row is never
214    /// re-enqueued, and pure-observability producers record degradation.
215    /// The driver reattempts an append on the next generation as soon as an
216    /// outstanding append returns (commit or failure) and the count drops
217    /// back below the cap — recovery needs no timer of its own.
218    StoreWedged,
219}
220
221/// One row accepted for batching: the immutable event identity plus the
222/// producer that minted it, used for classification and, on failure,
223/// degradation accounting.
224pub struct PreparedAuditRow {
225    pub event: Event,
226    pub producer: AuditProducer,
227}
228
229/// What [`AuditBatchControl::submit`] resolves to on a non-error outcome.
230#[derive(Debug, Clone, Copy, PartialEq, Eq)]
231pub enum AuditCommitOutcome {
232    /// The row was freshly inserted.
233    Committed,
234    /// A prior row with the same identity already matched exactly (internal
235    /// retry replaying the same producer-minted identity).
236    AlreadyPresentIdentical,
237}
238
239/// Production-visible snapshot of [`AuditBatch::health_metrics`]. See there
240/// for field semantics. `khive_db::diagnostics::RuntimeAuditBatchMetrics` carries
241/// these three fields plus `admission_refused_obligations`,
242/// `retry_exhausted_obligations`, and `admission_unresolved_obligations`,
243/// which are sourced from process-wide
244/// counters outside `AuditBatch` rather than from this struct — see
245/// `VerbRegistry::audit_batch_metrics`.
246#[derive(Debug, Clone, Copy, PartialEq, Eq)]
247pub struct AuditBatchHealthMetrics {
248    pub flush_failures: u64,
249    pub degraded_rows: u64,
250    pub degraded: bool,
251    /// Detached [`AuditTerminalReason::DriverAppendAbandoned`] appends that
252    /// eventually returned a commit, after their generation had already
253    /// resolved every waiter with `DriverAppendAbandoned`. Lets an operator
254    /// see that a store recorded as wedged later drained.
255    pub late_append_commits: u64,
256    /// Detached [`AuditTerminalReason::DriverAppendAbandoned`] appends that
257    /// eventually returned a failure (store error, panic, or cancellation),
258    /// after their generation had already resolved every waiter with
259    /// `DriverAppendAbandoned`.
260    pub late_append_failures: u64,
261}
262
263/// Tunables for the batch seam. Defaults are conservative; every field is
264/// exercised by at least one mechanism test.
265#[derive(Debug, Clone)]
266pub struct AuditBatchConfig {
267    pub max_pending_rows: std::num::NonZeroUsize,
268    pub max_rows_per_generation: std::num::NonZeroUsize,
269    pub max_commit_attempts: std::num::NonZeroU8,
270    pub retry_backoff: Duration,
271    pub admission_deadline: Duration,
272    /// Caps how much longer `AuditBatch::submit_until_resolved` keeps
273    /// waiting on a row's real generation outcome once `admission_deadline`
274    /// has already elapsed on it. Without this bound a generation stuck on a
275    /// stalled `EventStore::append_events_idempotent` call retains the
276    /// completed write's caller, its request slot, and its audit-lane waiter
277    /// forever, exhausting both request and audit capacity (khive#2331).
278    /// Must be at least `admission_deadline` — validated (debug-only) in
279    /// [`AuditBatch::new`]. Defaults to 6x `admission_deadline`.
280    ///
281    /// Also the single source value the driver's own per-generation append
282    /// bound (`supervisor_loop`'s `driver_append_deadline`) is derived from
283    /// — that bound exists so a stalled append cannot keep the driver from
284    /// ever draining `pending` again, which is what actually exhausts
285    /// admission for every later caller, not just the one already waiting
286    /// on the stuck row. See [`AuditTerminalReason::DriverAppendAbandoned`].
287    pub resolution_deadline: Duration,
288    /// Caps how many [`AuditTerminalReason::DriverAppendAbandoned`] appends
289    /// may be outstanding (handed to a detached task, still running) at
290    /// once. Each outstanding append retains up to
291    /// `max_rows_per_generation` events until it finally returns, so the
292    /// retained-buffer bound this places on a wedged store is
293    /// `max_abandoned_appends * max_rows_per_generation` rows — without it,
294    /// a store whose append never returns mints one such task per
295    /// `driver_append_deadline` with no cap, and both the live task count
296    /// and the retained event batches inside them grow until the process
297    /// exits (khive#2331). Once the cap is reached, `supervisor_loop` sheds
298    /// further generations with [`AuditTerminalReason::StoreWedged`] instead
299    /// of attempting another append; it resumes attempting appends as soon
300    /// as an outstanding one returns and the count drops back below the
301    /// cap. Defaults to 4.
302    pub max_abandoned_appends: std::num::NonZeroUsize,
303}
304
305impl Default for AuditBatchConfig {
306    fn default() -> Self {
307        let admission_deadline = Duration::from_secs(5);
308        Self {
309            max_pending_rows: std::num::NonZeroUsize::new(4096).unwrap(),
310            max_rows_per_generation: std::num::NonZeroUsize::new(256).unwrap(),
311            max_commit_attempts: std::num::NonZeroU8::new(3).unwrap(),
312            retry_backoff: Duration::from_millis(20),
313            admission_deadline,
314            resolution_deadline: admission_deadline * 6,
315            max_abandoned_appends: std::num::NonZeroUsize::new(4).unwrap(),
316        }
317    }
318}
319
320#[async_trait::async_trait]
321pub trait AuditBatchControl: Send + Sync {
322    async fn submit(
323        &self,
324        row: PreparedAuditRow,
325    ) -> Result<AuditCommitOutcome, AuditTerminalReason>;
326    async fn quiesce(&self) -> Result<(), AuditTerminalReason>;
327    async fn close_and_drain(&self) -> Result<(), AuditTerminalReason>;
328}
329
330#[derive(Debug, Clone, PartialEq, Eq)]
331enum Lifecycle {
332    Open,
333    Closing,
334    Closed,
335    Failed(AuditTerminalReason),
336}
337
338struct Waiting {
339    event: Event,
340    producer: AuditProducer,
341    responder: oneshot::Sender<Result<AuditCommitOutcome, AuditTerminalReason>>,
342}
343
344/// A committed/failed generation's accounting. Detailed history is retained when
345/// compiled for tests or with `test-internals`; ordinary builds keep cumulative counters.
346#[derive(Debug, Clone, PartialEq, Eq)]
347pub struct AuditGenerationSnapshot {
348    pub generation_id: u64,
349    pub submitted_rows: u64,
350    pub committed_rows: u64,
351    pub store_batch_calls: u64,
352    pub terminal_reason: Option<AuditTerminalReason>,
353}
354
355struct State {
356    lifecycle: Lifecycle,
357    pending: Vec<Waiting>,
358    in_flight_generation: Option<u64>,
359    driver_active: bool,
360    physical_reservations: usize,
361    next_generation_id: u64,
362    submitted_rows: u64,
363    committed_rows: u64,
364    store_batch_calls: u64,
365    #[cfg(any(test, feature = "test-internals"))]
366    generations: Vec<AuditGenerationSnapshot>,
367    flush_failures: u64,
368    degraded_rows: u64,
369    degraded: bool,
370    /// Detached `DriverAppendAbandoned` appends currently still running,
371    /// bounded by `AuditBatchConfig::max_abandoned_appends`. Incremented
372    /// when `supervisor_loop` abandons a generation and hands its child to a
373    /// detached task; decremented by that task once the child finally
374    /// returns.
375    outstanding_abandoned_appends: usize,
376    late_append_commits: u64,
377    late_append_failures: u64,
378}
379
380impl State {
381    fn new() -> Self {
382        Self {
383            lifecycle: Lifecycle::Open,
384            pending: Vec::new(),
385            in_flight_generation: None,
386            driver_active: false,
387            physical_reservations: 0,
388            next_generation_id: 0,
389            submitted_rows: 0,
390            committed_rows: 0,
391            store_batch_calls: 0,
392            #[cfg(any(test, feature = "test-internals"))]
393            generations: Vec::new(),
394            flush_failures: 0,
395            degraded_rows: 0,
396            degraded: false,
397            outstanding_abandoned_appends: 0,
398            late_append_commits: 0,
399            late_append_failures: 0,
400        }
401    }
402
403    /// No-op outside `test`/`test-internals`; operators read [`AuditBatch::health_metrics`].
404    fn record_generation(&mut self, generation: AuditGenerationSnapshot) {
405        #[cfg(any(test, feature = "test-internals"))]
406        self.generations.push(generation);
407        #[cfg(not(any(test, feature = "test-internals")))]
408        let _ = generation;
409    }
410
411    fn is_idle(&self) -> bool {
412        self.pending.is_empty() && self.in_flight_generation.is_none() && !self.driver_active
413    }
414}
415
416struct Inner {
417    state: Mutex<State>,
418    // Completion witnesses only: physical shutdown never aborts admitted work.
419    physical_tasks: Mutex<Vec<tokio::task::AbortHandle>>,
420}
421
422struct PhysicalReservation(Arc<Inner>);
423
424impl Drop for PhysicalReservation {
425    fn drop(&mut self) {
426        self.0.state.lock().physical_reservations -= 1;
427    }
428}
429
430impl Inner {
431    fn reserve_physical(self: &Arc<Self>) -> PhysicalReservation {
432        self.state.lock().physical_reservations += 1;
433        PhysicalReservation(Arc::clone(self))
434    }
435
436    fn spawn_physical<F, T>(
437        self: &Arc<Self>,
438        registration: PhysicalReservation,
439        future: F,
440    ) -> JoinHandle<T>
441    where
442        F: std::future::Future<Output = T> + Send + 'static,
443        T: Send + 'static,
444    {
445        let running = self.reserve_physical();
446        let handle = tokio::spawn(async move {
447            let result = future.await;
448            drop(running);
449            result
450        });
451        {
452            let mut tasks = self.physical_tasks.lock();
453            tasks.retain(|task| !task.is_finished());
454            tasks.push(handle.abort_handle());
455        }
456        // A very short task can finish before its witness is published. Keep
457        // registration reserved until publication to close that zero-work gap.
458        drop(registration);
459        handle
460    }
461
462    fn physically_finished(&self) -> bool {
463        self.state.lock().physical_reservations == 0
464            && self
465                .physical_tasks
466                .lock()
467                .iter()
468                .all(|task| task.is_finished())
469    }
470
471    /// Wins once: the first abnormal driver/supervisor exit sets `Failed`
472    /// and drains every accepted waiter (this generation's, plus anything
473    /// still queued for a future one) with the same typed reason. A later
474    /// abnormal path observes the existing terminal state and does not
475    /// double-count (R1).
476    fn fail_driver(&self, reason: AuditTerminalReason, in_flight_waiters: Vec<Waiting>) {
477        let mut state = self.state.lock();
478        let already_failed = matches!(state.lifecycle, Lifecycle::Failed(_));
479        if !already_failed {
480            state.lifecycle = Lifecycle::Failed(reason);
481        }
482        state.driver_active = false;
483        state.in_flight_generation = None;
484        let drained: Vec<Waiting> = std::mem::take(&mut state.pending);
485        if !already_failed {
486            state.flush_failures += 1;
487            state.degraded = true;
488            let generation_id = state.next_generation_id;
489            state.record_generation(AuditGenerationSnapshot {
490                generation_id,
491                submitted_rows: (in_flight_waiters.len() + drained.len()) as u64,
492                committed_rows: 0,
493                store_batch_calls: 0,
494                terminal_reason: Some(reason),
495            });
496            state.next_generation_id += 1;
497        }
498        drop(state);
499        for waiting in in_flight_waiters.into_iter().chain(drained) {
500            record_degradation_if_pure(self, waiting.producer);
501            let _ = waiting.responder.send(Err(reason));
502        }
503    }
504
505    fn record_degradation(&self) {
506        let mut state = self.state.lock();
507        state.degraded_rows += 1;
508        state.degraded = true;
509    }
510}
511
512fn record_degradation_if_pure(inner: &Inner, producer: AuditProducer) {
513    if classify(producer) == AuditProductionClass::PureObservability {
514        inner.record_degradation();
515    }
516}
517
518/// Armed for the lifetime of one generation's supervision. If dropped while
519/// still armed — the supervisor's own frame unwinding (panic) or being
520/// dropped mid-await (cancellation/shutdown abort) — the guard fails the
521/// generation before returning control to whatever tore it down, so the
522/// caller never observes a background-task-count restoration that raced
523/// ahead of the failure broadcast (R1).
524struct SupervisorGuard<'a> {
525    inner: &'a Arc<Inner>,
526    waiting: Option<Vec<Waiting>>,
527}
528
529impl<'a> SupervisorGuard<'a> {
530    fn armed(inner: &'a Arc<Inner>, waiting: Vec<Waiting>) -> Self {
531        Self {
532            inner,
533            waiting: Some(waiting),
534        }
535    }
536
537    /// Clean disarm: the caller has fully classified the outcome and taken
538    /// ownership of the waiters to resolve them itself.
539    fn disarm(mut self) -> Vec<Waiting> {
540        self.waiting.take().unwrap_or_default()
541    }
542
543    /// Explicit failure classification reached without unwinding/cancelling
544    /// this frame (e.g. a `JoinError` was returned normally). Equivalent to
545    /// what `Drop` does when armed, but callable inline.
546    fn fail(mut self, reason: AuditTerminalReason) {
547        if let Some(waiting) = self.waiting.take() {
548            self.inner.fail_driver(reason, waiting);
549        }
550    }
551}
552
553impl Drop for SupervisorGuard<'_> {
554    fn drop(&mut self) {
555        let Some(waiting) = self.waiting.take() else {
556            return;
557        };
558        let reason = if std::thread::panicking() {
559            AuditTerminalReason::DriverPanicked
560        } else {
561            AuditTerminalReason::DriverCancelled
562        };
563        self.inner.fail_driver(reason, waiting);
564    }
565}
566
567enum RetryDecision {
568    Retry,
569    Terminal(AuditTerminalReason),
570}
571
572fn classify_store_error(err: &StorageError) -> RetryDecision {
573    match err.without_sqlite_write_stage() {
574        // This replay is safe only because the batch append is idempotent.
575        StorageError::WriteOutcomeUnknown { .. } => RetryDecision::Retry,
576        StorageError::WriterOwnerUnavailable { source, .. } => classify_store_error(source),
577        StorageError::WriteQueueFull { .. } | StorageError::WriterTaskBusy { .. } => {
578            RetryDecision::Retry
579        }
580        StorageError::CapacityUnavailable {
581            phase: khive_storage::CapacityUnavailablePhase::LockContended,
582            ..
583        } => RetryDecision::Retry,
584        // Transient availability conditions, not judgments on the batch. The
585        // events-daemon forwarding lane (ADR-170) reports an unreachable or
586        // stalled daemon as `Pool`/`Timeout`; the direct SQL path reports
587        // acquisition pressure the same way. Both are exactly what the
588        // configured bounded retries exist for — treating them as terminal
589        // would abandon a generation on the first blip of a daemon restart.
590        StorageError::Pool { .. } | StorageError::Timeout { .. } => RetryDecision::Retry,
591        // The writer task wraps any request operation that failed and was rolled
592        // back, whatever the cause, so the wrapper alone does not say whether a
593        // retry can succeed: a missing column fails the same way on every
594        // attempt. Judge the cause it carries. A request left in an unknown
595        // state is replayed regardless, because the append is idempotent.
596        StorageError::WriterTaskRequestFailed {
597            request_state,
598            source,
599        } => match request_state {
600            WriterTaskRequestState::NotStarted | WriterTaskRequestState::TransactionRolledBack => {
601                if is_sqlite_busy_or_locked(source) {
602                    RetryDecision::Retry
603                } else {
604                    classify_store_error(source)
605                }
606            }
607            WriterTaskRequestState::SideEffectsUnknown => RetryDecision::Retry,
608        },
609        StorageError::WriterTaskTerminated { request_state, .. } => match request_state {
610            WriterTaskRequestState::NotStarted | WriterTaskRequestState::TransactionRolledBack => {
611                RetryDecision::Retry
612            }
613            WriterTaskRequestState::SideEffectsUnknown => RetryDecision::Retry,
614        },
615        StorageError::Unsupported { operation, .. }
616            if operation.as_ref() == "append_events_idempotent" =>
617        {
618            RetryDecision::Terminal(AuditTerminalReason::IdempotencyUnsupported)
619        }
620        _ => RetryDecision::Terminal(AuditTerminalReason::StoreFailure),
621    }
622}
623
624/// Whether a failed writer request's preserved cause is SQLite contention. The
625/// writer keeps the request body's driver error as the cause, and a driver
626/// error is not one of the typed transient variants `classify_store_error`
627/// retries on its own.
628fn is_sqlite_busy_or_locked(err: &StorageError) -> bool {
629    let StorageError::Driver { source, .. } = err.without_sqlite_write_stage() else {
630        return false;
631    };
632    matches!(
633        source
634            .downcast_ref::<rusqlite::Error>()
635            .and_then(|error| error.sqlite_error_code()),
636        Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked)
637    )
638}
639
640enum GenerationResult {
641    Committed(Vec<EventAppendDisposition>),
642    Failed(AuditTerminalReason),
643    /// Fault-injection only: proves the supervisor's post-`Ok` consistency
644    /// check actually runs.
645    #[cfg_attr(not(any(test, feature = "fault-injection")), allow(dead_code))]
646    FakedInconsistent,
647}
648
649/// The driver's own bound on how long `supervisor_loop` waits for one
650/// generation's `run_generation` child task before abandoning it
651/// (khive#2331; see [`AuditTerminalReason::DriverAppendAbandoned`]).
652///
653/// Derived from `resolution_deadline` rather than a second config field, at
654/// 3x it. That margin is load-bearing, not arbitrary: `AuditBatch::new`
655/// already enforces `resolution_deadline >= admission_deadline`, so the
656/// worst-case *caller*-side wait for a `submit_until_resolved` row —
657/// `admission_deadline` then `resolution_deadline` in sequence — is always
658/// `<= 2 * resolution_deadline`. Bounding the driver at `3 *
659/// resolution_deadline` guarantees it can never resolve a still-waiting
660/// caller's row with `DriverAppendAbandoned` ahead of that caller's own,
661/// more specific `ResolutionDeadlineExpired` reason.
662///
663/// This bound alone only stops the driver from holding one stalled
664/// generation forever — a store whose append never returns still mints one
665/// detached append per `driver_append_deadline` with nothing capping how
666/// many run at once. `AuditBatchConfig::max_abandoned_appends` closes that:
667/// once that many detached appends are outstanding, further generations are
668/// shed with [`AuditTerminalReason::StoreWedged`] instead of attempting
669/// another append. Combined, the two bounds guarantee at most
670/// `max_abandoned_appends` live detached appends at any time, at most that
671/// many retained event batches, and a shed generation costs no store work
672/// at all — it resolves inside this function's caller without ever calling
673/// `run_generation`.
674fn driver_append_deadline(config: &AuditBatchConfig) -> Duration {
675    config.resolution_deadline.saturating_mul(3)
676}
677
678async fn run_generation(
679    store: Arc<dyn EventStore>,
680    events: Vec<Event>,
681    config: Arc<AuditBatchConfig>,
682) -> GenerationResult {
683    #[cfg(any(test, feature = "fault-injection"))]
684    if fault::CHILD_PANIC.swap(false, Ordering::SeqCst) {
685        panic!("adr133 fault injection: audit_batch child_panic");
686    }
687    #[cfg(any(test, feature = "fault-injection"))]
688    if fault::INCONSISTENT_EXIT.swap(false, Ordering::SeqCst) {
689        return GenerationResult::FakedInconsistent;
690    }
691
692    let mut attempt: u8 = 0;
693    loop {
694        attempt += 1;
695        match store.append_events_idempotent(events.clone()).await {
696            Ok(result) => return GenerationResult::Committed(result.rows),
697            Err(err) => {
698                let reason = match classify_store_error(&err) {
699                    RetryDecision::Retry if attempt < config.max_commit_attempts.get() => {
700                        tokio::time::sleep(config.retry_backoff).await;
701                        continue;
702                    }
703                    RetryDecision::Retry => AuditTerminalReason::RetryExhausted,
704                    RetryDecision::Terminal(reason) => reason,
705                };
706                tracing::warn!(
707                    error = %err,
708                    attempts = attempt,
709                    ?reason,
710                    "audit generation failed; its rows were not committed"
711                );
712                return GenerationResult::Failed(reason);
713            }
714        }
715    }
716}
717
718/// The batch owner. Constructed once per configured `EventStore`; every
719/// dispatch-audit call site routes its row through [`AuditBatch::submit`]
720/// instead of taking its own writer-task acquisition.
721pub struct AuditBatch {
722    inner: Arc<Inner>,
723    store: Arc<dyn EventStore>,
724    config: Arc<AuditBatchConfig>,
725    supervisor: Mutex<Option<JoinHandle<()>>>,
726}
727
728impl AuditBatch {
729    pub fn new(store: Arc<dyn EventStore>, config: AuditBatchConfig) -> Arc<Self> {
730        debug_assert!(
731            config.resolution_deadline >= config.admission_deadline,
732            "resolution_deadline ({:?}) must be at least admission_deadline ({:?})",
733            config.resolution_deadline,
734            config.admission_deadline
735        );
736        Arc::new(Self {
737            inner: Arc::new(Inner {
738                state: Mutex::new(State::new()),
739                physical_tasks: Mutex::new(Vec::new()),
740            }),
741            store,
742            config: Arc::new(config),
743            supervisor: Mutex::new(None),
744        })
745    }
746
747    /// Process-lifetime audit-batch health counters, for the registry/
748    /// runtime owner to feed into `db_diagnostics` (D8's operator surface).
749    /// Unlike `test_internals::AuditBatchSnapshot::metrics_snapshot`,
750    /// which is test-only (the module is cfg-gated and invisible to the
751    /// default doc build, so an intra-doc link cannot resolve), this is
752    /// always available.
753    pub fn health_metrics(&self) -> AuditBatchHealthMetrics {
754        let state = self.inner.state.lock();
755        AuditBatchHealthMetrics {
756            flush_failures: state.flush_failures,
757            degraded_rows: state.degraded_rows,
758            degraded: state.degraded,
759            late_append_commits: state.late_append_commits,
760            late_append_failures: state.late_append_failures,
761        }
762    }
763
764    fn spawn_supervisor_if_idle(&self, registration: PhysicalReservation) {
765        let inner = self.inner.clone();
766        let store = self.store.clone();
767        let config = self.config.clone();
768        let handle = self.inner.spawn_physical(registration, async move {
769            supervisor_loop(inner, store, config).await;
770        });
771        *self.supervisor.lock() = Some(handle);
772    }
773
774    /// Stop admission and wait until all accepted append tasks and their
775    /// supervisors/reapers have physically finished. Unlike `close_and_drain`,
776    /// a driver-abandoned append still holds this barrier until it returns.
777    ///
778    /// Task completion witnesses stay on the batch, so cancelling this wait
779    /// neither detaches a taken join nor loses the barrier for a later call.
780    /// No admitted write is aborted. This may wait indefinitely for a store
781    /// that never returns; callers may cancel their wait without cancelling it.
782    /// The batch's own store Arc remains owned until the batch is dropped.
783    pub(crate) async fn close_and_wait_for_physical_completion(
784        &self,
785    ) -> Result<(), AuditTerminalReason> {
786        {
787            let mut state = self.inner.state.lock();
788            if matches!(state.lifecycle, Lifecycle::Open) {
789                state.lifecycle = Lifecycle::Closing;
790            }
791        }
792        let result = AuditBatchControl::quiesce(self).await;
793        while !self.inner.physically_finished() {
794            tokio::time::sleep(Duration::from_millis(2)).await;
795        }
796        {
797            let mut state = self.inner.state.lock();
798            if result.is_ok() && matches!(state.lifecycle, Lifecycle::Closing) {
799                state.lifecycle = Lifecycle::Closed;
800            }
801        }
802        result
803    }
804
805    /// Enqueue one row and keep waiting for its real generation outcome past
806    /// the ordinary admission wait deadline, up to `resolution_deadline`.
807    ///
808    /// This is the narrow khive#2256 seam for a successful operation whose
809    /// domain effect has already committed. Returning
810    /// [`AuditTerminalReason::AdmissionDeadlineExpired`] there would report a
811    /// false operation failure and invite an unsafe retry while the same
812    /// audit row remains enqueued. Pre-enqueue refusal and genuine terminal
813    /// generation failures still return normally. The post-admission wait is
814    /// itself bounded by `resolution_deadline`
815    /// ([`AuditTerminalReason::ResolutionDeadlineExpired`]) so a stalled
816    /// store cannot retain this caller, its request slot, and its audit-lane
817    /// waiter forever (khive#2331).
818    ///
819    /// The two deadlines therefore mean different things to the caller, and
820    /// callers key on the difference. Admission expiry means the row is
821    /// enqueued and this seam keeps waiting; the generation commits it
822    /// independently, so a dispatch that reaches it reports its committed
823    /// result and counts the row as unresolved. Resolution expiry means the
824    /// caller waited for the real outcome and never received one: the commit
825    /// is unconfirmed, not proven absent. That is returned as a terminal
826    /// reason and the dispatch propagates it as a structured
827    /// committed-outcome error carrying the domain result, with no retryable
828    /// context, for every verb rather than only for the receipt verb. Never
829    /// replay the handler or re-enqueue the row on it; the original driver
830    /// work continues on its own.
831    pub(crate) async fn submit_until_resolved(
832        &self,
833        row: PreparedAuditRow,
834    ) -> Result<AuditCommitOutcome, AuditTerminalReason> {
835        self.submit_with_wait_policy(row, true).await
836    }
837
838    async fn submit_with_wait_policy(
839        &self,
840        row: PreparedAuditRow,
841        wait_until_resolved: bool,
842    ) -> Result<AuditCommitOutcome, AuditTerminalReason> {
843        // Pre-enqueue validation (invariant 3): a malformed row is rejected
844        // before it can share a generation with anyone else's.
845        if self.store.preflight_event(&row.event).is_err() {
846            return Err(AuditTerminalReason::PreflightRejected);
847        }
848
849        let producer = row.producer;
850        let (tx, mut rx) = oneshot::channel();
851        let supervisor_registration = {
852            let mut state = self.inner.state.lock();
853            match state.lifecycle {
854                Lifecycle::Closed | Lifecycle::Closing => {
855                    return Err(AuditTerminalReason::AdmissionClosed)
856                }
857                Lifecycle::Failed(reason) => return Err(reason),
858                Lifecycle::Open => {}
859            }
860            if state.pending.len() >= self.config.max_pending_rows.get() {
861                return Err(AuditTerminalReason::QueueAdmissionExhausted);
862            }
863            state.pending.push(Waiting {
864                event: row.event,
865                producer,
866                responder: tx,
867            });
868            state.submitted_rows += 1;
869            let need_spawn = !state.driver_active;
870            if need_spawn {
871                state.driver_active = true;
872                // Publish this obligation under the admission lock, before a
873                // concurrent physical close can observe terminal/idle state.
874                state.physical_reservations += 1;
875                Some(PhysicalReservation(Arc::clone(&self.inner)))
876            } else {
877                None
878            }
879        };
880        if let Some(registration) = supervisor_registration {
881            self.spawn_supervisor_if_idle(registration);
882        }
883
884        if wait_until_resolved {
885            match tokio::time::timeout(self.config.admission_deadline, &mut rx).await {
886                Ok(Ok(result)) => return result,
887                Ok(Err(_recv_error)) => return Err(AuditTerminalReason::DriverJoinLost),
888                Err(_elapsed) => tracing::warn!(
889                    ?producer,
890                    "strict audit obligation remains enqueued after the admission wait deadline; \
891                     waiting up to the resolution deadline for its real terminal outcome"
892                ),
893            }
894            return match tokio::time::timeout(self.config.resolution_deadline, rx).await {
895                Ok(Ok(result)) => result,
896                Ok(Err(_recv_error)) => Err(AuditTerminalReason::DriverJoinLost),
897                // Same non-removal contract as the `AdmissionDeadlineExpired`
898                // arm below: the row is left exactly where the driver holds
899                // it. The caller's domain effect already committed by the
900                // time it reached this wait, so it is never retried here;
901                // the audit outcome itself is what remains unresolved.
902                Err(_elapsed) => {
903                    tracing::warn!(
904                        ?producer,
905                        "strict audit obligation remains unresolved after the resolution \
906                         deadline; giving up on this caller's wait — the committed effect is \
907                         not retried and the row is not re-enqueued"
908                    );
909                    Err(AuditTerminalReason::ResolutionDeadlineExpired)
910                }
911            };
912        }
913
914        match tokio::time::timeout(self.config.admission_deadline, rx).await {
915            Ok(Ok(result)) => result,
916            Ok(Err(_recv_error)) => Err(AuditTerminalReason::DriverJoinLost),
917            // The row was already pushed onto `state.pending` above (and
918            // `submitted_rows` incremented) before this wait began — this is
919            // a deadline elapsing on an enqueued row, not a queue-full
920            // refusal, so it gets its own terminal reason (khive#2117,
921            // khive#2208). The row is left in place for the driver to drain;
922            // this arm performs no removal.
923            Err(_elapsed) => Err(AuditTerminalReason::AdmissionDeadlineExpired),
924        }
925    }
926}
927
928async fn supervisor_loop(
929    inner: Arc<Inner>,
930    store: Arc<dyn EventStore>,
931    config: Arc<AuditBatchConfig>,
932) {
933    loop {
934        let (waiting, wedged_generation_id) = {
935            let mut state = inner.state.lock();
936            if matches!(state.lifecycle, Lifecycle::Failed(_)) {
937                state.driver_active = false;
938                break;
939            }
940            if state.pending.is_empty() {
941                state.driver_active = false;
942                break;
943            }
944            let take_n = state
945                .pending
946                .len()
947                .min(config.max_rows_per_generation.get());
948            let waiting: Vec<Waiting> = state.pending.drain(..take_n).collect();
949            let generation_id = state.next_generation_id;
950            state.next_generation_id += 1;
951            // khive#2331: a store whose append never returns must not mint
952            // an unbounded number of detached appends. Before committing to
953            // spawning this generation's child, check whether the cap is
954            // already saturated by prior abandoned generations still
955            // running in the background — if so, this generation is shed
956            // below instead of ever calling the store.
957            if state.outstanding_abandoned_appends >= config.max_abandoned_appends.get() {
958                (waiting, Some(generation_id))
959            } else {
960                state.in_flight_generation = Some(generation_id);
961                (waiting, None)
962            }
963        };
964
965        if let Some(generation_id) = wedged_generation_id {
966            tracing::warn!(
967                generation_id,
968                max_abandoned_appends = config.max_abandoned_appends.get(),
969                "audit store treated as wedged: max_abandoned_appends detached appends are \
970                 already outstanding; shedding this generation without attempting an append"
971            );
972            let submitted = waiting.len() as u64;
973            {
974                let mut state = inner.state.lock();
975                state.flush_failures += 1;
976                state.record_generation(AuditGenerationSnapshot {
977                    generation_id,
978                    submitted_rows: submitted,
979                    committed_rows: 0,
980                    store_batch_calls: 0,
981                    terminal_reason: Some(AuditTerminalReason::StoreWedged),
982                });
983            }
984            for w in waiting {
985                record_degradation_if_pure(&inner, w.producer);
986                let _ = w.responder.send(Err(AuditTerminalReason::StoreWedged));
987            }
988            continue;
989        }
990
991        #[cfg(any(test, feature = "fault-injection"))]
992        if fault::SUPERVISOR_PANIC.swap(false, Ordering::SeqCst) {
993            let _guard = SupervisorGuard::armed(&inner, waiting);
994            panic!("adr133 fault injection: audit_batch supervisor_panic");
995        }
996        #[cfg(any(test, feature = "fault-injection"))]
997        if fault::SUPERVISOR_SLEEP_BEFORE_SPAWN.swap(false, Ordering::SeqCst) {
998            let guard = SupervisorGuard::armed(&inner, waiting);
999            tokio::time::sleep(Duration::from_secs(3600)).await;
1000            drop(guard);
1001            continue;
1002        }
1003
1004        let guard = SupervisorGuard::armed(&inner, waiting);
1005        let events: Vec<Event> = guard
1006            .waiting
1007            .as_ref()
1008            .expect("guard freshly armed")
1009            .iter()
1010            .map(|w| w.event.clone())
1011            .collect();
1012
1013        let mut child: JoinHandle<GenerationResult> = inner.spawn_physical(
1014            inner.reserve_physical(),
1015            run_generation(store.clone(), events, config.clone()),
1016        );
1017
1018        #[cfg(any(test, feature = "fault-injection"))]
1019        if fault::CHILD_CANCEL.swap(false, Ordering::SeqCst) {
1020            child.abort();
1021        }
1022        #[cfg(any(test, feature = "fault-injection"))]
1023        if fault::JOIN_LOST.swap(false, Ordering::SeqCst) {
1024            drop(child);
1025            guard.fail(AuditTerminalReason::DriverJoinLost);
1026            continue;
1027        }
1028
1029        let append_deadline = driver_append_deadline(&config);
1030        let join_result = match tokio::time::timeout(append_deadline, &mut child).await {
1031            Ok(join_result) => join_result,
1032            Err(_elapsed) => {
1033                // The append is still in flight and may not be safely
1034                // cancellable mid-write (a blocking storage call cannot be
1035                // forced to stop), so `child` is not aborted here — it is
1036                // handed to a detached reaper that drives it to completion
1037                // and discards whatever it eventually returns. Every waiter
1038                // on this generation is resolved now, and the driver loops
1039                // back to `pending` immediately instead of staying pinned on
1040                // this one stalled call, which is what actually starves
1041                // later admission (khive#2331).
1042                tracing::warn!(
1043                    ?append_deadline,
1044                    "audit generation append exceeded the driver's own bound; abandoning \
1045                     this generation so admission capacity recovers for later callers"
1046                );
1047                let waiting = guard.disarm();
1048                let submitted = waiting.len() as u64;
1049                {
1050                    let mut state = inner.state.lock();
1051                    state.in_flight_generation = None;
1052                    state.flush_failures += 1;
1053                    state.outstanding_abandoned_appends += 1;
1054                    let generation_id = state.next_generation_id.saturating_sub(1);
1055                    state.record_generation(AuditGenerationSnapshot {
1056                        generation_id,
1057                        submitted_rows: submitted,
1058                        committed_rows: 0,
1059                        store_batch_calls: 0,
1060                        terminal_reason: Some(AuditTerminalReason::DriverAppendAbandoned),
1061                    });
1062                }
1063                for w in waiting {
1064                    record_degradation_if_pure(&inner, w.producer);
1065                    let _ = w
1066                        .responder
1067                        .send(Err(AuditTerminalReason::DriverAppendAbandoned));
1068                }
1069                let reaper_inner = inner.clone();
1070                inner.spawn_physical(inner.reserve_physical(), async move {
1071                    let result = child.await;
1072                    let mut state = reaper_inner.state.lock();
1073                    state.outstanding_abandoned_appends =
1074                        state.outstanding_abandoned_appends.saturating_sub(1);
1075                    match result {
1076                        Ok(GenerationResult::Committed(_)) => state.late_append_commits += 1,
1077                        Ok(GenerationResult::Failed(_) | GenerationResult::FakedInconsistent)
1078                        | Err(_) => state.late_append_failures += 1,
1079                    }
1080                });
1081                continue;
1082            }
1083        };
1084        match join_result {
1085            Err(join_err) => {
1086                let reason = if join_err.is_panic() {
1087                    AuditTerminalReason::DriverPanicked
1088                } else {
1089                    AuditTerminalReason::DriverCancelled
1090                };
1091                guard.fail(reason);
1092            }
1093            Ok(GenerationResult::FakedInconsistent) => {
1094                guard.fail(AuditTerminalReason::DriverExitedInconsistent);
1095            }
1096            Ok(GenerationResult::Failed(reason)) => {
1097                let waiting = guard.disarm();
1098                let submitted = waiting.len() as u64;
1099                {
1100                    let mut state = inner.state.lock();
1101                    state.in_flight_generation = None;
1102                    state.store_batch_calls += 1;
1103                    state.flush_failures += 1;
1104                    state.degraded = true;
1105                    let generation_id = state.next_generation_id.saturating_sub(1);
1106                    state.record_generation(AuditGenerationSnapshot {
1107                        generation_id,
1108                        submitted_rows: submitted,
1109                        committed_rows: 0,
1110                        store_batch_calls: 1,
1111                        terminal_reason: Some(reason),
1112                    });
1113                }
1114                for w in waiting {
1115                    record_degradation_if_pure(&inner, w.producer);
1116                    let _ = w.responder.send(Err(reason));
1117                }
1118            }
1119            Ok(GenerationResult::Committed(dispositions)) => {
1120                let waiting = guard.disarm();
1121                if dispositions.len() != waiting.len() {
1122                    // Defensive: the store must preserve input order/length.
1123                    // Treat a mismatch as the driver having exited in a
1124                    // state that cannot be reconciled with the accepted
1125                    // waiters.
1126                    {
1127                        let mut state = inner.state.lock();
1128                        state.in_flight_generation = None;
1129                    }
1130                    inner.fail_driver(AuditTerminalReason::DriverExitedInconsistent, waiting);
1131                    continue;
1132                }
1133                let mut committed_n = 0u64;
1134                for d in &dispositions {
1135                    if !matches!(d, EventAppendDisposition::IdentityConflict) {
1136                        committed_n += 1;
1137                    }
1138                }
1139                let submitted = dispositions.len() as u64;
1140                {
1141                    let mut state = inner.state.lock();
1142                    state.in_flight_generation = None;
1143                    state.store_batch_calls += 1;
1144                    state.committed_rows += committed_n;
1145                    let generation_id = state.next_generation_id.saturating_sub(1);
1146                    state.record_generation(AuditGenerationSnapshot {
1147                        generation_id,
1148                        submitted_rows: submitted,
1149                        committed_rows: committed_n,
1150                        store_batch_calls: 1,
1151                        terminal_reason: None,
1152                    });
1153                }
1154                for (w, disposition) in waiting.into_iter().zip(dispositions) {
1155                    let result = match disposition {
1156                        EventAppendDisposition::Inserted => Ok(AuditCommitOutcome::Committed),
1157                        EventAppendDisposition::AlreadyPresentIdentical => {
1158                            Ok(AuditCommitOutcome::AlreadyPresentIdentical)
1159                        }
1160                        EventAppendDisposition::IdentityConflict => {
1161                            record_degradation_if_pure(&inner, w.producer);
1162                            Err(AuditTerminalReason::IdentityConflict)
1163                        }
1164                    };
1165                    let _ = w.responder.send(result);
1166                }
1167            }
1168        }
1169    }
1170}
1171
1172#[async_trait::async_trait]
1173impl AuditBatchControl for AuditBatch {
1174    async fn submit(
1175        &self,
1176        row: PreparedAuditRow,
1177    ) -> Result<AuditCommitOutcome, AuditTerminalReason> {
1178        self.submit_with_wait_policy(row, false).await
1179    }
1180
1181    async fn quiesce(&self) -> Result<(), AuditTerminalReason> {
1182        loop {
1183            let (lifecycle, idle) = {
1184                let state = self.inner.state.lock();
1185                (state.lifecycle.clone(), state.is_idle())
1186            };
1187            match lifecycle {
1188                Lifecycle::Failed(reason) => return Err(reason),
1189                Lifecycle::Closed => return Ok(()),
1190                _ if idle => return Ok(()),
1191                _ => tokio::time::sleep(Duration::from_millis(2)).await,
1192            }
1193        }
1194    }
1195
1196    async fn close_and_drain(&self) -> Result<(), AuditTerminalReason> {
1197        {
1198            let mut state = self.inner.state.lock();
1199            if matches!(state.lifecycle, Lifecycle::Open) {
1200                state.lifecycle = Lifecycle::Closing;
1201            }
1202        }
1203        let result = AuditBatchControl::quiesce(self).await;
1204        {
1205            let mut state = self.inner.state.lock();
1206            if result.is_ok() && matches!(state.lifecycle, Lifecycle::Closing) {
1207                state.lifecycle = Lifecycle::Closed;
1208            }
1209        }
1210        let handle = self.supervisor.lock().take();
1211        if let Some(handle) = handle {
1212            let _ = handle.await;
1213        }
1214        result
1215    }
1216}
1217
1218#[cfg(any(test, feature = "fault-injection"))]
1219mod fault {
1220    use std::sync::atomic::AtomicBool;
1221
1222    pub(super) static CHILD_PANIC: AtomicBool = AtomicBool::new(false);
1223    pub(super) static CHILD_CANCEL: AtomicBool = AtomicBool::new(false);
1224    pub(super) static SUPERVISOR_PANIC: AtomicBool = AtomicBool::new(false);
1225    pub(super) static SUPERVISOR_SLEEP_BEFORE_SPAWN: AtomicBool = AtomicBool::new(false);
1226    pub(super) static JOIN_LOST: AtomicBool = AtomicBool::new(false);
1227    pub(super) static INCONSISTENT_EXIT: AtomicBool = AtomicBool::new(false);
1228}
1229
1230/// Deterministic fault-injection arms consumed by exactly one subsequent
1231/// generation each. Test-only surface, gated identically to the rest of this
1232/// crate's `fault-injection` fixtures (see `crate::operations`).
1233#[cfg(any(test, feature = "fault-injection"))]
1234pub mod fault_injection {
1235    use std::sync::atomic::Ordering;
1236
1237    pub fn arm_child_panic() {
1238        super::fault::CHILD_PANIC.store(true, Ordering::SeqCst);
1239    }
1240    pub fn arm_child_cancel() {
1241        super::fault::CHILD_CANCEL.store(true, Ordering::SeqCst);
1242    }
1243    pub fn arm_supervisor_panic() {
1244        super::fault::SUPERVISOR_PANIC.store(true, Ordering::SeqCst);
1245    }
1246    pub fn arm_supervisor_sleep_before_spawn() {
1247        super::fault::SUPERVISOR_SLEEP_BEFORE_SPAWN.store(true, Ordering::SeqCst);
1248    }
1249    pub fn arm_join_lost() {
1250        super::fault::JOIN_LOST.store(true, Ordering::SeqCst);
1251    }
1252    pub fn arm_inconsistent_exit() {
1253        super::fault::INCONSISTENT_EXIT.store(true, Ordering::SeqCst);
1254    }
1255}
1256
1257// ── Test-attribution surface (R2) ────────────────────────────────────────
1258//
1259// Additive only: production attribution still lands in the public
1260// `db_diagnostics` counters via `metrics_snapshot`-shaped data supplied at
1261// that seam. This surface never substitutes for it.
1262#[cfg(any(test, feature = "test-internals"))]
1263mod test_internals {
1264    use super::*;
1265
1266    /// A point-in-time view of the batch's counters and generation history,
1267    /// used by mechanism tests to compute a delta across a measured
1268    /// operation.
1269    #[derive(Debug, Clone)]
1270    pub struct AuditBatchSnapshot {
1271        pub pending_rows: usize,
1272        pub in_flight_generation: Option<u64>,
1273        pub driver_active: bool,
1274        pub next_generation_id: u64,
1275        pub submitted_rows: u64,
1276        pub committed_rows: u64,
1277        pub store_batch_calls: u64,
1278        pub per_generation: Vec<AuditGenerationSnapshot>,
1279        /// Live count of detached `DriverAppendAbandoned` appends still
1280        /// running, bounded by `AuditBatchConfig::max_abandoned_appends`.
1281        pub outstanding_abandoned_appends: usize,
1282    }
1283
1284    impl AuditBatchSnapshot {
1285        pub fn is_idle(&self) -> bool {
1286            self.pending_rows == 0 && self.in_flight_generation.is_none() && !self.driver_active
1287        }
1288    }
1289
1290    #[derive(Debug, Clone, Copy)]
1291    pub struct AuditBatchMetricsSnapshot {
1292        pub flush_failures: u64,
1293        pub degraded_rows: u64,
1294        pub degraded: bool,
1295        pub late_append_commits: u64,
1296        pub late_append_failures: u64,
1297    }
1298
1299    #[derive(Debug, Clone, PartialEq)]
1300    pub struct AuditBatchDelta {
1301        pub submitted_rows: u64,
1302        pub committed_rows: u64,
1303        pub store_batch_calls: u64,
1304        pub per_generation: Vec<AuditGenerationSnapshot>,
1305    }
1306
1307    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1308    pub enum AuditSnapshotError {
1309        CounterRegressed,
1310        GenerationHistoryRegressed,
1311    }
1312
1313    impl AuditBatch {
1314        pub fn test_snapshot(&self) -> AuditBatchSnapshot {
1315            let state = self.inner.state.lock();
1316            AuditBatchSnapshot {
1317                pending_rows: state.pending.len(),
1318                in_flight_generation: state.in_flight_generation,
1319                driver_active: state.driver_active,
1320                next_generation_id: state.next_generation_id,
1321                submitted_rows: state.submitted_rows,
1322                committed_rows: state.committed_rows,
1323                store_batch_calls: state.store_batch_calls,
1324                per_generation: state.generations.clone(),
1325                outstanding_abandoned_appends: state.outstanding_abandoned_appends,
1326            }
1327        }
1328
1329        pub fn metrics_snapshot(&self) -> AuditBatchMetricsSnapshot {
1330            let state = self.inner.state.lock();
1331            AuditBatchMetricsSnapshot {
1332                flush_failures: state.flush_failures,
1333                degraded_rows: state.degraded_rows,
1334                degraded: state.degraded,
1335                late_append_commits: state.late_append_commits,
1336                late_append_failures: state.late_append_failures,
1337            }
1338        }
1339
1340        /// Abort the currently-retained supervisor `JoinHandle`, if any,
1341        /// simulating a shutdown abort landing mid-generation. Returns
1342        /// whether a handle was found and aborted. Test-only: exercises the
1343        /// R1 supervisor-cancellation path without reaching into a private
1344        /// field from outside this module.
1345        pub fn test_abort_supervisor(&self) -> bool {
1346            let handle = self.supervisor.lock().take();
1347            match handle {
1348                Some(handle) => {
1349                    handle.abort();
1350                    true
1351                }
1352                None => false,
1353            }
1354        }
1355    }
1356
1357    /// Checked monotonic subtraction. Rejects a regressed counter or a
1358    /// generation history that is not an append-only extension of `before`.
1359    pub fn audit_delta(
1360        before: &AuditBatchSnapshot,
1361        after: &AuditBatchSnapshot,
1362    ) -> Result<AuditBatchDelta, AuditSnapshotError> {
1363        let submitted_rows = after
1364            .submitted_rows
1365            .checked_sub(before.submitted_rows)
1366            .ok_or(AuditSnapshotError::CounterRegressed)?;
1367        let committed_rows = after
1368            .committed_rows
1369            .checked_sub(before.committed_rows)
1370            .ok_or(AuditSnapshotError::CounterRegressed)?;
1371        let store_batch_calls = after
1372            .store_batch_calls
1373            .checked_sub(before.store_batch_calls)
1374            .ok_or(AuditSnapshotError::CounterRegressed)?;
1375        if after.per_generation.len() < before.per_generation.len() {
1376            return Err(AuditSnapshotError::GenerationHistoryRegressed);
1377        }
1378        if after.per_generation[..before.per_generation.len()] != before.per_generation[..] {
1379            return Err(AuditSnapshotError::GenerationHistoryRegressed);
1380        }
1381        let per_generation = after.per_generation[before.per_generation.len()..].to_vec();
1382        Ok(AuditBatchDelta {
1383            submitted_rows,
1384            committed_rows,
1385            store_batch_calls,
1386            per_generation,
1387        })
1388    }
1389
1390    /// Exhaustive, non-wildcard producer classification (see
1391    /// [`super::AuditProducer`] and [`super::classify`], the real
1392    /// crate-private definitions this mirrors one-for-one). The doctest
1393    /// below proves the general property those definitions rely on: a match
1394    /// over a non-`#[non_exhaustive]` enum that omits a variant, with no
1395    /// wildcard arm to silently absorb it, fails to compile rather than
1396    /// passing an incomplete classification.
1397    ///
1398    /// ```compile_fail
1399    /// enum AuditProducer {
1400    ///     GateDenied,
1401    ///     DispatchSucceeded,
1402    ///     DispatchFailed,
1403    ///     UnknownVerb,
1404    ///     GitDigestReceipt,
1405    ///     ConfigLocked,
1406    ///     RecallExecuted,
1407    /// }
1408    ///
1409    /// fn describe(p: AuditProducer) -> &'static str {
1410    ///     match p {
1411    ///         AuditProducer::GateDenied => "obligation",
1412    ///         AuditProducer::DispatchSucceeded => "obligation",
1413    ///         AuditProducer::DispatchFailed => "obligation",
1414    ///         AuditProducer::UnknownVerb => "obligation",
1415    ///         AuditProducer::GitDigestReceipt => "obligation",
1416    ///         AuditProducer::ConfigLocked => "observability",
1417    ///         // RecallExecuted intentionally omitted: a non-exhaustive
1418    ///         // match must fail to compile, proving no variant can
1419    ///         // silently fall through an absent wildcard arm.
1420    ///     }
1421    /// }
1422    /// ```
1423    #[allow(dead_code)]
1424    struct DoctestAnchor;
1425}
1426
1427#[cfg(any(test, feature = "test-internals"))]
1428pub use test_internals::*;
1429
1430#[cfg(test)]
1431#[path = "audit_batch_physical_close_tests.rs"]
1432mod physical_close_tests;
1433
1434#[cfg(test)]
1435mod commit_reply_tests {
1436    use super::*;
1437    use khive_storage::event::IdempotentEventBatchResult;
1438    use khive_storage::{BatchWriteSummary, EventFilter, Page, PageRequest, StorageResult};
1439    use std::sync::atomic::AtomicUsize;
1440
1441    struct LostReplyStore {
1442        inner: Arc<dyn EventStore>,
1443        calls: AtomicUsize,
1444    }
1445
1446    #[async_trait::async_trait]
1447    impl EventStore for LostReplyStore {
1448        async fn append_event(&self, event: Event) -> StorageResult<()> {
1449            self.inner.append_event(event).await
1450        }
1451        async fn append_events(&self, events: Vec<Event>) -> StorageResult<BatchWriteSummary> {
1452            self.inner.append_events(events).await
1453        }
1454        async fn get_event(&self, id: uuid::Uuid) -> StorageResult<Option<Event>> {
1455            self.inner.get_event(id).await
1456        }
1457        async fn query_events(
1458            &self,
1459            filter: EventFilter,
1460            page: PageRequest,
1461        ) -> StorageResult<Page<Event>> {
1462            self.inner.query_events(filter, page).await
1463        }
1464        async fn count_events(&self, filter: EventFilter) -> StorageResult<u64> {
1465            self.inner.count_events(filter).await
1466        }
1467        fn preflight_event(&self, event: &Event) -> StorageResult<()> {
1468            self.inner.preflight_event(event)
1469        }
1470        fn supports_idempotent_audit_batch(&self) -> bool {
1471            true
1472        }
1473        async fn append_events_idempotent(
1474            &self,
1475            events: Vec<Event>,
1476        ) -> StorageResult<IdempotentEventBatchResult> {
1477            let result = self.inner.append_events_idempotent(events).await?;
1478            if self.calls.fetch_add(1, Ordering::SeqCst) == 0 {
1479                return Err(StorageError::WriteOutcomeUnknown {
1480                    source: Box::new(std::io::Error::other("reply lost after stored append")),
1481                });
1482            }
1483            Ok(result)
1484        }
1485    }
1486
1487    #[tokio::test]
1488    async fn lost_commit_reply_reconciles_identical_stored_audit_without_duplicate() {
1489        let backend = khive_db::StorageBackend::memory().expect("memory event store");
1490        let store = Arc::new(LostReplyStore {
1491            inner: backend.events().expect("event capability"),
1492            calls: AtomicUsize::new(0),
1493        });
1494        let event = Event::new(
1495            "local",
1496            "memory.recall",
1497            khive_types::EventKind::Audit,
1498            khive_types::SubstrateKind::Event,
1499            "test:lost-reply",
1500        );
1501        let result = run_generation(
1502            store.clone(),
1503            vec![event.clone()],
1504            Arc::new(AuditBatchConfig {
1505                max_commit_attempts: std::num::NonZeroU8::new(2).unwrap(),
1506                retry_backoff: Duration::ZERO,
1507                ..AuditBatchConfig::default()
1508            }),
1509        )
1510        .await;
1511        let GenerationResult::Committed(rows) = result else {
1512            panic!("idempotent audit replay must reconcile the lost COMMIT reply");
1513        };
1514        assert_eq!(rows, vec![EventAppendDisposition::AlreadyPresentIdentical]);
1515        assert_eq!(store.calls.load(Ordering::SeqCst), 2);
1516        assert_eq!(store.get_event(event.id).await.unwrap(), Some(event));
1517        assert_eq!(
1518            store.count_events(EventFilter::default()).await.unwrap(),
1519            1,
1520            "audit recovery must not duplicate the already stored row"
1521        );
1522    }
1523
1524    #[test]
1525    fn unavailable_owner_retries_pool_but_not_missing_runtime_or_route_violation() {
1526        let unavailable = |source| StorageError::WriterOwnerUnavailable {
1527            operation: "audit append".into(),
1528            source: Box::new(source),
1529        };
1530        assert!(matches!(
1531            classify_store_error(&unavailable(StorageError::Pool {
1532                operation: "owner".into(),
1533                message: "stopped".into(),
1534            })),
1535            RetryDecision::Retry
1536        ));
1537        for error in [
1538            unavailable(StorageError::WriterTaskNoRuntime),
1539            StorageError::WriterRouteViolation {
1540                operation: "audit append".into(),
1541            },
1542        ] {
1543            assert!(
1544                matches!(
1545                    classify_store_error(&error),
1546                    RetryDecision::Terminal(AuditTerminalReason::StoreFailure)
1547                ),
1548                "configuration/routing refusal is not an idempotent replay candidate"
1549            );
1550        }
1551    }
1552}