Skip to main content

khive_db/checkpoint/
task_loop.rs

1//! Periodic checkpoint work and pressure-episode transitions.
2
3use std::sync::atomic::Ordering;
4use std::sync::Arc;
5use std::time::Instant;
6
7use crate::pool::ConnectionPool;
8
9#[cfg(unix)]
10use super::{
11    complete_walpin_attribution, run_walpin_housekeeping_if_due, WalpinSidecarState,
12    DEFAULT_SESSION_SWEEP_INTERVAL,
13};
14use super::{
15    crossing_warn, log_tx_age_emission, log_tx_registry_oldest_debug, log_tx_registry_oldest_warn,
16    log_tx_registry_snapshot_warn, log_wal_high_water_warn, note_checkpoint_pressure_observation,
17    note_checkpoint_skipped, off_worker, CheckpointConfig, CheckpointConnection,
18    CheckpointLifecycleEmitter, CheckpointLifecycleOwner, CheckpointPressureEpisode,
19    CheckpointRunTaskGuard, CheckpointSeverityRung, CheckpointSeverityState, CheckpointTick,
20    ReaderCheckoutSweepState, TruncateState, TxAgeSweepState, CHECKPOINT_LIFECYCLE_ENQUEUE_DROPS,
21};
22
23/// Run one due FTS5 maintenance step off this task's Tokio worker thread.
24///
25/// A due step can issue up to `config.merge_pages` pages of synchronous
26/// SQLite incremental-merge I/O against a trigram index over a corpus of
27/// hundreds of thousands of rows — the same class of blocking work the
28/// WAL-pin beacon writes above already move off the worker via
29/// `tokio::task::spawn_blocking`. An ordinary write owner is constructed
30/// inside the blocking closure; the checkpoint file is never lent to FTS DML.
31/// Scheduler state returns on non-panicking paths. A panic surfaces as the
32/// `JoinError`, and the owner settles before releasing its lease.
33pub(super) async fn run_fts_maintenance_off_worker(
34    pool: Arc<ConnectionPool>,
35    config: crate::fts_maintenance::FtsMaintenanceConfig,
36    mut state: crate::fts_maintenance::FtsMaintenanceState,
37    now: Instant,
38) -> Result<
39    (
40        crate::fts_maintenance::FtsMaintenanceState,
41        Result<Option<crate::fts_maintenance::FtsMaintenanceStep>, String>,
42    ),
43    tokio::task::JoinError,
44> {
45    if pool.config().write_routing_strict && state.is_due(&config, now) {
46        let writer = match pool.writer_task_for_write(None, "fts_maintenance") {
47            Ok(Some(writer)) => writer,
48            Ok(None) => unreachable!("strict routing requires an owner"),
49            Err(error) => return Ok((state, Err(error.to_string()))),
50        };
51        // The caller retains the scheduler state even if admission drops the queued closure.
52        let shared_state = Arc::new(parking_lot::Mutex::new(Some(state)));
53        let queued_state = Arc::clone(&shared_state);
54        let admission = pool.write_admission();
55        let class = {
56            let _scope =
57                crate::disk_guard::lease_class_scope(crate::disk_guard::LeaseClass::Maintenance);
58            crate::disk_guard::WriteClassContext::capture()
59        };
60        let result = class
61            .scope_future(writer.send_top_level_bounded(move |conn| {
62                let result = crate::fts_maintenance::run_if_due_with_admission(
63                    conn,
64                    &config,
65                    queued_state
66                        .lock()
67                        .as_mut()
68                        .expect("scheduler state remains owned until reply"),
69                    now,
70                    || {
71                        admission
72                            .check()
73                            .map_err(|error| format!("FTS maintenance admission: {error}"))
74                    },
75                );
76                Ok(result)
77            }))
78            .await;
79        let state = shared_state
80            .lock()
81            .take()
82            .expect("caller retains scheduler state across admission and execution");
83        return Ok((state, result.unwrap_or_else(|error| Err(error.to_string()))));
84    }
85    tokio::task::spawn_blocking(move || {
86        let result = pool.run_fts_maintenance(&config, &mut state, now);
87        (state, result)
88    })
89    .await
90}
91
92/// Run the WAL checkpoint background task.
93///
94/// Long-running async task — spawn with `tokio::spawn`. Loops until
95/// `shutdown_rx` observes a change (or its sender is dropped). Callers MUST
96/// hold the paired `tokio::sync::watch::Sender` for the daemon's run scope
97/// and send on it to shut down — do NOT rely on `pool`'s `Arc` refcount
98/// reaching zero; a sibling owner (e.g. `event_store`) holding its own clone
99/// makes that check unreachable (issue #774).
100///
101/// Issues `PRAGMA wal_checkpoint(PASSIVE)` every tick on the task's dedicated
102/// `CheckpointConnection` — never the pool's writer mutex, so a concurrent
103/// `pool.writer()` checkout can never queue behind a checkpoint tick. That
104/// guarantee is admission-only: an armed TRUNCATE still takes SQLite's writer
105/// lock and can block new write transactions, on any connection, for up to
106/// `truncate_busy_timeout` (see `CheckpointConnection`'s contract). The
107/// checkpoint call itself runs on `spawn_blocking`, so that wait never holds
108/// one of the runtime's worker threads. A tick is
109/// `Skipped` when that connection is unavailable or SQLite returns a busy
110/// PASSIVE row without a usable pressure observation. A
111/// WARNING fires once per below→above threshold crossing, not every tick.
112///
113/// `lifecycle_owner` (ADR-094): exactly one task in a multi-backend fan-out
114/// should receive `Some`. That task appends a best-effort
115/// `CheckpointOutcomeRecorded` event on the elevation transition and one
116/// recovery summary when pressure falls back below `warn_pages`. Sustained
117/// elevated ticks aggregate in memory and in `db_diagnostics`; they never
118/// write one primary-store row per checkpoint attempt. `None` explicitly
119/// marks a non-owner. See `crates/khive-db/docs/api/checkpoint.md` for the
120/// full shutdown-mechanism and event-emission design history.
121///
122/// `is_main` (ADR-091 Amendment 3): whether `pool` is the deployment's main
123/// backend. A daemon owning several file-backed backends spawns one task per
124/// backend, each with its own pool and shutdown-channel clone (the sender
125/// broadcasts to every receiver clone alike). Lifecycle ownership is selected
126/// independently through `lifecycle_owner`; `is_main` only controls registry
127/// filtering. See the `tx_filter` construction below.
128pub async fn run_checkpoint_task(
129    pool: Arc<ConnectionPool>,
130    config: CheckpointConfig,
131    lifecycle_owner: Option<CheckpointLifecycleOwner>,
132    mut shutdown_rx: tokio::sync::watch::Receiver<()>,
133    is_main: bool,
134) {
135    let _checkpoint_run_guard = CheckpointRunTaskGuard::start(&pool, config.interval);
136    // This task IS the dedicated checkpoint owner: claim the pool so writer
137    // connections drop the bounded autocheckpoint fallback and routine
138    // checkpoint I/O stays off application commit paths. Pools without a
139    // running checkpoint task never claim and keep SQLite's bounded WAL
140    // reclamation. A failed claim leaves connections on the bounded fallback
141    // — safe, just not the low-latency posture — so it warns and continues.
142    match pool.claim_checkpoint_ownership() {
143        Ok(()) => {
144            if let Err(e) = pool.propagate_checkpoint_claim_to_writer_task().await {
145                tracing::warn!(target: "khive_db::checkpoint",
146                    error = %e,
147                    "checkpoint task could not reach the writer task's connection; it keeps the \
148                     bounded autocheckpoint fallback"
149                );
150            }
151        }
152        Err(e) => {
153            tracing::warn!(target: "khive_db::checkpoint",
154                error = %e,
155                "checkpoint task could not re-apply the ownership pragma on the pooled writer; \
156                 writer connections keep the bounded autocheckpoint fallback unless ownership is \
157                 claimed later"
158            );
159        }
160    }
161    let mut interval = tokio::time::interval(config.interval);
162    interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
163    let mut severity_state = CheckpointSeverityState::default();
164    let mut tx_age_state = TxAgeSweepState::default();
165    let mut reader_checkout_state = ReaderCheckoutSweepState::default();
166    let mut was_above_high_water = false;
167    #[cfg(unix)]
168    let legacy_walpin_fallback_interval = DEFAULT_SESSION_SWEEP_INTERVAL;
169    #[cfg(unix)]
170    let mut truncate_state =
171        TruncateState::with_legacy_walpin_fallback(legacy_walpin_fallback_interval);
172    #[cfg(not(unix))]
173    let mut truncate_state = TruncateState::default();
174    let mut lifecycle_emitter = CheckpointLifecycleEmitter::new(lifecycle_owner);
175    // Independent of `severity_state` (which owns the WARN ladder): this
176    // tracks the lifecycle sink's accepted elevation state. A full queue
177    // leaves it unchanged, so an opening or recovery transition is retried
178    // without admitting more than one primary-store write for that edge.
179    let mut event_elevation_open = false;
180    let mut pressure_episode: Option<CheckpointPressureEpisode> = None;
181    // A recovery row whose `try_emit` lost the race against a full queue.
182    // Retried on later ticks (before that tick's own transition handling)
183    // instead of leaving `pressure_episode` open for a stale episode to
184    // absorb the next, genuinely separate, pressure incident (#1857).
185    let mut pending_recovery: Option<khive_storage::CheckpointOutcomeRecordedPayload> = None;
186    let mut was_observed_above_warn = false;
187    // ADR-091 Amendment 3: this task's own backend-scoped view of the
188    // registry. `is_main` selects which `TxOriginFilter` variant applies —
189    // the caller passes `true` for exactly the one checkpoint task covering
190    // the deployment's main backend, so only that task also observes legacy
191    // `Unscoped` spans from any call site not yet threaded to an origin, the
192    // designed never-silently-drop fallback. A secondary backend's task
193    // never falls back to `Unscoped`: those spans belong to the main view or
194    // to no view, never to a database they were never registered against.
195    // `None` only when this pool's own origin isn't `Database` (an in-memory
196    // checkpoint pool) — degrades to "no open span observed" for the tick
197    // rather than panicking a long-running daemon loop on an
198    // assumed-impossible state.
199    let tx_filter = match pool.origin() {
200        khive_storage::tx_registry::TxOrigin::Database(id) => Some(if is_main {
201            khive_storage::tx_registry::TxOriginFilter::Main(id)
202        } else {
203            khive_storage::tx_registry::TxOriginFilter::Secondary(id)
204        }),
205        khive_storage::tx_registry::TxOrigin::Memory
206        | khive_storage::tx_registry::TxOrigin::Unscoped => None,
207    };
208    // ADR-091 Amendment 2 Plank B: the checkpoint pool is only ever wired for
209    // file-backed backends (`checkpoint_pool_for`), so `is_file_backed: true`
210    // is always correct here. `canonical_path()` (not `pool.config().path`)
211    // so the sidecar directory is keyed off the same minted identity every
212    // alias of this backend's configured path converges to.
213    #[cfg(unix)]
214    let mut walpin_state =
215        WalpinSidecarState::new(pool.canonical_path(), true, "daemon", config.interval);
216    #[cfg(unix)]
217    if let Some(sidecar) = walpin_state.as_mut() {
218        sidecar.register_beacon().await;
219    }
220
221    // Opened once here, at task startup; `ensure_open` is a no-op in steady
222    // state and only reopens after a connection-level failure or (for an
223    // in-memory/read-only pool) retries the open on every subsequent tick.
224    let mut checkpoint_conn = CheckpointConnection::new();
225    checkpoint_conn.ensure_open(&pool);
226    // FTS5 segment maintenance uses an admitted write owner at its own
227    // cadence. Each due call performs at most one bounded
228    // merge step on one table and refuses immediately when another writer
229    // owns SQLite's write lock.
230    let mut fts_maintenance_config = crate::fts_maintenance::FtsMaintenanceConfig::from_env();
231    // Secondary backends have independent schemas (for example the code-map
232    // database) and are not required to contain the substrate FTS tables.
233    // Exactly the main backend owns this derived-index maintenance.
234    fts_maintenance_config.enabled &= is_main;
235    let mut fts_maintenance_state =
236        crate::fts_maintenance::FtsMaintenanceState::new(Instant::now());
237
238    loop {
239        // A closed sender (the daemon returning without an explicit send)
240        // makes `changed()` resolve with `Err` immediately, which `select!`
241        // treats as ready — so shutdown is observed either way, not just on
242        // an explicit send.
243        tokio::select! {
244            _ = interval.tick() => {}
245            _ = shutdown_rx.changed() => break,
246        }
247
248        #[cfg(unix)]
249        truncate_state.begin_tick();
250
251        #[cfg(unix)]
252        let mut pending_sidecar_attribution = None;
253
254        let tick = if checkpoint_conn.ensure_open(&pool).is_none() {
255            note_checkpoint_skipped();
256            CheckpointTick::Skipped
257        } else {
258            // `ensure_open` above just confirmed a connection is open. Take
259            // ownership of it so the checkpoint cycle and the FTS maintenance
260            // step below can each move it onto a blocking thread; every path
261            // either restores it to `checkpoint_conn` or lets it drop, which is
262            // the moved-ownership equivalent of the former `drop_connection()`
263            // call.
264            let conn = checkpoint_conn
265                .conn
266                .take()
267                .expect("ensure_open just confirmed a connection is open");
268            match off_worker::run_checkpoint_core_off_worker(
269                Arc::clone(&pool),
270                conn,
271                config.clone(),
272                truncate_state,
273            )
274            .await
275            {
276                Ok((conn, state, Ok(outcome))) => {
277                    truncate_state = state;
278                    #[cfg(unix)]
279                    {
280                        pending_sidecar_attribution = outcome.sidecar_attribution;
281                    }
282                    #[cfg(not(unix))]
283                    let _ = outcome.sidecar_attribution;
284
285                    // Restore the recovery file before starting independent
286                    // FTS work. Only a due tick constructs an admitted writer
287                    // on a blocking thread; other ticks record a no-op.
288                    checkpoint_conn.conn = Some(conn);
289                    let fts_result = if !fts_maintenance_state
290                        .is_due(&fts_maintenance_config, Instant::now())
291                    {
292                        Ok(None)
293                    } else {
294                        match run_fts_maintenance_off_worker(
295                            Arc::clone(&pool),
296                            fts_maintenance_config.clone(),
297                            fts_maintenance_state,
298                            Instant::now(),
299                        )
300                        .await
301                        {
302                            Ok((state, fts_result)) => {
303                                fts_maintenance_state = state;
304                                fts_result
305                            }
306                            Err(join_err) => {
307                                // The step panicked on the blocking thread. The
308                                // scheduler state moved into it is gone; its
309                                // admitted writer has settled. The independent
310                                // checkpoint connection remains usable.
311                                // The checkpoint pragma itself already succeeded.
312                                fts_maintenance_state =
313                                    crate::fts_maintenance::FtsMaintenanceState::new(Instant::now());
314                                Err(format!(
315                                    "bounded FTS5 segment maintenance task panicked: {join_err}"
316                                ))
317                            }
318                        }
319                    };
320
321                    match fts_result {
322                        Ok(Some(step)) => match step.outcome {
323                            crate::fts_maintenance::FtsMaintenanceOutcome::Worked => {
324                                tracing::info!(target: "khive_db::checkpoint",
325                                    table = step.table,
326                                    requested_pages = step.requested_pages,
327                                    segments_before = step.segments_before,
328                                    segments_after = step.segments_after,
329                                    "bounded FTS5 segment maintenance made progress"
330                                );
331                            }
332                            crate::fts_maintenance::FtsMaintenanceOutcome::Busy => {
333                                tracing::debug!(target: "khive_db::checkpoint",
334                                    table = step.table,
335                                    requested_pages = step.requested_pages,
336                                    segments = step.segments_before,
337                                    "bounded FTS5 segment maintenance skipped a busy writer"
338                                );
339                            }
340                            crate::fts_maintenance::FtsMaintenanceOutcome::Noop
341                            | crate::fts_maintenance::FtsMaintenanceOutcome::BelowThreshold => {
342                                tracing::debug!(target: "khive_db::checkpoint",
343                                    table = step.table,
344                                    outcome = ?step.outcome,
345                                    segments = step.segments_before,
346                                    "bounded FTS5 segment maintenance had no work"
347                                );
348                            }
349                        },
350                        Ok(None) => {}
351                        Err(error) => {
352                            // The checkpoint pragma already succeeded. An FTS
353                            // structure/read/merge error is an independent,
354                            // best-effort maintenance failure and must not make
355                            // the task discard an otherwise healthy connection.
356                            tracing::warn!(target: "khive_db::checkpoint",
357                                error = %error,
358                                "bounded FTS5 segment maintenance failed"
359                            );
360                        }
361                    }
362                    match outcome.wal_pages {
363                        Some(wal_pages) => CheckpointTick::Observed(wal_pages),
364                        None => {
365                            note_checkpoint_skipped();
366                            CheckpointTick::Skipped
367                        }
368                    }
369                }
370                Ok((_conn, state, Err(e))) => {
371                    truncate_state = state;
372                    tracing::warn!(target: "khive_db::checkpoint",
373                        error = %e,
374                        "dedicated checkpoint connection failed a pragma; \
375                         dropping it for a fresh reopen next tick"
376                    );
377                    note_checkpoint_skipped();
378                    CheckpointTick::Skipped
379                }
380                Err(panicked) => {
381                    // The cycle panicked: its connection is gone (reopened next tick) and the
382                    // escalation state is the one it was handed, counted as a TRUNCATE attempt.
383                    truncate_state = panicked.truncate_state;
384                    tracing::warn!(target: "khive_db::checkpoint",
385                        error = %panicked.join_error,
386                        "WAL checkpoint cycle panicked on its blocking thread; \
387                         dropping the connection for a fresh reopen next tick"
388                    );
389                    note_checkpoint_skipped();
390                    CheckpointTick::Skipped
391                }
392            }
393        };
394
395        // A no-progress or unmeasured TRUNCATE returns a bounded attribution
396        // request alongside the core outcome. Consume it before any
397        // report-derived decision or ordinary housekeeping for this tick.
398        // The await is intentional: enumeration may perform up to 512
399        // filesystem reads/classifications, so none of that work is allowed
400        // to run on this Tokio worker, while one-pass-per-tick ordering still
401        // requires the result (or an honest worker/enumeration failure) before
402        // the fallback housekeeping arm is considered.
403        #[cfg(unix)]
404        if let Err(error) =
405            complete_walpin_attribution(pending_sidecar_attribution, &mut truncate_state).await
406        {
407            tracing::warn!(target: "khive_db::checkpoint",
408                error = %error,
409                failure_kind = error.kind(),
410                "ADR-091 Amendment 2 Plank B: no-progress sidecar attribution failed"
411            );
412        }
413
414        // ADR-091 Plank 1: age-based sweep over the registry's oldest entry
415        // MUST run on every tick, including a Skipped one — deliberately
416        // BEFORE the Skipped early-continue below. Since the dedicated
417        // checkpoint connection amendment, a `Skipped` tick means that
418        // connection was unavailable or its PASSIVE result was busy, not that
419        // some registered span held the pool's writer mutex (a checkpoint
420        // tick no longer touches it at all) — but the sweep must not go blind for the
421        // duration of that outage, and the two failure surfaces are
422        // independent: a registry span can go stale
423        // (KHIVE_TX_WARN_SECS / KHIVE_TX_MAX_AGE_SECS) while wal_pages sits
424        // well under warn_pages, or while the checkpoint connection itself is
425        // down. Edge-triggered per rung, same debounce idiom as the severity
426        // ladder below, so a sustained stale span logs once per rung rather
427        // than once per tick.
428        reader_checkout_state.sweep_at(&pool, Instant::now());
429        let oldest_tx = tx_filter
430            .as_ref()
431            .and_then(khive_storage::tx_registry::oldest_for);
432        for emission in tx_age_state.observe(
433            oldest_tx.as_ref().map(|s| (s.id, s.age, s.label.clone())),
434            config.tx_warn_secs,
435            config.tx_max_age_secs,
436        ) {
437            log_tx_age_emission(&emission);
438        }
439        // ADR-091 Amendment 2 Plank B: refresh (or clear) this daemon
440        // process's own walpin heartbeat on the same cadence, so its own
441        // pin — if any — is attributable the same way a session's is.
442        #[cfg(unix)]
443        if let Some(sidecar) = walpin_state.as_mut() {
444            sidecar
445                .observe(oldest_tx.clone(), config.tx_warn_secs)
446                .await;
447            let _ = run_walpin_housekeeping_if_due(
448                sidecar,
449                &mut truncate_state,
450                legacy_walpin_fallback_interval,
451            )
452            .await;
453        }
454
455        // Skipped ticks leave crossing state unchanged — a busy tick must not
456        // re-arm the rate limit while WAL pressure is still elevated.
457        let wal_pages = match tick {
458            CheckpointTick::Skipped => continue,
459            CheckpointTick::Observed(n) => n,
460        };
461
462        let above_warn = wal_pages >= config.warn_pages;
463        let above_high_water = wal_pages >= config.high_water_pages;
464        let above_truncate_high_water = wal_pages >= config.truncate_high_water_pages;
465        note_checkpoint_pressure_observation(above_warn, was_observed_above_warn);
466        was_observed_above_warn = above_warn;
467
468        // Per-tick debug for the oldest open entry always fires (cheap —
469        // reuses this tick's already-computed `oldest_tx`); the two
470        // `warn!`-level registry logs below are gated on the SAME crossing
471        // state as the WAL-threshold WARNs above, so sustained pressure
472        // logs once per crossing, not once per tick.
473        log_tx_registry_oldest_debug(wal_pages, oldest_tx.as_ref());
474
475        // ADR-091 severity ladder: INFO on the first below→above crossing,
476        // WARN once `warn_sustained_cycles` consecutive ticks stay elevated.
477        // The oldest-entry registry WARN rides the same INFO edge the old
478        // binary crossing_warn used to gate on.
479        for emission in severity_state.observe_wal_pages(wal_pages, &config) {
480            match emission.rung {
481                CheckpointSeverityRung::Info => {
482                    log_tx_registry_oldest_warn(wal_pages, oldest_tx.as_ref());
483                    tracing::info!(target: "khive_db::checkpoint",
484                        wal_pages = emission.wal_pages,
485                        warn_threshold = emission.threshold_pages,
486                        "WAL page count crossed warn threshold"
487                    );
488                }
489                CheckpointSeverityRung::Warn => {
490                    tracing::warn!(target: "khive_db::checkpoint",
491                        wal_pages = emission.wal_pages,
492                        warn_threshold = emission.threshold_pages,
493                        consecutive_cycles = emission.consecutive_cycles,
494                        "WAL page count failed to drain below warn threshold"
495                    );
496                }
497                CheckpointSeverityRung::Alarm => {
498                    // Never produced by `observe_wal_pages`; see its doc.
499                }
500            }
501        }
502
503        let high_water_crossed = crossing_warn(above_high_water, &mut was_above_high_water);
504        if high_water_crossed {
505            log_tx_registry_snapshot_warn(wal_pages);
506            log_wal_high_water_warn(
507                wal_pages,
508                config.high_water_pages,
509                oldest_tx.as_ref(),
510                config.tx_warn_secs,
511            );
512        }
513
514        // ADR-094/#1838, #1857: one elevation row and one recovery summary
515        // per genuinely continuous episode. Sustained elevated ticks update
516        // only the bounded in-memory aggregate and process diagnostics
517        // above; a dropped recovery handoff must not fold the next,
518        // separate, pressure incident into this episode's aggregate.
519        observe_checkpoint_pressure_tick(
520            above_warn,
521            wal_pages,
522            above_high_water,
523            above_truncate_high_water,
524            &config,
525            &mut event_elevation_open,
526            &mut pressure_episode,
527            &mut pending_recovery,
528            |payload| {
529                lifecycle_emitter
530                    .try_emit(khive_types::EventKind::CheckpointOutcomeRecorded, payload)
531            },
532        );
533    }
534
535    lifecycle_emitter.shutdown().await;
536
537    #[cfg(unix)]
538    if let Some(sidecar) = walpin_state.as_mut() {
539        sidecar.shutdown().await;
540    }
541}
542
543/// Whether a `CheckpointOutcomeRecorded` transition should be enqueued for
544/// this tick. Repeated observations in either state aggregate in memory;
545/// only elevation and recovery edges reach the primary store.
546pub(super) fn checkpoint_outcome_should_emit(above_warn: bool, was_elevated: bool) -> bool {
547    above_warn != was_elevated
548}
549
550/// Advance the pressure-episode/lifecycle-emission state machine for one
551/// observed tick. `try_emit` mirrors [`CheckpointLifecycleEmitter::try_emit`]
552/// — `true` means the row was handed off, `false` means the queue was full
553/// or closed.
554///
555/// #1857: on a dropped recovery handoff (`try_emit` returns `false` while
556/// `above_warn` is `false`), the closed episode's summary is stashed in
557/// `pending_recovery` for retry on later ticks — flushed here before this
558/// tick's own transition is evaluated — instead of leaving
559/// `event_elevation_open` and `pressure_episode` open for the next elevated
560/// tick to silently extend, which would report two separate pressure
561/// incidents as one merged episode.
562#[allow(clippy::too_many_arguments)]
563pub(super) fn observe_checkpoint_pressure_tick(
564    above_warn: bool,
565    wal_pages: u64,
566    above_high_water: bool,
567    above_truncate_high_water: bool,
568    config: &CheckpointConfig,
569    event_elevation_open: &mut bool,
570    pressure_episode: &mut Option<CheckpointPressureEpisode>,
571    pending_recovery: &mut Option<khive_storage::CheckpointOutcomeRecordedPayload>,
572    mut try_emit: impl FnMut(khive_storage::CheckpointOutcomeRecordedPayload) -> bool,
573) {
574    // An undelivered recovery summary is a BARRIER, not merely a retry:
575    // lifecycle consumers assert on the ordered event history (ADR-094), so
576    // a later episode's opening must never be appended ahead of an earlier
577    // episode's recovery. If the retry fails, the in-memory aggregate still
578    // advances below, but no other emission is attempted this tick — a
579    // deferred opening or recovery re-derives from state on a later tick,
580    // after the pending summary has been delivered in order.
581    let pending_blocks_emission = if let Some(payload) = pending_recovery.clone() {
582        if try_emit(payload) {
583            *pending_recovery = None;
584            false
585        } else {
586            true
587        }
588    } else {
589        false
590    };
591
592    if above_warn {
593        match pressure_episode.as_mut() {
594            Some(episode) => episode.observe(wal_pages),
595            None => *pressure_episode = Some(CheckpointPressureEpisode::start(wal_pages)),
596        }
597    } else if !*event_elevation_open {
598        // No elevation row reached the bounded handoff, so from any
599        // consumer's view this episode never opened; discarding it keeps
600        // the delivered history self-consistent. When the discard happens
601        // because the barrier suppressed the opening attempt entirely, the
602        // loss would otherwise be invisible even to the drop counters that
603        // record failed attempts, so it is counted and logged here.
604        if pending_blocks_emission && pressure_episode.is_some() {
605            CHECKPOINT_LIFECYCLE_ENQUEUE_DROPS.fetch_add(1, Ordering::Relaxed);
606            tracing::warn!(target: "khive_db::checkpoint",
607                wal_pages,
608                "checkpoint pressure episode elapsed unreported behind an undelivered recovery summary"
609            );
610        }
611        *pressure_episode = None;
612    }
613
614    if pending_blocks_emission || !checkpoint_outcome_should_emit(above_warn, *event_elevation_open)
615    {
616        return;
617    }
618    let Some(episode) = *pressure_episode else {
619        tracing::warn!(target: "khive_db::checkpoint",
620            above_warn,
621            event_elevation_open = *event_elevation_open,
622            "checkpoint pressure transition has no episode aggregate"
623        );
624        return;
625    };
626    let payload = khive_storage::CheckpointOutcomeRecordedPayload {
627        wal_pages,
628        warn_pages: config.warn_pages,
629        high_water_pages: config.high_water_pages,
630        truncate_high_water_pages: config.truncate_high_water_pages,
631        above_warn,
632        above_high_water,
633        above_truncate_high_water,
634        episode_elevated_ticks: Some(episode.elevated_ticks),
635        episode_peak_wal_pages: Some(episode.peak_wal_pages),
636    };
637    if try_emit(payload.clone()) {
638        *event_elevation_open = above_warn;
639        if !above_warn {
640            *pressure_episode = None;
641        }
642    } else if !above_warn {
643        // The recovery handoff was dropped. Close this episode locally
644        // anyway — `event_elevation_open` MUST NOT stay true, or the next
645        // elevated tick would extend this (already finished) episode's
646        // aggregate instead of starting a fresh one for what is genuinely a
647        // new pressure incident. The dropped summary itself isn't thrown
648        // away: it is stashed in `pending_recovery` and delivered on a
649        // later tick, ahead of (and as a barrier to) every subsequent
650        // emission, so lifecycle ordering survives the retry. The slot is
651        // structurally empty here: a tick that entered with an undelivered
652        // summary returned at the barrier above and never reached this arm.
653        debug_assert!(
654            pending_recovery.is_none(),
655            "recovery emission attempted while an earlier summary was still pending"
656        );
657        *event_elevation_open = false;
658        *pressure_episode = None;
659        *pending_recovery = Some(payload);
660    }
661}
662
663#[cfg(test)]
664mod routing_tests {
665    use super::*;
666    use crate::pool::PoolConfig;
667    use std::time::Duration;
668
669    #[tokio::test]
670    async fn strict_fts_capacity_refusal_preserves_unstarted_scheduler_state() {
671        let dir = tempfile::tempdir().unwrap();
672        let path = dir.path().join("fts-capacity.db");
673        let conn = rusqlite::Connection::open(&path).unwrap();
674        conn.execute_batch("CREATE VIRTUAL TABLE fts_entities USING fts5(value); CREATE VIRTUAL TABLE fts_notes USING fts5(value)").unwrap();
675        drop(conn);
676        let mut pool = ConnectionPool::new(PoolConfig {
677            path: Some(path),
678            write_routing_strict: true,
679            write_queue_enabled: Some(true),
680            ..PoolConfig::for_test()
681        })
682        .unwrap();
683        let available = Arc::new(std::sync::atomic::AtomicU64::new(100));
684        let sampled = Arc::clone(&available);
685        pool.set_test_write_admission(100, move |_| Ok(sampled.load(Ordering::SeqCst)));
686        let pool = Arc::new(pool);
687        let initial = Instant::now();
688        let due = initial + Duration::from_secs(1);
689        let config = crate::fts_maintenance::FtsMaintenanceConfig {
690            enabled: true,
691            interval: Duration::from_secs(1),
692            merge_pages: 1,
693            minimum_segments: 0,
694        };
695        let state = crate::fts_maintenance::FtsMaintenanceState::new(initial);
696        let (state, result) =
697            run_fts_maintenance_off_worker(Arc::clone(&pool), config.clone(), state, due)
698                .await
699                .unwrap();
700        assert!(
701            result.is_err(),
702            "actual capacity admission must refuse before FTS execution"
703        );
704        assert!(
705            state.is_due(&config, due),
706            "pre-execution refusal must not consume the due step"
707        );
708        available.store(101, Ordering::SeqCst);
709        let (_, result) = run_fts_maintenance_off_worker(pool, config, state, due)
710            .await
711            .unwrap();
712        let step = result
713            .unwrap()
714            .expect("the same state must recover its due work");
715        assert_eq!(
716            step.table, "fts_entities",
717            "refusal must not advance the next-table cursor"
718        );
719        assert_ne!(step.requested_pages, 0);
720    }
721
722    #[tokio::test]
723    async fn fts_strict_queue_preserves_state_and_default_maintenance() {
724        for (strict, enabled) in [(true, true), (true, false), (false, false)] {
725            let dir = tempfile::tempdir().unwrap();
726            let path = dir.path().join("fts-routing.db");
727            let conn = rusqlite::Connection::open(&path).unwrap();
728            conn.execute_batch("CREATE VIRTUAL TABLE fts_entities USING fts5(value); CREATE VIRTUAL TABLE fts_notes USING fts5(value)").unwrap();
729            drop(conn);
730            let pool = Arc::new(
731                ConnectionPool::new(PoolConfig {
732                    path: Some(path),
733                    write_routing_strict: strict,
734                    write_queue_enabled: Some(enabled),
735                    ..PoolConfig::for_test()
736                })
737                .unwrap(),
738            );
739            let now = Instant::now();
740            let config = crate::fts_maintenance::FtsMaintenanceConfig {
741                enabled: true,
742                interval: Duration::ZERO,
743                merge_pages: 1,
744                minimum_segments: 0,
745            };
746            let state = crate::fts_maintenance::FtsMaintenanceState::new(now);
747            let (state, result) =
748                run_fts_maintenance_off_worker(Arc::clone(&pool), config.clone(), state, now)
749                    .await
750                    .unwrap();
751            if strict && !enabled {
752                assert!(
753                    result.is_err(),
754                    "strict FTS work must refuse a missing writer owner"
755                );
756                assert!(
757                    state.is_due(&config, now),
758                    "queue refusal must preserve scheduler state"
759                );
760            } else {
761                assert!(
762                    result.is_ok(),
763                    "queued strict and default FTS work must both complete: {result:?}"
764                );
765                let step = result.unwrap().expect("due FTS work has a step");
766                assert_eq!(step.table, "fts_entities");
767                assert_ne!(
768                    step.requested_pages, 0,
769                    "fixture must execute the logical FTS merge path"
770                );
771            }
772            let raw = pool.writer().unwrap().autocommit(|conn| {
773                conn.execute_batch("CREATE TABLE unexpected(value)")?;
774                Ok(())
775            });
776            assert_eq!(
777                raw.is_err(),
778                strict,
779                "FTS scheduling must not grant raw-write authority"
780            );
781        }
782    }
783}