Skip to main content

khive_db/
backend.rs

1//! Concrete storage backend providing capability traits.
2//!
3//! `StorageBackend` owns a `ConnectionPool` and provides factory methods for all
4//! ten capability traits (`SqlAccess`, `NoteStore`, `EntityStore`, `GraphStore`,
5//! `EventStore`, `VectorStore`, `SparseStore`, `TextSearch`, `BlobStore`, and
6//! `AttachmentStore`). File-backed for production; in-memory for tests.
7
8use std::path::Path;
9use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
10use std::sync::Arc;
11
12use rusqlite::OptionalExtension;
13
14use crate::database_owner_identity::{DatabaseOwnerIdentity, DatabaseOwnerIdentityError};
15use crate::error::SqliteError;
16use crate::pool::{ConnectionPool, PoolConfig, WalCeilingPolicy};
17use crate::stores::{agents, attachment, blob, entity, event, graph, note, sparse, text, vectors};
18
19mod capability_factories;
20mod code_map;
21mod core_stores;
22#[path = "backend/schema_readiness.rs"]
23mod memory_visibility;
24mod pack_schema;
25mod policy_open;
26mod schema_barrier;
27pub use schema_barrier::ForcedIndexCheck;
28
29#[cfg(test)]
30#[path = "backend/filter_builder_tests.rs"]
31mod filter_builder_tests;
32
33mod namespace_listing;
34pub use namespace_listing::NamespaceLiveness;
35
36#[cfg(test)]
37#[path = "backend/memory_visibility_tests.rs"]
38mod memory_visibility_tests;
39
40#[cfg(any(unix, windows))]
41mod claimed_file_identity;
42
43fn sqlite_table_exists(conn: &rusqlite::Connection, table: &str) -> Result<bool, SqliteError> {
44    conn.query_row(
45        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1",
46        rusqlite::params![table],
47        |row| row.get::<_, i64>(0),
48    )
49    .optional()
50    .map(|row| row.is_some())
51    .map_err(SqliteError::Rusqlite)
52}
53
54/// Populate `table`'s [`text::rowid_map_table`] from `table` itself, and
55/// record completion in [`text::rowid_map_state_table`], the first time this
56/// backend opens a database that predates the map.
57///
58/// `StorageBackend::text()`/`text_with_tokenizer()` is called uncached on
59/// essentially every text-store access (`khive-runtime` builds a fresh
60/// `Fts5TextSearch` per call, never caching the `Arc`). The backend skips this
61/// check once a table's marker is durable, and the marker is written on the
62/// first check even for a table with no rows, so a request after that never
63/// takes the writer for it. The completion probe below is an index probe
64/// (`SELECT EXISTS(... LIMIT 1)`) that stops at the first row.
65///
66/// Completion is read from a durable marker row rather than inferred from
67/// the map's own row count: a map can legitimately be empty for a table with
68/// no rows yet, which is indistinguishable from "never backfilled" by row
69/// count alone, and every runtime write path (`text.rs`'s
70/// `delete_document_dml`, `upsert_document_dml`, `batch_upsert_documents_dml`,
71/// and the raw SQL in `khive-runtime`'s
72/// `atomic_prepare`/`atomic_message`/`curation`) maintains the FTS row and
73/// its map row atomically, inside one transaction, so a legitimately
74/// half-empty map from a live write path never happens either. The marker is
75/// written for an empty table too (ADR-154 Amendment 3: no schema work on a
76/// request path after readiness), so rows that later reach the table through
77/// anything other than the maintained write paths (a raw-SQL seed) must
78/// maintain the map themselves; a restored or imported store whose marker is
79/// absent is still reconciled the next time this runs. The marker asserts a
80/// bijection — every live row has exactly one map row pointing at it, and
81/// every map row points at a live row with the same key — so reconciliation
82/// runs in three steps before the marker is written, all inside one
83/// transaction; on an empty table the first step removes any leftover map
84/// rows and the other two do nothing:
85///
86/// 1. Any existing map row that no longer has a matching live FTS row at the
87///    same rowid AND the same `(namespace, subject_id)` is removed first. A
88///    map row can otherwise survive with the wrong key after FTS5 reuses its
89///    rowid for a different document (the crash window
90///    `delete_document_dml` guards against at the single-delete level; this
91///    is the same class of staleness surviving into a legacy/reconciliation
92///    pass instead).
93/// 2. The map is (re)built from every current FTS row (`INSERT OR REPLACE`),
94///    which reconciles a partially populated map rather than only filling a
95///    wholly empty one. `updated_at ASC, rowid ASC` matches migration 024's
96///    own backfill ordering: for any legacy duplicate `(namespace,
97///    subject_id)` pair, the row with the newest `updated_at` survives
98///    `INSERT OR REPLACE`, breaking a tie toward the higher rowid.
99/// 3. Any non-NULL-key FTS row that lost step 2's survivor race — a
100///    duplicate whose rowid the map no longer points at — is deleted,
101///    mirroring migration 024's own first sweep, so no live row is left
102///    without a map entry.
103///
104/// The marker is then written in the same transaction, and is durable when
105/// this returns `Ok`. This function never
106/// checks `entities`/`notes` for orphaned subjects — that sweep is specific
107/// to those two backing tables and stays in migration 024's SQL; a generic
108/// `table_key` here has no fixed backing table to check against.
109fn ensure_fts_rowid_map_backfilled(
110    conn: &rusqlite::Connection,
111    table: &str,
112    admission: &crate::pool::WriteAdmission,
113) -> Result<(), SqliteError> {
114    let map = text::rowid_map_table(table);
115    let state = text::rowid_map_state_table(table);
116    let already_backfilled: bool = conn.query_row(
117        &format!("SELECT EXISTS(SELECT 1 FROM {state} WHERE key = 'backfill' AND value = ?1)"),
118        rusqlite::params![text::ROWID_MAP_BACKFILL_COMPLETE],
119        |row| row.get(0),
120    )?;
121    if already_backfilled {
122        return Ok(());
123    }
124
125    conn.execute_batch("BEGIN IMMEDIATE")?;
126    if let Err(error) = admission.check() {
127        return Err(crate::migrations::capacity_refusal_after_rollback(
128            conn,
129            conn.execute_batch("ROLLBACK"),
130            error,
131            "FTS rowid-map backfill",
132        ));
133    }
134    let result: Result<(), SqliteError> = (|| {
135        conn.execute_batch(&format!(
136            "DELETE FROM {map} WHERE NOT EXISTS ( \
137                 SELECT 1 FROM {table} \
138                 WHERE {table}.rowid = {map}.rowid \
139                   AND {table}.namespace = {map}.namespace \
140                   AND {table}.subject_id = {map}.subject_id \
141             )"
142        ))?;
143        conn.execute_batch(&format!(
144            "INSERT OR REPLACE INTO {map} (namespace, subject_id, rowid) \
145             SELECT namespace, subject_id, rowid FROM {table} \
146             WHERE namespace IS NOT NULL AND subject_id IS NOT NULL \
147             ORDER BY updated_at ASC, rowid ASC"
148        ))?;
149        conn.execute_batch(&format!(
150            "DELETE FROM {table} \
151             WHERE namespace IS NOT NULL AND subject_id IS NOT NULL \
152               AND rowid NOT IN (SELECT rowid FROM {map})"
153        ))?;
154        conn.execute(
155            &format!("INSERT OR REPLACE INTO {state} (key, value) VALUES ('backfill', ?1)"),
156            rusqlite::params![text::ROWID_MAP_BACKFILL_COMPLETE],
157        )?;
158        Ok(())
159    })();
160
161    match result {
162        Ok(()) => {
163            conn.execute_batch("COMMIT")?;
164            Ok(())
165        }
166        Err(e) => {
167            let _ = conn.execute_batch("ROLLBACK");
168            Err(e)
169        }
170    }
171}
172
173/// Emit exactly one `tracing::warn!` for the whole process the first time
174/// any table falls back to scan-fallback mode, rather than once per `text()`
175/// call — `StorageBackend::text()` is called fresh on essentially every
176/// access (see `ensure_fts_rowid_map_backfilled`'s doc comment), so an
177/// unconditional warning here would spam the log on a hot path.
178fn warn_scan_fallback_once(table: &str) {
179    static WARNED: std::sync::Once = std::sync::Once::new();
180    WARNED.call_once(|| {
181        tracing::warn!(
182            table,
183            "opened a read-only text-search table with no rowid-map sidecar, or with a sidecar \
184             that has never proven a completed backfill (no durable completion marker); a \
185             read-only connection cannot create, backfill, or reconcile the map itself, so this \
186             falls back to pre-map namespace/subject_id scan predicates for get/delete on this \
187             table rather than trusting a map that might be partial"
188        );
189    });
190}
191
192fn validate_vector_model_key(model_key: &str) -> Result<(), SqliteError> {
193    if model_key.is_empty()
194        || !model_key
195            .chars()
196            .all(|c| c.is_ascii_alphanumeric() || c == '_')
197    {
198        return Err(SqliteError::InvalidData(format!(
199            "invalid model_key '{}': must be non-empty and contain only \
200             alphanumeric/underscore characters",
201            model_key
202        )));
203    }
204    Ok(())
205}
206
207fn validate_vector_table_columns(
208    conn: &rusqlite::Connection,
209    table: &str,
210) -> Result<(), SqliteError> {
211    let pragma = format!("PRAGMA table_xinfo({table})");
212    let mut stmt = conn.prepare(&pragma)?;
213    let mut rows = stmt.query([])?;
214    let mut has_field = false;
215    let mut has_embedding_model = false;
216    while let Some(row) = rows.next()? {
217        let name: String = row.get(1)?;
218        if name == "field" {
219            has_field = true;
220        }
221        if name == "embedding_model" {
222            has_embedding_model = true;
223        }
224    }
225    if !has_field || !has_embedding_model {
226        return Err(SqliteError::InvalidData(format!(
227            "vec0 table '{table}' is missing required column(s) (field={has_field}, \
228             embedding_model={has_embedding_model}); this is a pre-v0.2.8 vector schema and is \
229             not supported — recreate the database"
230        )));
231    }
232    Ok(())
233}
234
235#[derive(Clone, Copy, Debug, PartialEq, Eq)]
236pub(crate) enum StoreSchemaKind {
237    Entities,
238    Graph,
239    Notes,
240    Events,
241    Agents,
242}
243
244impl StoreSchemaKind {
245    pub(crate) fn initializer(self) -> fn(&rusqlite::Connection) -> Result<(), rusqlite::Error> {
246        match self {
247            Self::Entities => entity::ensure_entities_schema,
248            Self::Graph => graph::ensure_graph_schema,
249            Self::Notes => note::ensure_notes_schema,
250            Self::Events => event::ensure_events_schema,
251            Self::Agents => agents::ensure_agents_schema,
252        }
253    }
254}
255
256#[derive(Default)]
257pub(crate) struct StoreSchemaGate {
258    pub(crate) ready: AtomicBool,
259    #[cfg(test)]
260    pub(crate) attempts: AtomicUsize,
261}
262
263impl StoreSchemaGate {
264    pub(crate) fn ensure(
265        &self,
266        conn: &rusqlite::Connection,
267        ensure: fn(&rusqlite::Connection) -> Result<(), rusqlite::Error>,
268    ) -> Result<(), rusqlite::Error> {
269        // A concurrent initializer or repair may have completed while this
270        // caller waited for the writer. Publish readiness only after success.
271        if self.ready.load(Ordering::Acquire) {
272            return Ok(());
273        }
274        #[cfg(test)]
275        self.attempts.fetch_add(1, Ordering::Relaxed);
276        ensure(conn)?;
277        self.ready.store(true, Ordering::Release);
278        Ok(())
279    }
280}
281
282/// Concrete storage backend providing capability traits.
283///
284/// Capability schemas are initialized once per backend. An index removed by
285/// another process is repaired on the narrowly supported forced-index read paths. Other
286/// external schema changes may still require reopening the backend.
287pub struct StorageBackend {
288    pool: Arc<ConnectionPool>,
289    is_file_backed: bool,
290    path: Option<std::path::PathBuf>,
291    /// Vector model keys whose `vec_<model_key>` table this backend has already
292    /// validated or created. A key is recorded only after the whole check
293    /// succeeded under the writer, so a failed check is retried by the next
294    /// fetch and a warm fetch never takes the writer. A table another process
295    /// creates for a model this set does not hold is still validated before
296    /// use; one that is replaced after its model was recorded is not
297    /// re-checked until the backend reopens.
298    vector_tables_ready: parking_lot::Mutex<std::collections::HashSet<String>>,
299    /// FTS tables whose schema, rowid-map sidecar and durable backfill marker
300    /// this backend has already proven under the writer. A table is recorded
301    /// once its marker exists, which the first check writes even for an empty
302    /// table, so a warm fetch never takes the writer or the volume lease: a
303    /// search opens its text store on every call. As with
304    /// `vector_tables_ready`, a table dropped behind a live backend is not
305    /// recreated until the backend is reopened.
306    text_tables_ready: parking_lot::Mutex<std::collections::HashSet<String>>,
307    /// Sparse model keys whose `sparse_<model_key>` table and namespace index
308    /// this backend has already ensured under the writer. Recorded only after
309    /// the ensure succeeded, so a failed ensure is retried, and a warm fetch
310    /// takes no write unit. As with `vector_tables_ready`, a table dropped
311    /// behind a live backend is not recreated until the backend is reopened.
312    sparse_tables_ready: parking_lot::Mutex<std::collections::HashSet<String>>,
313    /// How many times the lazy `notes_seq` anti-join repair has actually
314    /// executed against this backend's pool. Gates `notes_for_namespace` so
315    /// the repair (a full `notes` scan) runs at most once per backend for
316    /// the process's lifetime instead of on every store acquisition (khive
317    /// #827). Also exposed via
318    /// `notes_seq_repair_run_count` for regression tests.
319    notes_seq_repair_runs: AtomicUsize,
320    store_schemas: [Arc<StoreSchemaGate>; 5],
321}
322
323impl StorageBackend {
324    /// File-backed SQLite database.
325    ///
326    /// Opens (or creates) the database at `path`. An existing filesystem path
327    /// whose mode is read-only is opened with the same locked-down pool
328    /// configuration as [`Self::sqlite_read_only`]. The writable pool provides
329    /// 1 writer + N readers in WAL mode for concurrent access.
330    /// No service schema is applied — call `apply_schema()` for each service.
331    /// The pool may create its internal `_khive_database_identity` singleton
332    /// table on a writable open before service migrations run.
333    ///
334    /// The volume-lock directory is the per-user default of
335    /// [`crate::default_volume_lock_dir`], shared by every process of the user;
336    /// [`Self::sqlite_with_volume_lock_dir`] names one explicitly.
337    pub fn sqlite(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
338        Self::sqlite_with_pool_config(path, PoolConfig::default(), None)
339    }
340
341    /// A private test database with a small, explicitly sized reader pool.
342    #[cfg(any(test, feature = "test-support"))]
343    pub fn sqlite_for_test(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
344        Self::sqlite_with_pool_config(path, PoolConfig::for_test(), None)
345    }
346
347    /// Open a private test database with an explicit journal mode and busy wait.
348    #[cfg(any(test, feature = "test-support"))]
349    pub fn sqlite_for_test_with_journal_mode(
350        path: impl AsRef<Path>,
351        wal_mode: bool,
352        busy_timeout: std::time::Duration,
353    ) -> Result<Self, SqliteError> {
354        Self::sqlite_with_pool_config(
355            path,
356            PoolConfig {
357                wal_mode,
358                busy_timeout,
359                write_queue_enabled: Some(true),
360                ..PoolConfig::for_test()
361            },
362            None,
363        )
364    }
365
366    /// [`Self::sqlite_for_test_with_journal_mode`] with its volume-lock files in
367    /// `volume_lock_dir`. Under the test harness each directory takes its own
368    /// in-process lease slot, so a fixture that holds the lease across a pause
369    /// does not stall fixtures on other databases in the same test binary.
370    #[cfg(any(test, feature = "test-support"))]
371    pub fn sqlite_for_test_with_journal_mode_in(
372        path: impl AsRef<Path>,
373        wal_mode: bool,
374        busy_timeout: std::time::Duration,
375        volume_lock_dir: std::path::PathBuf,
376    ) -> Result<Self, SqliteError> {
377        Self::sqlite_with_pool_config(
378            path,
379            PoolConfig {
380                wal_mode,
381                busy_timeout,
382                write_queue_enabled: Some(true),
383                volume_lock_dir: Some(volume_lock_dir),
384                ..PoolConfig::for_test()
385            },
386            None,
387        )
388    }
389
390    /// Open SQLite with a reader count selected before any connections are opened.
391    /// `None` preserves the default pool size and filesystem read-only detection.
392    /// The volume-lock directory defaults as in [`Self::sqlite`].
393    pub fn sqlite_with_max_readers(
394        path: impl AsRef<Path>,
395        max_readers: Option<usize>,
396    ) -> Result<Self, SqliteError> {
397        Self::sqlite_with_pool_config(path, PoolConfig::default(), max_readers)
398    }
399
400    /// Open a file-backed SQLite backend with its already-resolved WAL ceiling.
401    /// A nonzero policy fails closed until the WAL I/O limiter is installed.
402    /// The volume-lock directory defaults as in [`Self::sqlite`].
403    pub fn sqlite_with_max_readers_and_wal_ceiling(
404        path: impl AsRef<Path>,
405        max_readers: Option<usize>,
406        wal_ceiling: WalCeilingPolicy,
407    ) -> Result<Self, SqliteError> {
408        Self::sqlite_with_pool_config(
409            path,
410            PoolConfig {
411                wal_ceiling,
412                ..PoolConfig::default()
413            },
414            max_readers,
415        )
416    }
417
418    pub(crate) fn sqlite_with_pool_config(
419        path: impl AsRef<Path>,
420        pool_config: PoolConfig,
421        max_readers: Option<usize>,
422    ) -> Result<Self, SqliteError> {
423        crate::extension::ensure_extensions_loaded();
424        let resolved = path.as_ref().to_path_buf();
425        let read_only =
426            std::fs::metadata(&resolved).is_ok_and(|metadata| metadata.permissions().readonly());
427        let mut config = PoolConfig {
428            path: Some(resolved.clone()),
429            read_only,
430            ..pool_config
431        };
432        if let Some(max_readers) = max_readers {
433            config.max_readers = max_readers;
434        }
435        if read_only {
436            config.write_queue_enabled = Some(false);
437        }
438        let pool = ConnectionPool::new(config)?;
439        Ok(Self {
440            pool: Arc::new(pool),
441            is_file_backed: true,
442            path: Some(resolved),
443            vector_tables_ready: Default::default(),
444            text_tables_ready: Default::default(),
445            sparse_tables_ready: Default::default(),
446            notes_seq_repair_runs: AtomicUsize::new(0),
447            store_schemas: std::array::from_fn(|_| Arc::new(StoreSchemaGate::default())),
448        })
449    }
450
451    /// File-backed SQLite database opened read-only.
452    ///
453    /// Opens the database at `path` and sets `PRAGMA query_only = ON` on the
454    /// writer connection so that any write attempt (INSERT/UPDATE/DELETE) returns
455    /// an error. Reader connections are opened with `SQLITE_OPEN_READ_ONLY` by the
456    /// pool; at least one remains dedicated even for a rollback-journal snapshot,
457    /// while this PRAGMA extends the protection to the otherwise-unused writer slot.
458    ///
459    /// The database file must already exist — unlike `sqlite()` this constructor
460    /// does not create a new file.
461    pub fn sqlite_read_only(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
462        Self::sqlite_read_only_with_pool_config(path, PoolConfig::default(), None)
463    }
464
465    /// A private read-only test database with a small reader pool.
466    #[cfg(any(test, feature = "test-support"))]
467    pub fn sqlite_read_only_for_test(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
468        Self::sqlite_read_only_with_pool_config(path, PoolConfig::for_test(), None)
469    }
470
471    /// Open a read-only SQLite store with a construction-time reader count.
472    pub fn sqlite_read_only_with_max_readers(
473        path: impl AsRef<Path>,
474        max_readers: Option<usize>,
475    ) -> Result<Self, SqliteError> {
476        Self::sqlite_read_only_with_pool_config(path, PoolConfig::default(), max_readers)
477    }
478
479    /// Open a read-only SQLite backend while retaining WAL ceiling
480    /// configuration for diagnostics. No writer policy is enforced.
481    pub fn sqlite_read_only_with_max_readers_and_wal_ceiling(
482        path: impl AsRef<Path>,
483        max_readers: Option<usize>,
484        wal_ceiling: WalCeilingPolicy,
485    ) -> Result<Self, SqliteError> {
486        Self::sqlite_read_only_with_pool_config(
487            path,
488            PoolConfig {
489                wal_ceiling,
490                ..PoolConfig::default()
491            },
492            max_readers,
493        )
494    }
495
496    /// Open an existing live database with read-only SQLite connections.
497    ///
498    /// Ordinary WAL locking and change detection remain enabled, so a writable
499    /// shared-memory sidecar is accepted. No schema preparation is performed.
500    pub fn sqlite_read_only_live_with_max_readers_and_wal_ceiling(
501        path: impl AsRef<Path>,
502        max_readers: Option<usize>,
503        wal_ceiling: WalCeilingPolicy,
504    ) -> Result<Self, SqliteError> {
505        Self::sqlite_read_only_with_pool_constructor(
506            path,
507            PoolConfig {
508                wal_ceiling,
509                ..PoolConfig::default()
510            },
511            max_readers,
512            ConnectionPool::new_read_only_live,
513        )
514    }
515
516    fn sqlite_read_only_with_pool_config(
517        path: impl AsRef<Path>,
518        pool_config: PoolConfig,
519        max_readers: Option<usize>,
520    ) -> Result<Self, SqliteError> {
521        Self::sqlite_read_only_with_pool_constructor(
522            path,
523            pool_config,
524            max_readers,
525            ConnectionPool::new,
526        )
527    }
528
529    fn sqlite_read_only_with_pool_constructor(
530        path: impl AsRef<Path>,
531        pool_config: PoolConfig,
532        max_readers: Option<usize>,
533        open_pool: fn(PoolConfig) -> Result<ConnectionPool, SqliteError>,
534    ) -> Result<Self, SqliteError> {
535        crate::extension::ensure_extensions_loaded();
536        let resolved = path.as_ref().to_path_buf();
537        let mut config = PoolConfig {
538            path: Some(resolved.clone()),
539            read_only: true,
540            write_queue_enabled: Some(false),
541            ..pool_config
542        };
543        if let Some(max_readers) = max_readers {
544            config.max_readers = max_readers;
545        }
546        // Both pool constructors open the writer slot with `SQLITE_OPEN_READ_ONLY`
547        // (no `SQLITE_OPEN_CREATE`) and set `PRAGMA query_only = ON` on it, so a
548        // missing path is rejected instead of created, and any write attempt is
549        // rejected at the SQLite level regardless of which code path reaches the
550        // writer.
551        let pool = open_pool(config)?;
552        Ok(Self {
553            pool: Arc::new(pool),
554            is_file_backed: true,
555            path: Some(resolved),
556            vector_tables_ready: Default::default(),
557            text_tables_ready: Default::default(),
558            sparse_tables_ready: Default::default(),
559            notes_seq_repair_runs: AtomicUsize::new(0),
560            store_schemas: std::array::from_fn(|_| Arc::new(StoreSchemaGate::default())),
561        })
562    }
563
564    /// In-memory SQLite database (for tests).
565    ///
566    /// All data is lost when the backend is dropped. The pool degrades to
567    /// single-connection mode since in-memory databases cannot be shared
568    /// across multiple connections.
569    pub fn memory() -> Result<Self, SqliteError> {
570        crate::extension::ensure_extensions_loaded();
571        let config = PoolConfig {
572            path: None,
573            ..PoolConfig::default()
574        };
575        let pool = ConnectionPool::new(config)?;
576        Ok(Self {
577            pool: Arc::new(pool),
578            is_file_backed: false,
579            path: None,
580            vector_tables_ready: Default::default(),
581            text_tables_ready: Default::default(),
582            sparse_tables_ready: Default::default(),
583            notes_seq_repair_runs: AtomicUsize::new(0),
584            store_schemas: std::array::from_fn(|_| Arc::new(StoreSchemaGate::default())),
585        })
586    }
587
588    /// Apply a service's schema plan (run migrations).
589    ///
590    /// Each migration in the plan's `sqlite` list is applied idempotently,
591    /// including when another opener commits it first. Already-applied
592    /// migrations are skipped after taking the SQLite write lock. The
593    /// `_schema_versions` table tracks which migrations have been run.
594    pub fn apply_schema(
595        &self,
596        plan: &crate::migrations::ServiceSchemaPlan,
597    ) -> Result<(), SqliteError> {
598        let admission = self.pool.write_admission();
599        crate::migrations::apply_schema_plan_with_admission(
600            &mut self.pool.migration_transactions(),
601            plan,
602            &admission,
603        )
604    }
605
606    /// Apply pack-auxiliary DDL statements.
607    ///
608    /// Executes the full plan in one transaction, applying each DDL statement
609    /// idempotently via `execute_batch`. Each statement MUST be self-contained
610    /// and use `CREATE TABLE IF NOT EXISTS` (or equivalent idempotent DDL) so
611    /// that calling this method more than once does not fail.
612    ///
613    /// Pack auxiliary tables are NOT tracked in `_schema_versions` — they are
614    /// non-versioned. Use `apply_schema` with a `ServiceSchemaPlan` when version
615    /// tracking is needed.
616    ///
617    /// Plans declaring nullable-column upgrades must use
618    /// [`Self::apply_pack_ddl_statements_with_columns`]. The runtime supplies
619    /// the SQL slice and column metadata separately because its `SchemaPlan`
620    /// type lives above this crate in the dependency chain.
621    pub fn apply_pack_ddl_statements(
622        &self,
623        statements: &[&'static str],
624    ) -> Result<(), SqliteError> {
625        self.apply_pack_ddl_statements_with_columns(statements, &[])
626    }
627
628    /// Apply a pack's nullable-column upgrades and idempotent SQL atomically.
629    ///
630    /// Missing columns are added only to existing tables; the full SQL plan
631    /// creates fresh tables. Existing columns and the final schema must match
632    /// the declarations. Schema inspection, additions, and SQL all run under
633    /// one writer transaction, including rollback if any later step fails.
634    pub fn apply_pack_ddl_statements_with_columns(
635        &self,
636        statements: &[&'static str],
637        additions: &[khive_types::PackColumnAddition],
638    ) -> Result<(), SqliteError> {
639        let writer = self.pool.writer_for_schema()?;
640        writer.transaction(|conn| {
641            pack_schema::add_missing_columns(conn, additions)?;
642            for &stmt in statements {
643                conn.execute_batch(stmt)?;
644            }
645            pack_schema::validate_columns(conn, additions)?;
646            Ok(())
647        })
648    }
649
650    /// Validate a pack's declared columns without applying SQL or acquiring a writer.
651    /// Read-only hosts use this before exposing verbs that require these columns.
652    pub fn validate_pack_schema_columns(
653        &self,
654        additions: &[khive_types::PackColumnAddition],
655    ) -> Result<(), SqliteError> {
656        if additions.is_empty() {
657            return Ok(());
658        }
659        let reader = self.pool.reader()?;
660        pack_schema::validate_columns(reader.conn(), additions)
661    }
662
663    /// Prepare the core schema for runtime boot.
664    ///
665    /// Writable backends acquire the canonical database-GC owner before the
666    /// writer, apply the ordinary versioned prefix, and may finish V21 only
667    /// through its zero-legacy-reference fast path. A legacy V20 database
668    /// remains at V20 for the async host's application-assisted attachment
669    /// cutover; this method alone is not a serving boot gate.
670    /// Read-only backends perform a query-only compatibility check and require
671    /// the snapshot to be at this build's exact latest schema version.
672    pub fn prepare_core_schema(&self) -> Result<u32, SqliteError> {
673        if self.is_read_only() {
674            let reader = self.pool.reader()?;
675            crate::migrations::validate_schema_is_current(reader.conn())
676        } else {
677            let latest = crate::migrations::MIGRATIONS
678                .last()
679                .map(|migration| migration.version)
680                .unwrap_or(0);
681            {
682                let reader = self.pool.reader()?;
683                let current = crate::migrations::read_schema_version(reader.conn())?;
684                if current >= latest {
685                    return crate::migrations::validate_schema_is_current(reader.conn());
686                }
687            }
688            let owner = crate::stores::blob::acquire_database_gc_owner_for_path_blocking(
689                self.pool.canonical_path().map(Path::to_path_buf),
690            )
691            .map_err(|error| {
692                SqliteError::InvalidData(format!(
693                    "failed to acquire database GC owner before schema preparation: {error}"
694                ))
695            })?;
696            self.run_core_migrations(&owner)
697        }
698    }
699
700    /// Inspect the coordinated V21 attachment cutover state.
701    pub fn attachment_cutover_status(
702        &self,
703    ) -> Result<crate::migrations::AttachmentCutoverStatus, SqliteError> {
704        let reader = self.pool.reader()?;
705        crate::migrations::attachment_cutover_status(reader.conn())
706    }
707
708    fn require_attachment_cutover_owner(
709        &self,
710        owner: &crate::stores::blob::DatabaseGcOwnerGuard,
711    ) -> Result<(), SqliteError> {
712        let sql = self.sql();
713        let backend_path = sql.database_path();
714        if owner.database_path() != backend_path.as_deref() {
715            return Err(SqliteError::InvalidData(format!(
716                "attachment cutover GC owner targets {:?}, but this backend is {:?}",
717                owner.database_path(),
718                backend_path.as_deref()
719            )));
720        }
721        Ok(())
722    }
723
724    /// Commit resumable V21 stage 1 while the caller owns this database's GC
725    /// protocol. The owner must remain live through verified application
726    /// backfill and finalization.
727    pub fn stage_attachment_cutover(
728        &self,
729        owner: &crate::stores::blob::DatabaseGcOwnerGuard,
730    ) -> Result<(), SqliteError> {
731        self.require_attachment_cutover_owner(owner)?;
732        if self.is_read_only() {
733            return Err(SqliteError::InvalidData(
734                "cannot stage attachment cutover on a read-only backend".into(),
735            ));
736        }
737        // Cutover work is schema work whoever calls it: its leases are migration class.
738        let _class = crate::disk_guard::lease_class_scope(crate::disk_guard::LeaseClass::Migration);
739        let mut writer = self.pool.writer_for_attachment_cutover()?;
740        let admission = self.pool.write_admission();
741        writer.conn_mut().with_mut(|conn| {
742            crate::migrations::stage_attachment_cutover_with_admission(conn, &admission)
743        })
744    }
745
746    /// Atomically publish a verified batch of pack-owned attachment roles.
747    pub fn apply_verified_attachments(
748        &self,
749        owner: &crate::stores::blob::DatabaseGcOwnerGuard,
750        attachments: &[khive_storage::Attachment],
751    ) -> Result<(), SqliteError> {
752        self.require_attachment_cutover_owner(owner)?;
753        if self.is_read_only() {
754            return Err(SqliteError::InvalidData(
755                "cannot apply verified attachments on a read-only backend".into(),
756            ));
757        }
758        let _class = crate::disk_guard::lease_class_scope(crate::disk_guard::LeaseClass::Migration);
759        let mut writer = self.pool.writer_for_attachment_cutover()?;
760        let admission = self.pool.write_admission();
761        writer.conn_mut().with_mut(|conn| {
762            let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
763            if let Err(error) = admission.check() {
764                let rollback = tx.rollback();
765                return Err(crate::migrations::capacity_refusal_after_rollback(
766                    conn,
767                    rollback,
768                    error,
769                    "verified attachment publication",
770                ));
771            }
772            for attachment in attachments {
773                attachment
774                    .validate()
775                    .map_err(|error| SqliteError::InvalidData(error.to_string()))?;
776                crate::migrations::apply_generic_verified_attachment(
777                    &tx,
778                    &attachment.record_uuid.to_string(),
779                    attachment.substrate.as_str(),
780                    &attachment.role,
781                    &attachment.content_ref,
782                    attachment.media_type.as_deref(),
783                    attachment.size_bytes,
784                    attachment.created_at,
785                )?;
786            }
787            tx.commit()?;
788            Ok(())
789        })
790    }
791
792    /// Atomically swap GC liveness/fences to attachments, remove the legacy
793    /// entity column, and record V21 while the canonical owner is held.
794    pub fn finalize_attachment_cutover(
795        &self,
796        owner: &crate::stores::blob::DatabaseGcOwnerGuard,
797    ) -> Result<(), SqliteError> {
798        self.require_attachment_cutover_owner(owner)?;
799        if self.is_read_only() {
800            return Err(SqliteError::InvalidData(
801                "cannot finalize attachment cutover on a read-only backend".into(),
802            ));
803        }
804        let _class = crate::disk_guard::lease_class_scope(crate::disk_guard::LeaseClass::Migration);
805        let mut writer = self.pool.writer_for_attachment_cutover()?;
806        let admission = self.pool.write_admission();
807        writer.conn_mut().with_mut(|conn| {
808            crate::migrations::finalize_attachment_cutover_with_admission(conn, &admission)
809        })
810    }
811
812    /// Get an EntityStore. Applies the entities DDL if not already present.
813    ///
814    /// Idempotent — safe to call multiple times.
815    pub fn entities(&self) -> Result<Arc<dyn khive_storage::EntityStore>, SqliteError> {
816        self.entities_for_namespace("local")
817    }
818
819    /// Get an EntityStore. The namespace parameter is validated (non-empty) and
820    /// the entities schema is applied, but the store itself is unscoped — namespace
821    /// is the caller's responsibility on each query/delete call.
822    pub fn entities_for_namespace(
823        &self,
824        namespace: &str,
825    ) -> Result<Arc<dyn khive_storage::EntityStore>, SqliteError> {
826        if namespace.trim().is_empty() {
827            return Err(SqliteError::InvalidData(
828                "entities namespace must be non-empty".to_string(),
829            ));
830        }
831        self.ensure_store_schema(StoreSchemaKind::Entities)?;
832
833        Ok(Arc::new(entity::SqlEntityStore::new(
834            Arc::clone(&self.pool),
835            self.is_file_backed,
836        )))
837    }
838
839    /// Get the role-keyed attachment store.
840    ///
841    /// Unlike the legacy capability accessors, this does not install DDL on
842    /// demand. The coordinated V21 core cutover owns creation of the table,
843    /// reference fences, GC liveness swap, and removal of the legacy entity
844    /// column as one boot-gated operation.
845    pub fn attachments(&self) -> Result<Arc<dyn khive_storage::AttachmentStore>, SqliteError> {
846        Ok(Arc::new(attachment::SqlAttachmentStore::new(
847            Arc::clone(&self.pool),
848            self.is_file_backed,
849        )))
850    }
851
852    /// Get a GraphStore for the default namespace.
853    ///
854    /// Creates the `graph_edges` table (with indexes) if it does not already
855    /// exist. Idempotent — safe to call multiple times.
856    pub fn graph(&self) -> Result<Arc<dyn khive_storage::GraphStore>, SqliteError> {
857        self.graph_for_namespace("local")
858    }
859
860    /// Get a GraphStore scoped to a namespace.
861    pub fn graph_for_namespace(
862        &self,
863        namespace: &str,
864    ) -> Result<Arc<dyn khive_storage::GraphStore>, SqliteError> {
865        if namespace.trim().is_empty() {
866            return Err(SqliteError::InvalidData(
867                "graph namespace must be non-empty".to_string(),
868            ));
869        }
870        self.ensure_store_schema(StoreSchemaKind::Graph)?;
871
872        Ok(Arc::new(
873            graph::SqlGraphStore::new_scoped(
874                Arc::clone(&self.pool),
875                self.is_file_backed,
876                namespace.trim().to_string(),
877            )
878            .with_index_repair(crate::stores::index_repair::IndexRepairContext::new(
879                Arc::clone(&self.pool),
880                self.store_schemas.clone(),
881                crate::stores::index_repair::IndexReadKind::Graph,
882            )),
883        ))
884    }
885
886    fn ensure_store_schema(&self, kind: StoreSchemaKind) -> Result<(), SqliteError> {
887        if self.is_read_only()
888            || self.store_schemas[kind as usize]
889                .ready
890                .load(Ordering::Acquire)
891        {
892            return Ok(());
893        }
894        let writer = self.constructor_writer()?;
895        self.ensure_store_schema_with_writer(kind, writer.conn())
896    }
897
898    fn ensure_store_schema_with_writer(
899        &self,
900        kind: StoreSchemaKind,
901        conn: &rusqlite::Connection,
902    ) -> Result<(), SqliteError> {
903        self.store_schemas[kind as usize].ensure(conn, kind.initializer())?;
904        Ok(())
905    }
906
907    fn constructor_writer(
908        &self,
909    ) -> Result<crate::pool::PooledAutocommitWriteUnit<'_>, SqliteError> {
910        let context = khive_storage::capture_request_read_context();
911        let Some(operation) = context.store_acquisition_operation() else {
912            return self.pool.writer_for_schema()?.admit_autocommit();
913        };
914        let writer = self
915            .pool
916            .schema_writer_until_for_admitted_operation(|| {
917                context.blocking_stop_reason().is_some()
918            })?
919            .ok_or_else(|| {
920                SqliteError::RequestReadStopped(khive_storage::StorageError::Timeout {
921                    operation: operation.into(),
922                })
923            })?;
924        writer.admit_autocommit()
925    }
926
927    /// Get a NoteStore. Applies the notes DDL if not already present.
928    ///
929    /// Idempotent — safe to call multiple times.
930    pub fn notes(&self) -> Result<Arc<dyn khive_storage::NoteStore>, SqliteError> {
931        self.notes_for_namespace("local")
932    }
933
934    /// Get a NoteStore. The namespace parameter is validated (non-empty) and
935    /// the notes schema is applied, but the store itself is unscoped — namespace
936    /// is the caller's responsibility on each query/delete call.
937    pub fn notes_for_namespace(
938        &self,
939        namespace: &str,
940    ) -> Result<Arc<dyn khive_storage::NoteStore>, SqliteError> {
941        if namespace.trim().is_empty() {
942            return Err(SqliteError::InvalidData(
943                "notes namespace must be non-empty".to_string(),
944            ));
945        }
946        if !self.is_read_only()
947            && (!self.store_schemas[StoreSchemaKind::Notes as usize]
948                .ready
949                .load(Ordering::Acquire)
950                || self.notes_seq_repair_runs.load(Ordering::Relaxed) == 0)
951        {
952            let writer = self.constructor_writer()?;
953            self.ensure_store_schema_with_writer(StoreSchemaKind::Notes, writer.conn())?;
954
955            // The anti-join repair is a full `notes` scan -- gate it to run at
956            // most once per backend/pool. `try_writer()` blocks for exclusive
957            // access to the single writer connection for this whole function,
958            // so this load-then-run-then-store is race-free: no other caller on
959            // this pool can observe or advance `notes_seq_repair_runs` while we
960            // hold the writer guard (khive #827).
961            self.repair_notes_seq_with_writer(writer.conn())?;
962        }
963
964        Ok(Arc::new(
965            note::SqlNoteStore::new(Arc::clone(&self.pool), self.is_file_backed).with_index_repair(
966                crate::stores::index_repair::IndexRepairContext::new(
967                    Arc::clone(&self.pool),
968                    self.store_schemas.clone(),
969                    crate::stores::index_repair::IndexReadKind::Notes,
970                ),
971            ),
972        ))
973    }
974
975    /// How many times the lazy `notes_seq` anti-join repair has actually
976    /// executed against this backend's pool. Exposed for regression tests
977    /// asserting the repair runs at most once per backend for the process's
978    /// lifetime, not once per `notes_for_namespace` call (khive #827).
979    pub fn notes_seq_repair_run_count(&self) -> usize {
980        self.notes_seq_repair_runs.load(Ordering::Relaxed)
981    }
982
983    /// Get an EventStore for the default namespace.
984    ///
985    /// Creates the `events` table (with indexes) if it does not already exist.
986    /// Idempotent — safe to call multiple times.
987    pub fn events(&self) -> Result<Arc<dyn khive_storage::EventStore>, SqliteError> {
988        self.events_for_namespace("local")
989    }
990
991    /// Get an EventStore scoped to a namespace.
992    pub fn events_for_namespace(
993        &self,
994        namespace: &str,
995    ) -> Result<Arc<dyn khive_storage::EventStore>, SqliteError> {
996        if namespace.trim().is_empty() {
997            return Err(SqliteError::InvalidData(
998                "events namespace must be non-empty".to_string(),
999            ));
1000        }
1001        self.ensure_store_schema(StoreSchemaKind::Events)?;
1002
1003        Ok(Arc::new(event::SqlEventStore::new_scoped(
1004            Arc::clone(&self.pool),
1005            self.is_file_backed,
1006            namespace.trim().to_string(),
1007        )))
1008    }
1009
1010    /// Get the agent-process store (ADR-142 §1). Applies the agents DDL if not
1011    /// already present. Idempotent — safe to call multiple times. Unlike the
1012    /// other stores here, agent-process records are not namespace-scoped, so
1013    /// there is no `_for_namespace` variant.
1014    pub fn agents(&self) -> Result<Arc<dyn khive_storage::AgentStore>, SqliteError> {
1015        self.ensure_store_schema(StoreSchemaKind::Agents)?;
1016
1017        Ok(Arc::new(agents::SqlAgentStore::new(
1018            Arc::clone(&self.pool),
1019            self.is_file_backed,
1020        )))
1021    }
1022
1023    /// Get a VectorStore for a specific embedding model, scoped to the default namespace.
1024    ///
1025    /// Creates the vec0 virtual table if it does not already exist. The `model_key`
1026    /// must contain only ASCII alphanumeric/underscore characters. The `embedding_model`
1027    /// is the canonical display name stored in each vector row.
1028    pub fn vectors(
1029        &self,
1030        model_key: &str,
1031        embedding_model: &str,
1032        dimensions: usize,
1033    ) -> Result<Arc<dyn khive_storage::VectorStore>, SqliteError> {
1034        self.vectors_for_namespace(model_key, embedding_model, dimensions, "local")
1035    }
1036
1037    /// Get a VectorStore for a specific embedding model with a default namespace.
1038    ///
1039    /// Creates the vec0 virtual table if it does not already exist. The `namespace`
1040    /// is a default for trait methods that lack a per-call namespace parameter
1041    /// (count, delete, info). Access control is enforced at the runtime layer.
1042    ///
1043    /// The `model_key` must contain only ASCII alphanumeric/underscore characters.
1044    /// The `embedding_model` is the canonical display name stored in the `embedding_model`
1045    /// column of each vector row (e.g. `"all-minilm-l6-v2"`).
1046    pub fn vectors_for_namespace(
1047        &self,
1048        model_key: &str,
1049        embedding_model: &str,
1050        dimensions: usize,
1051        namespace: &str,
1052    ) -> Result<Arc<dyn khive_storage::VectorStore>, SqliteError> {
1053        validate_vector_model_key(model_key)?;
1054        if namespace.trim().is_empty() {
1055            return Err(SqliteError::InvalidData(
1056                "vector store namespace must be non-empty".to_string(),
1057            ));
1058        }
1059        self.ensure_vector_tables(&[(model_key, dimensions)])?;
1060        Ok(Arc::new(vectors::SqliteVecStore::new(
1061            Arc::clone(&self.pool),
1062            self.is_file_backed,
1063            model_key.to_string(),
1064            embedding_model.to_string(),
1065            dimensions,
1066            namespace.trim().to_string(),
1067        )?))
1068    }
1069
1070    /// Ensure all requested vector tables with one schema-writer acquisition.
1071    /// Read-only backends inspect the same tables using one reader instead.
1072    ///
1073    /// A writable backend prepares each model's table once: the first call that
1074    /// names it validates or creates the table under the writer and records the
1075    /// model only after every step succeeded. Later calls for a recorded model
1076    /// return without taking the writer, and a failed call records nothing, so
1077    /// the next call repeats the check.
1078    pub fn ensure_vector_tables(&self, models: &[(&str, usize)]) -> Result<(), SqliteError> {
1079        for (model_key, _) in models {
1080            validate_vector_model_key(model_key)?;
1081        }
1082        if models.is_empty() {
1083            return Ok(());
1084        }
1085
1086        // Ensure sqlite-vec is registered before creating vec0 tables.
1087        crate::extension::ensure_extensions_loaded();
1088
1089        // Direct store construction may precede migrations. Leave the
1090        // provenance sidecar to its versioned migration; vector operations
1091        // tolerate its absence until the ledger advances.
1092
1093        if self.is_read_only() {
1094            // Snapshot inspection must not check schema through the pool's
1095            // query-only writer slot: even a SELECT there is a writer-class
1096            // acquisition and violates ADR-028 A2's write-free lifecycle.
1097            let reader = self.pool.reader()?;
1098            for (model_key, _) in models {
1099                let table = format!("vec_{model_key}");
1100                if !sqlite_table_exists(reader.conn(), &table)? {
1101                    return Err(SqliteError::InvalidData(format!(
1102                        "read-only database has no vector table '{table}'; create and populate it in \
1103                         a writable copy before opening the snapshot"
1104                    )));
1105                }
1106                validate_vector_table_columns(reader.conn(), &table)?;
1107            }
1108            return Ok(());
1109        }
1110
1111        let pending = self.unprepared_vector_tables(models);
1112        if pending.is_empty() {
1113            return Ok(());
1114        }
1115        let writer = self.constructor_writer()?;
1116        // A concurrent caller may have prepared these tables while this one
1117        // waited for the writer.
1118        let pending = self.unprepared_vector_tables(&pending);
1119        if pending.is_empty() {
1120            return Ok(());
1121        }
1122
1123        // Detect old-schema vec0 tables that predate the `field` column.
1124        // Use pragma_table_info to check columns directly; substring matching on the
1125        // CREATE DDL is fragile (a model_key containing "field" would false-match).
1126        for (model_key, _) in &pending {
1127            let table = format!("vec_{model_key}");
1128            // V17 migration (vector_embedding_model_tag_preserving_rebuild) adds
1129            // `field` and `embedding_model` to all pre-existing vec0 tables at
1130            // migration time.  If this table still lacks either column post-migration
1131            // that indicates the database was not migrated — return a hard error
1132            // rather than silently dropping data.
1133            if sqlite_table_exists(writer.conn(), &table)? {
1134                validate_vector_table_columns(writer.conn(), &table)?;
1135            }
1136        }
1137
1138        // Ensure the _embedding_models registry table exists.
1139        // This is a no-op when the table already exists. Running it here ensures
1140        // the registry is present for any caller that opens a vector store without
1141        // first calling run_migrations() (e.g., tests that create stores directly).
1142        // Production callers are expected to call run_migrations() at startup, which
1143        // creates the registry via V14; this is a belt-and-suspenders fallback.
1144        // Schema is defined in `migrations::EMBEDDING_MODELS_DDL` (single source of
1145        // truth) to prevent the two copies from silently drifting.
1146        writer
1147            .conn()
1148            .execute_batch(crate::migrations::EMBEDDING_MODELS_DDL)?;
1149
1150        // Same guarantee for the ANN write log: vector write paths append to it
1151        // in the same transaction as the vec0 mutation, so it must exist in any
1152        // database that hosts vec_* tables.
1153        writer
1154            .conn()
1155            .execute_batch(crate::migrations::ANN_WRITE_LOG_DDL)?;
1156        writer
1157            .conn()
1158            .execute_batch(crate::migrations::ANN_WRITE_LOG_MODEL_SEQ_INDEX_DDL)?;
1159        writer
1160            .conn()
1161            .execute_batch(crate::migrations::ANN_CONSUMER_PENDING_DDL)?;
1162        // Create missing vec0 tables without changing existing vector data.
1163        for (model_key, dimensions) in &pending {
1164            let ddl = format!(
1165                "CREATE VIRTUAL TABLE IF NOT EXISTS vec_{} USING vec0(\
1166                 subject_id TEXT PRIMARY KEY, \
1167                 namespace TEXT NOT NULL, \
1168                 kind TEXT NOT NULL, \
1169                 field TEXT NOT NULL, \
1170                 embedding_model TEXT NOT NULL, \
1171                 embedding float[{}] distance_metric=cosine\
1172                 )",
1173                model_key, dimensions
1174            );
1175            writer.conn().execute_batch(&ddl)?;
1176        }
1177        // Record the models only now that every step above succeeded, while the
1178        // writer is still held so a waiting caller re-checks against this set.
1179        self.vector_tables_ready.lock().extend(
1180            pending
1181                .iter()
1182                .map(|(model_key, _)| (*model_key).to_string()),
1183        );
1184        Ok(())
1185    }
1186
1187    /// The requested vector models this backend has not yet validated or created.
1188    fn unprepared_vector_tables<'a>(&self, models: &[(&'a str, usize)]) -> Vec<(&'a str, usize)> {
1189        let ready = self.vector_tables_ready.lock();
1190        models
1191            .iter()
1192            .filter(|(model_key, _)| !ready.contains(*model_key))
1193            .copied()
1194            .collect()
1195    }
1196
1197    /// Register an embedding model in the `_embedding_models` registry table.
1198    ///
1199    /// Idempotent: if a row with the same `canonical_key` already exists, updates its
1200    /// status back to `'active'` without changing other fields.
1201    pub fn register_embedding_model(
1202        &self,
1203        engine_name: &str,
1204        model_id: &str,
1205        key_version: &str,
1206        dimensions: u32,
1207    ) -> Result<(), SqliteError> {
1208        let writer = self.pool.writer_for_schema()?.admit_autocommit()?;
1209        writer
1210            .conn()
1211            .execute_batch(crate::migrations::EMBEDDING_MODELS_DDL)
1212            .map_err(|error| {
1213                SqliteError::write(error, khive_storage::error::SqliteWriteStage::Statement)
1214            })?;
1215
1216        let now = chrono::Utc::now().timestamp_micros();
1217        let canonical_key =
1218            format!("{engine_name}:{model_id}:{key_version}:{dimensions}").into_bytes();
1219        let id = uuid::Uuid::new_v4();
1220        writer
1221            .conn()
1222            .execute(
1223                "INSERT INTO _embedding_models \
1224             (id, engine_name, model_id, key_version, dim, output_dim, status, \
1225              activated_at, superseded_at, superseded_by, canonical_key, created_at) \
1226             VALUES (?1, ?2, ?3, ?4, ?5, NULL, 'active', ?6, NULL, NULL, ?7, ?8) \
1227             ON CONFLICT(canonical_key) DO UPDATE SET \
1228                status = 'active', \
1229                activated_at = COALESCE(_embedding_models.activated_at, excluded.activated_at)",
1230                rusqlite::params![
1231                    id.as_bytes().as_slice(),
1232                    engine_name,
1233                    model_id,
1234                    key_version,
1235                    dimensions as i64,
1236                    now,
1237                    canonical_key,
1238                    now,
1239                ],
1240            )
1241            .map_err(|error| {
1242                SqliteError::write(error, khive_storage::error::SqliteWriteStage::Statement)
1243            })?;
1244        Ok(())
1245    }
1246
1247    /// Get a SparseStore for a specific model key, scoped to the default namespace.
1248    ///
1249    /// Creates the sparse table if it does not already exist.
1250    pub fn sparse(
1251        &self,
1252        model_key: &str,
1253    ) -> Result<Arc<dyn khive_storage::SparseStore>, SqliteError> {
1254        self.sparse_for_namespace(model_key, "local")
1255    }
1256
1257    /// Forget which FTS tables this backend has proven, so the next fetch
1258    /// re-checks each table under the writer. For tests that clear a durable
1259    /// marker to reproduce a store that predates it.
1260    #[cfg(test)]
1261    pub(crate) fn forget_proven_text_tables_for_test(&self) {
1262        self.text_tables_ready.lock().clear();
1263    }
1264
1265    /// Ensure `sparse_<model_key>` and its namespace index exist, once per
1266    /// backend: a warm call takes no write unit. The caller has validated
1267    /// `model_key`. The set is checked before the writer is taken and again
1268    /// under it, and a key is recorded only after the ensure succeeded, while
1269    /// the writer is still held.
1270    fn ensure_sparse_table(&self, model_key: &str) -> Result<(), SqliteError> {
1271        if self.sparse_tables_ready.lock().contains(model_key) {
1272            return Ok(());
1273        }
1274        let writer = self.pool.writer_for_schema()?.admit_autocommit()?;
1275        if !self.sparse_tables_ready.lock().contains(model_key) {
1276            sparse::ensure_sparse_schema(writer.conn(), model_key)
1277                .map_err(SqliteError::Rusqlite)?;
1278            self.sparse_tables_ready
1279                .lock()
1280                .insert(model_key.to_string());
1281        }
1282        Ok(())
1283    }
1284
1285    /// Get a SparseStore for a specific model key with an explicit default namespace.
1286    ///
1287    /// The `model_key` must contain only ASCII alphanumeric/underscore characters.
1288    pub fn sparse_for_namespace(
1289        &self,
1290        model_key: &str,
1291        namespace: &str,
1292    ) -> Result<Arc<dyn khive_storage::SparseStore>, SqliteError> {
1293        if model_key.is_empty()
1294            || !model_key
1295                .chars()
1296                .all(|c| c.is_ascii_alphanumeric() || c == '_')
1297        {
1298            return Err(SqliteError::InvalidData(format!(
1299                "invalid model_key '{}': must be non-empty and contain only alphanumeric/underscore characters",
1300                model_key
1301            )));
1302        }
1303        if namespace.trim().is_empty() {
1304            return Err(SqliteError::InvalidData(
1305                "sparse store namespace must be non-empty".to_string(),
1306            ));
1307        }
1308
1309        if self.is_read_only() {
1310            let table = format!("sparse_{model_key}");
1311            let reader = self.pool.reader()?;
1312            if !sqlite_table_exists(reader.conn(), &table)? {
1313                return Err(SqliteError::InvalidData(format!(
1314                    "read-only database has no sparse table '{table}'; create and populate it in \
1315                     a writable copy before opening the snapshot"
1316                )));
1317            }
1318        } else {
1319            self.ensure_sparse_table(model_key)?;
1320        }
1321
1322        Ok(Arc::new(sparse::SqliteSparseStore::new(
1323            Arc::clone(&self.pool),
1324            self.is_file_backed,
1325            model_key.to_string(),
1326            namespace.trim().to_string(),
1327        )?))
1328    }
1329
1330    /// Get a TextSearch for a specific table key.
1331    ///
1332    /// Creates the FTS5 virtual table if it does not already exist. Uses the
1333    /// `trigram` tokenizer by default (CJK-safe).
1334    ///
1335    /// The `table_key` must contain only ASCII alphanumeric/underscore characters.
1336    pub fn text(&self, table_key: &str) -> Result<Arc<dyn khive_storage::TextSearch>, SqliteError> {
1337        self.text_with_tokenizer(table_key, "trigram")
1338    }
1339
1340    /// Get a TextSearch with an explicit FTS5 tokenizer.
1341    ///
1342    /// Use when you need a tokenizer other than the default `trigram` — for
1343    /// example `unicode61` for Latin-only corpora.
1344    ///
1345    /// Both `table_key` and `tokenizer` must contain only ASCII
1346    /// alphanumeric/underscore characters.
1347    pub fn text_with_tokenizer(
1348        &self,
1349        table_key: &str,
1350        tokenizer: &str,
1351    ) -> Result<Arc<dyn khive_storage::TextSearch>, SqliteError> {
1352        if table_key.is_empty()
1353            || !table_key
1354                .chars()
1355                .all(|c| c.is_ascii_alphanumeric() || c == '_')
1356        {
1357            return Err(SqliteError::InvalidData(format!(
1358                "invalid table_key '{}': must be non-empty and contain only \
1359                 alphanumeric/underscore characters",
1360                table_key
1361            )));
1362        }
1363        // `text::rowid_map_table`/`text::rowid_map_state_table` name a
1364        // table's sidecar map `{table}_rowids` and that map's own completion
1365        // marker `{table}_rowids_state` — a `table_key` ending in either
1366        // reserved suffix (e.g. "entities_rowids" or "entities_rowids_state")
1367        // would resolve to the exact sidecar table name another key's own
1368        // map or marker already reserves. `CREATE VIRTUAL TABLE IF NOT
1369        // EXISTS` would then silently accept that ordinary (non-FTS5) table
1370        // as if it were this key's FTS table, and every later point
1371        // read/write against it would fail against the wrong schema.
1372        if table_key.ends_with("_rowids") || table_key.ends_with("_rowids_state") {
1373            return Err(SqliteError::InvalidData(format!(
1374                "invalid table_key '{}': must not end in '_rowids' or '_rowids_state' — those \
1375                 suffixes are reserved for a text table's own rowid-map sidecar and its \
1376                 completion-marker state table (see text::rowid_map_table, \
1377                 text::rowid_map_state_table)",
1378                table_key
1379            )));
1380        }
1381        if tokenizer.is_empty()
1382            || !tokenizer
1383                .chars()
1384                .all(|c| c.is_ascii_alphanumeric() || c == '_')
1385        {
1386            return Err(SqliteError::InvalidData(format!(
1387                "invalid tokenizer '{}': must be non-empty and contain only \
1388                 alphanumeric/underscore characters",
1389                tokenizer
1390            )));
1391        }
1392
1393        let ddl = format!(
1394            "CREATE VIRTUAL TABLE IF NOT EXISTS fts_{} USING fts5(\
1395             subject_id UNINDEXED, \
1396             kind UNINDEXED, \
1397             title, \
1398             body, \
1399             tags UNINDEXED, \
1400             namespace UNINDEXED, \
1401             metadata UNINDEXED, \
1402             updated_at UNINDEXED, \
1403             record_kind, \
1404             tokenize = '{}'\
1405             )",
1406            table_key, tokenizer
1407        );
1408        let table = format!("fts_{table_key}");
1409        if self.is_read_only() {
1410            let reader = self.pool.reader()?;
1411            if !sqlite_table_exists(reader.conn(), &table)? {
1412                return Err(SqliteError::InvalidData(format!(
1413                    "read-only database has no text-search table '{table}'; create and populate \
1414                     it in a writable copy before opening the snapshot"
1415                )));
1416            }
1417            // A read-only connection cannot create, backfill, or reconcile
1418            // the rowid-map sidecar: a snapshot taken before this migration
1419            // shipped can have the FTS table without its map at all, and a
1420            // snapshot taken mid-backfill (a crash, or a copy made between
1421            // the map's creation and its completion marker being written) can
1422            // have a map table that is only partially populated. Trusting an
1423            // unproven map here would silently hide live rows the map
1424            // doesn't yet know about — only a map with a durable completion
1425            // marker (`ROWID_MAP_BACKFILL_COMPLETE`) is safe to join against
1426            // read-only; anything else falls back to the pre-map scan
1427            // predicates rather than constructing a store whose get/delete
1428            // would either fail against a nonexistent sidecar or silently
1429            // miss rows an unreconciled map doesn't cover.
1430            let map = text::rowid_map_table(&table);
1431            let map_exists = sqlite_table_exists(reader.conn(), &map)?;
1432            let marker_present = if map_exists {
1433                let state = text::rowid_map_state_table(&table);
1434                sqlite_table_exists(reader.conn(), &state)?
1435                    && reader.conn().query_row(
1436                        &format!(
1437                            "SELECT EXISTS(SELECT 1 FROM {state} WHERE key = 'backfill' AND value = ?1)"
1438                        ),
1439                        rusqlite::params![text::ROWID_MAP_BACKFILL_COMPLETE],
1440                        |row| row.get(0),
1441                    )?
1442            } else {
1443                false
1444            };
1445            if !map_exists || !marker_present {
1446                warn_scan_fallback_once(&table);
1447                return Ok(Arc::new(text::Fts5TextSearch::new_scan_fallback(
1448                    Arc::clone(&self.pool),
1449                    self.is_file_backed,
1450                    table_key.to_string(),
1451                )));
1452            }
1453        } else if !self.text_tables_ready.lock().contains(&table) {
1454            let writer = self.pool.writer_for_schema()?.admit_autocommit()?;
1455            // A concurrent caller may have proven this table while this one
1456            // waited for the writer.
1457            if !self.text_tables_ready.lock().contains(&table) {
1458                writer.conn().execute_batch(&ddl)?;
1459                writer.conn().execute_batch(&text::rowid_map_ddl(&table))?;
1460                ensure_fts_rowid_map_backfilled(
1461                    writer.conn(),
1462                    &table,
1463                    &self.pool.write_admission(),
1464                )?;
1465                // Record the table only now that every step succeeded, while
1466                // the writer is still held.
1467                self.text_tables_ready.lock().insert(table.clone());
1468            }
1469        }
1470
1471        Ok(Arc::new(text::Fts5TextSearch::new(
1472            Arc::clone(&self.pool),
1473            self.is_file_backed,
1474            table_key.to_string(),
1475        )))
1476    }
1477
1478    /// Get a `BlobStore` rooted per khive#292's precedence chain:
1479    /// `KHIVE_BLOB_ROOT` env var > `config_root` (a caller-resolved
1480    /// `khive.toml` override — `khive-db` has no TOML parser of its own) >
1481    /// beside this backend's database directory. `floor_bytes` overrides the
1482    /// default fail-closed free-space floor (`None` keeps the default, 2% of
1483    /// the volume and at least 5 GiB; ADR-111 Amendment 6). Errors if none of
1484    /// the three roots apply — e.g. an in-memory
1485    /// backend with no override and no env var has nowhere to default to.
1486    pub fn blob_store(
1487        &self,
1488        config_root: Option<&Path>,
1489        floor_bytes: Option<u64>,
1490    ) -> Result<Arc<dyn khive_storage::BlobStore>, SqliteError> {
1491        Ok(Arc::new(self.fs_blob_store(config_root, floor_bytes)?))
1492    }
1493
1494    /// The concrete store behind [`Self::blob_store`], so its resolved floor
1495    /// can be read back.
1496    pub(crate) fn fs_blob_store(
1497        &self,
1498        config_root: Option<&Path>,
1499        floor_bytes: Option<u64>,
1500    ) -> Result<blob::FsBlobStore, SqliteError> {
1501        let root = blob::resolve_blob_root(self.data_dir().as_deref(), config_root)?;
1502        let floor = blob::FsBlobStore::resolve_floor(&root, floor_bytes);
1503        blob::FsBlobStore::new(root, floor)
1504    }
1505
1506    /// Resolve the filesystem blob root exactly like [`Self::blob_store`] but
1507    /// require it to exist instead of creating it. Snapshot runtimes wrap the
1508    /// returned capability so its read methods remain available while every
1509    /// mutator is refused.
1510    pub fn blob_store_read_only(
1511        &self,
1512        config_root: Option<&Path>,
1513        floor_bytes: Option<u64>,
1514    ) -> Result<Arc<dyn khive_storage::BlobStore>, SqliteError> {
1515        let root = blob::resolve_blob_root(self.data_dir().as_deref(), config_root)?;
1516        let floor = blob::FsBlobStore::resolve_floor(&root, floor_bytes);
1517        Ok(Arc::new(blob::FsBlobStore::open_existing(root, floor)?))
1518    }
1519
1520    /// Is this a file-backed backend?
1521    pub fn is_file_backed(&self) -> bool {
1522        self.is_file_backed
1523    }
1524
1525    /// Whether this backend was opened with SQLite's read-only/query-only
1526    /// contract, explicitly or after filesystem-mode detection.
1527    pub fn is_read_only(&self) -> bool {
1528        self.pool.config().read_only
1529    }
1530
1531    /// Return the directory containing the backend's database file, or `None`
1532    /// for an in-memory backend.
1533    pub fn data_dir(&self) -> Option<std::path::PathBuf> {
1534        self.path.as_ref()?.parent().map(|p| p.to_path_buf())
1535    }
1536
1537    /// Root directory for this database's ANN segment tree, or `None` for an
1538    /// in-memory backend. Derived from the database file name itself
1539    /// (`<db-file>.ann/` beside the file), so two databases sharing a parent
1540    /// directory can never adopt each other's segments or UUID maps. The
1541    /// suffix is appended at the `OsString` byte level — a lossy UTF-8
1542    /// conversion would collapse distinct non-UTF-8 filenames into one
1543    /// replacement-character root, breaking exactly that isolation.
1544    pub fn ann_root(&self) -> Option<std::path::PathBuf> {
1545        ann_root_for(self.path.as_ref()?)
1546    }
1547
1548    /// Access the underlying pool (escape hatch).
1549    pub fn pool(&self) -> &ConnectionPool {
1550        &self.pool
1551    }
1552
1553    /// Durable owner evidence for this backend, captured at pool construction.
1554    /// The caller must choose the effective main backend before comparing it.
1555    pub fn database_owner_identity(
1556        &self,
1557    ) -> Result<DatabaseOwnerIdentity, DatabaseOwnerIdentityError> {
1558        self.pool.database_owner_identity()
1559    }
1560
1561    /// Refuse a secondary or copied database as the supplied database owner.
1562    pub fn verify_database_owner(
1563        &self,
1564        expected: &DatabaseOwnerIdentity,
1565    ) -> Result<(), DatabaseOwnerIdentityError> {
1566        self.pool.verify_database_owner(expected)
1567    }
1568
1569    /// Clone the underlying pool Arc.
1570    pub fn pool_arc(&self) -> Arc<ConnectionPool> {
1571        Arc::clone(&self.pool)
1572    }
1573}
1574
1575/// `<db-file>.ann` sibling of a database file, appended at the `OsString`
1576/// byte level: a lossy UTF-8 conversion would collapse distinct non-UTF-8
1577/// filenames into one replacement-character root, breaking the per-database
1578/// segment isolation that `ann_root` exists to guarantee.
1579fn ann_root_for(path: &std::path::Path) -> Option<std::path::PathBuf> {
1580    let mut file = path.file_name()?.to_os_string();
1581    file.push(".ann");
1582    path.parent().map(|p| p.join(file))
1583}
1584
1585#[cfg(test)]
1586#[path = "backend/store_accessor_tests.rs"]
1587mod store_accessor_tests;
1588
1589#[cfg(test)]
1590#[path = "backend/store_accessor_index_tests.rs"]
1591mod store_accessor_index_tests;
1592
1593#[cfg(test)]
1594#[path = "backend/schema_barrier_tests.rs"]
1595mod schema_barrier_tests;
1596
1597#[cfg(test)]
1598#[path = "backend_tests.rs"]
1599mod tests;
1600
1601#[cfg(test)]
1602#[path = "backend_admission_tests.rs"]
1603mod admission_tests;
1604
1605#[cfg(test)]
1606mod live_readonly_tests;