Skip to main content

khive_db/
backend.rs

1//! Concrete storage backend providing capability traits.
2//!
3//! `StorageBackend` owns a `ConnectionPool` and provides factory methods for all
4//! ten capability traits (`SqlAccess`, `NoteStore`, `EntityStore`, `GraphStore`,
5//! `EventStore`, `VectorStore`, `SparseStore`, `TextSearch`, `BlobStore`, and
6//! `AttachmentStore`). File-backed for production; in-memory for tests.
7
8use std::path::Path;
9use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
10use std::sync::Arc;
11
12use rusqlite::OptionalExtension;
13
14use crate::database_owner_identity::{DatabaseOwnerIdentity, DatabaseOwnerIdentityError};
15use crate::error::SqliteError;
16use crate::pool::{ConnectionPool, PoolConfig, WalCeilingPolicy};
17use crate::stores::{agents, attachment, blob, entity, event, graph, note, sparse, text, vectors};
18
19mod capability_factories;
20mod code_map;
21mod core_stores;
22#[path = "backend/schema_readiness.rs"]
23mod memory_visibility;
24mod pack_schema;
25mod policy_open;
26
27#[cfg(test)]
28#[path = "backend/filter_builder_tests.rs"]
29mod filter_builder_tests;
30
31mod namespace_listing;
32pub use namespace_listing::NamespaceLiveness;
33
34#[cfg(test)]
35#[path = "backend/memory_visibility_tests.rs"]
36mod memory_visibility_tests;
37
38#[cfg(any(unix, windows))]
39mod claimed_file_identity;
40
41fn sqlite_table_exists(conn: &rusqlite::Connection, table: &str) -> Result<bool, SqliteError> {
42    conn.query_row(
43        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1",
44        rusqlite::params![table],
45        |row| row.get::<_, i64>(0),
46    )
47    .optional()
48    .map(|row| row.is_some())
49    .map_err(SqliteError::Rusqlite)
50}
51
52/// Populate `table`'s [`text::rowid_map_table`] from `table` itself, and
53/// record completion in [`text::rowid_map_state_table`], the first time this
54/// backend opens a database that predates the map.
55///
56/// `StorageBackend::text()`/`text_with_tokenizer()` is called uncached on
57/// essentially every text-store access (`khive-runtime` builds a fresh
58/// `Fts5TextSearch` per call, never caching the `Arc`). The backend skips this
59/// check once a table's marker is durable, but a table with no rows yet is
60/// re-checked on every fetch, so the check stays on the hot path — it must
61/// stay O(1), not scale with either
62/// table's row count, or it reintroduces the exact class of cost this
63/// migration exists to remove. `SELECT EXISTS(... LIMIT 1)` is an index probe
64/// that stops at the first row, unlike `COUNT(*)` which SQLite satisfies by
65/// walking every row of the smallest available index.
66///
67/// Completion is read from a durable marker row rather than inferred from
68/// the map's own row count: a map can legitimately be empty for a table with
69/// no rows yet, which is indistinguishable from "never backfilled" by row
70/// count alone, and every runtime write path (`text.rs`'s
71/// `delete_document_dml`, `upsert_document_dml`, `batch_upsert_documents_dml`,
72/// and the raw SQL in `khive-runtime`'s
73/// `atomic_prepare`/`atomic_message`/`curation`) maintains the FTS row and
74/// its map row atomically, inside one transaction, so a legitimately
75/// half-empty map from a live write path never happens either. Until the
76/// table actually holds a row, there is nothing to reconcile and the marker
77/// is deliberately left unwritten — both probes below stay O(1) index-only
78/// lookups on an empty table, so repeating them costs nothing, and a table
79/// that later gains rows through anything other than the maintained write
80/// paths (a raw-SQL legacy seed, or a restored pre-map snapshot) is still
81/// picked up and reconciled the next time this runs. Once the table holds at
82/// least one row, the marker asserts a bijection — every live row has
83/// exactly one map row pointing at it, and every map row points at a live
84/// row with the same key — so reconciliation runs in three steps before the
85/// marker is written, all inside one transaction:
86///
87/// 1. Any existing map row that no longer has a matching live FTS row at the
88///    same rowid AND the same `(namespace, subject_id)` is removed first. A
89///    map row can otherwise survive with the wrong key after FTS5 reuses its
90///    rowid for a different document (the crash window
91///    `delete_document_dml` guards against at the single-delete level; this
92///    is the same class of staleness surviving into a legacy/reconciliation
93///    pass instead).
94/// 2. The map is (re)built from every current FTS row (`INSERT OR REPLACE`),
95///    which reconciles a partially populated map rather than only filling a
96///    wholly empty one. `updated_at ASC, rowid ASC` matches migration 024's
97///    own backfill ordering: for any legacy duplicate `(namespace,
98///    subject_id)` pair, the row with the newest `updated_at` survives
99///    `INSERT OR REPLACE`, breaking a tie toward the higher rowid.
100/// 3. Any non-NULL-key FTS row that lost step 2's survivor race — a
101///    duplicate whose rowid the map no longer points at — is deleted,
102///    mirroring migration 024's own first sweep, so no live row is left
103///    without a map entry.
104///
105/// The marker is then written in the same transaction. Returns whether the
106/// marker is durable when this returns: `false` only for a table with no rows
107/// yet. This function never
108/// checks `entities`/`notes` for orphaned subjects — that sweep is specific
109/// to those two backing tables and stays in migration 024's SQL; a generic
110/// `table_key` here has no fixed backing table to check against.
111fn ensure_fts_rowid_map_backfilled(
112    conn: &rusqlite::Connection,
113    table: &str,
114    admission: &crate::pool::WriteAdmission,
115) -> Result<bool, SqliteError> {
116    let map = text::rowid_map_table(table);
117    let state = text::rowid_map_state_table(table);
118    let already_backfilled: bool = conn.query_row(
119        &format!("SELECT EXISTS(SELECT 1 FROM {state} WHERE key = 'backfill' AND value = ?1)"),
120        rusqlite::params![text::ROWID_MAP_BACKFILL_COMPLETE],
121        |row| row.get(0),
122    )?;
123    if already_backfilled {
124        return Ok(true);
125    }
126    let fts_has_a_row: bool = conn.query_row(
127        &format!("SELECT EXISTS(SELECT 1 FROM {table} LIMIT 1)"),
128        [],
129        |row| row.get(0),
130    )?;
131    if !fts_has_a_row {
132        return Ok(false);
133    }
134
135    conn.execute_batch("BEGIN IMMEDIATE")?;
136    if let Err(error) = admission.check() {
137        return Err(crate::migrations::capacity_refusal_after_rollback(
138            conn,
139            conn.execute_batch("ROLLBACK"),
140            error,
141            "FTS rowid-map backfill",
142        ));
143    }
144    let result: Result<(), SqliteError> = (|| {
145        conn.execute_batch(&format!(
146            "DELETE FROM {map} WHERE NOT EXISTS ( \
147                 SELECT 1 FROM {table} \
148                 WHERE {table}.rowid = {map}.rowid \
149                   AND {table}.namespace = {map}.namespace \
150                   AND {table}.subject_id = {map}.subject_id \
151             )"
152        ))?;
153        conn.execute_batch(&format!(
154            "INSERT OR REPLACE INTO {map} (namespace, subject_id, rowid) \
155             SELECT namespace, subject_id, rowid FROM {table} \
156             WHERE namespace IS NOT NULL AND subject_id IS NOT NULL \
157             ORDER BY updated_at ASC, rowid ASC"
158        ))?;
159        conn.execute_batch(&format!(
160            "DELETE FROM {table} \
161             WHERE namespace IS NOT NULL AND subject_id IS NOT NULL \
162               AND rowid NOT IN (SELECT rowid FROM {map})"
163        ))?;
164        conn.execute(
165            &format!("INSERT OR REPLACE INTO {state} (key, value) VALUES ('backfill', ?1)"),
166            rusqlite::params![text::ROWID_MAP_BACKFILL_COMPLETE],
167        )?;
168        Ok(())
169    })();
170
171    match result {
172        Ok(()) => {
173            conn.execute_batch("COMMIT")?;
174            Ok(true)
175        }
176        Err(e) => {
177            let _ = conn.execute_batch("ROLLBACK");
178            Err(e)
179        }
180    }
181}
182
183/// Emit exactly one `tracing::warn!` for the whole process the first time
184/// any table falls back to scan-fallback mode, rather than once per `text()`
185/// call — `StorageBackend::text()` is called fresh on essentially every
186/// access (see `ensure_fts_rowid_map_backfilled`'s doc comment), so an
187/// unconditional warning here would spam the log on a hot path.
188fn warn_scan_fallback_once(table: &str) {
189    static WARNED: std::sync::Once = std::sync::Once::new();
190    WARNED.call_once(|| {
191        tracing::warn!(
192            table,
193            "opened a read-only text-search table with no rowid-map sidecar, or with a sidecar \
194             that has never proven a completed backfill (no durable completion marker); a \
195             read-only connection cannot create, backfill, or reconcile the map itself, so this \
196             falls back to pre-map namespace/subject_id scan predicates for get/delete on this \
197             table rather than trusting a map that might be partial"
198        );
199    });
200}
201
202fn validate_vector_model_key(model_key: &str) -> Result<(), SqliteError> {
203    if model_key.is_empty()
204        || !model_key
205            .chars()
206            .all(|c| c.is_ascii_alphanumeric() || c == '_')
207    {
208        return Err(SqliteError::InvalidData(format!(
209            "invalid model_key '{}': must be non-empty and contain only \
210             alphanumeric/underscore characters",
211            model_key
212        )));
213    }
214    Ok(())
215}
216
217fn validate_vector_table_columns(
218    conn: &rusqlite::Connection,
219    table: &str,
220) -> Result<(), SqliteError> {
221    let pragma = format!("PRAGMA table_xinfo({table})");
222    let mut stmt = conn.prepare(&pragma)?;
223    let mut rows = stmt.query([])?;
224    let mut has_field = false;
225    let mut has_embedding_model = false;
226    while let Some(row) = rows.next()? {
227        let name: String = row.get(1)?;
228        if name == "field" {
229            has_field = true;
230        }
231        if name == "embedding_model" {
232            has_embedding_model = true;
233        }
234    }
235    if !has_field || !has_embedding_model {
236        return Err(SqliteError::InvalidData(format!(
237            "vec0 table '{table}' is missing required column(s) (field={has_field}, \
238             embedding_model={has_embedding_model}); this is a pre-v0.2.8 vector schema and is \
239             not supported — recreate the database"
240        )));
241    }
242    Ok(())
243}
244
245#[derive(Clone, Copy, Debug)]
246pub(crate) enum StoreSchemaKind {
247    Entities,
248    Graph,
249    Notes,
250    Events,
251    Agents,
252}
253
254impl StoreSchemaKind {
255    fn initializer(self) -> fn(&rusqlite::Connection) -> Result<(), rusqlite::Error> {
256        match self {
257            Self::Entities => entity::ensure_entities_schema,
258            Self::Graph => graph::ensure_graph_schema,
259            Self::Notes => note::ensure_notes_schema,
260            Self::Events => event::ensure_events_schema,
261            Self::Agents => agents::ensure_agents_schema,
262        }
263    }
264}
265
266#[derive(Default)]
267pub(crate) struct StoreSchemaGate {
268    pub(crate) ready: AtomicBool,
269    #[cfg(test)]
270    pub(crate) attempts: AtomicUsize,
271}
272
273impl StoreSchemaGate {
274    pub(crate) fn ensure(
275        &self,
276        conn: &rusqlite::Connection,
277        ensure: fn(&rusqlite::Connection) -> Result<(), rusqlite::Error>,
278    ) -> Result<(), rusqlite::Error> {
279        // A concurrent initializer or repair may have completed while this
280        // caller waited for the writer. Publish readiness only after success.
281        if self.ready.load(Ordering::Acquire) {
282            return Ok(());
283        }
284        #[cfg(test)]
285        self.attempts.fetch_add(1, Ordering::Relaxed);
286        ensure(conn)?;
287        self.ready.store(true, Ordering::Release);
288        Ok(())
289    }
290}
291
292/// Concrete storage backend providing capability traits.
293///
294/// Capability schemas are initialized once per backend. An index removed by
295/// another process is repaired on the narrowly supported forced-index read paths. Other
296/// external schema changes may still require reopening the backend.
297pub struct StorageBackend {
298    pool: Arc<ConnectionPool>,
299    is_file_backed: bool,
300    path: Option<std::path::PathBuf>,
301    /// Vector model keys whose `vec_<model_key>` table this backend has already
302    /// validated or created. A key is recorded only after the whole check
303    /// succeeded under the writer, so a failed check is retried by the next
304    /// fetch and a warm fetch never takes the writer. A table another process
305    /// creates for a model this set does not hold is still validated before
306    /// use; one that is replaced after its model was recorded is not
307    /// re-checked until the backend reopens.
308    vector_tables_ready: parking_lot::Mutex<std::collections::HashSet<String>>,
309    /// FTS tables whose schema, rowid-map sidecar and durable backfill marker
310    /// this backend has already proven under the writer. A table is recorded
311    /// only once its marker exists, so a table with no rows yet is re-checked
312    /// by the next fetch, and a warm fetch never takes the writer or the
313    /// volume lease: a search opens its text store on every call. As with
314    /// `vector_tables_ready`, a table dropped behind a live backend is not
315    /// recreated until the backend is reopened.
316    text_tables_ready: parking_lot::Mutex<std::collections::HashSet<String>>,
317    /// How many times the lazy `notes_seq` anti-join repair has actually
318    /// executed against this backend's pool. Gates `notes_for_namespace` so
319    /// the repair (a full `notes` scan) runs at most once per backend for
320    /// the process's lifetime instead of on every store acquisition (khive
321    /// #827). Also exposed via
322    /// `notes_seq_repair_run_count` for regression tests.
323    notes_seq_repair_runs: AtomicUsize,
324    store_schemas: [Arc<StoreSchemaGate>; 5],
325}
326
327impl StorageBackend {
328    /// File-backed SQLite database.
329    ///
330    /// Opens (or creates) the database at `path`. An existing filesystem path
331    /// whose mode is read-only is opened with the same locked-down pool
332    /// configuration as [`Self::sqlite_read_only`]. The writable pool provides
333    /// 1 writer + N readers in WAL mode for concurrent access.
334    /// No service schema is applied — call `apply_schema()` for each service.
335    /// The pool may create its internal `_khive_database_identity` singleton
336    /// table on a writable open before service migrations run.
337    ///
338    /// The volume-lock directory is the per-user default of
339    /// [`crate::default_volume_lock_dir`], shared by every process of the user;
340    /// [`Self::sqlite_with_volume_lock_dir`] names one explicitly.
341    pub fn sqlite(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
342        Self::sqlite_with_pool_config(path, PoolConfig::default(), None)
343    }
344
345    /// A private test database with a small, explicitly sized reader pool.
346    #[cfg(any(test, feature = "test-support"))]
347    pub fn sqlite_for_test(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
348        Self::sqlite_with_pool_config(path, PoolConfig::for_test(), None)
349    }
350
351    /// Open a private test database with an explicit journal mode and busy wait.
352    #[cfg(any(test, feature = "test-support"))]
353    pub fn sqlite_for_test_with_journal_mode(
354        path: impl AsRef<Path>,
355        wal_mode: bool,
356        busy_timeout: std::time::Duration,
357    ) -> Result<Self, SqliteError> {
358        Self::sqlite_with_pool_config(
359            path,
360            PoolConfig {
361                wal_mode,
362                busy_timeout,
363                write_queue_enabled: Some(true),
364                ..PoolConfig::for_test()
365            },
366            None,
367        )
368    }
369
370    /// [`Self::sqlite_for_test_with_journal_mode`] with its volume-lock files in
371    /// `volume_lock_dir`. Under the test harness each directory takes its own
372    /// in-process lease slot, so a fixture that holds the lease across a pause
373    /// does not stall fixtures on other databases in the same test binary.
374    #[cfg(any(test, feature = "test-support"))]
375    pub fn sqlite_for_test_with_journal_mode_in(
376        path: impl AsRef<Path>,
377        wal_mode: bool,
378        busy_timeout: std::time::Duration,
379        volume_lock_dir: std::path::PathBuf,
380    ) -> Result<Self, SqliteError> {
381        Self::sqlite_with_pool_config(
382            path,
383            PoolConfig {
384                wal_mode,
385                busy_timeout,
386                write_queue_enabled: Some(true),
387                volume_lock_dir: Some(volume_lock_dir),
388                ..PoolConfig::for_test()
389            },
390            None,
391        )
392    }
393
394    /// Open SQLite with a reader count selected before any connections are opened.
395    /// `None` preserves the default pool size and filesystem read-only detection.
396    /// The volume-lock directory defaults as in [`Self::sqlite`].
397    pub fn sqlite_with_max_readers(
398        path: impl AsRef<Path>,
399        max_readers: Option<usize>,
400    ) -> Result<Self, SqliteError> {
401        Self::sqlite_with_pool_config(path, PoolConfig::default(), max_readers)
402    }
403
404    /// Open a file-backed SQLite backend with its already-resolved WAL ceiling.
405    /// A nonzero policy fails closed until the WAL I/O limiter is installed.
406    /// The volume-lock directory defaults as in [`Self::sqlite`].
407    pub fn sqlite_with_max_readers_and_wal_ceiling(
408        path: impl AsRef<Path>,
409        max_readers: Option<usize>,
410        wal_ceiling: WalCeilingPolicy,
411    ) -> Result<Self, SqliteError> {
412        Self::sqlite_with_pool_config(
413            path,
414            PoolConfig {
415                wal_ceiling,
416                ..PoolConfig::default()
417            },
418            max_readers,
419        )
420    }
421
422    pub(crate) fn sqlite_with_pool_config(
423        path: impl AsRef<Path>,
424        pool_config: PoolConfig,
425        max_readers: Option<usize>,
426    ) -> Result<Self, SqliteError> {
427        crate::extension::ensure_extensions_loaded();
428        let resolved = path.as_ref().to_path_buf();
429        let read_only =
430            std::fs::metadata(&resolved).is_ok_and(|metadata| metadata.permissions().readonly());
431        let mut config = PoolConfig {
432            path: Some(resolved.clone()),
433            read_only,
434            ..pool_config
435        };
436        if let Some(max_readers) = max_readers {
437            config.max_readers = max_readers;
438        }
439        if read_only {
440            config.write_queue_enabled = Some(false);
441        }
442        let pool = ConnectionPool::new(config)?;
443        Ok(Self {
444            pool: Arc::new(pool),
445            is_file_backed: true,
446            path: Some(resolved),
447            vector_tables_ready: Default::default(),
448            text_tables_ready: Default::default(),
449            notes_seq_repair_runs: AtomicUsize::new(0),
450            store_schemas: std::array::from_fn(|_| Arc::new(StoreSchemaGate::default())),
451        })
452    }
453
454    /// File-backed SQLite database opened read-only.
455    ///
456    /// Opens the database at `path` and sets `PRAGMA query_only = ON` on the
457    /// writer connection so that any write attempt (INSERT/UPDATE/DELETE) returns
458    /// an error. Reader connections are opened with `SQLITE_OPEN_READ_ONLY` by the
459    /// pool; at least one remains dedicated even for a rollback-journal snapshot,
460    /// while this PRAGMA extends the protection to the otherwise-unused writer slot.
461    ///
462    /// The database file must already exist — unlike `sqlite()` this constructor
463    /// does not create a new file.
464    pub fn sqlite_read_only(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
465        Self::sqlite_read_only_with_pool_config(path, PoolConfig::default(), None)
466    }
467
468    /// A private read-only test database with a small reader pool.
469    #[cfg(any(test, feature = "test-support"))]
470    pub fn sqlite_read_only_for_test(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
471        Self::sqlite_read_only_with_pool_config(path, PoolConfig::for_test(), None)
472    }
473
474    /// Open a read-only SQLite store with a construction-time reader count.
475    pub fn sqlite_read_only_with_max_readers(
476        path: impl AsRef<Path>,
477        max_readers: Option<usize>,
478    ) -> Result<Self, SqliteError> {
479        Self::sqlite_read_only_with_pool_config(path, PoolConfig::default(), max_readers)
480    }
481
482    /// Open a read-only SQLite backend while retaining WAL ceiling
483    /// configuration for diagnostics. No writer policy is enforced.
484    pub fn sqlite_read_only_with_max_readers_and_wal_ceiling(
485        path: impl AsRef<Path>,
486        max_readers: Option<usize>,
487        wal_ceiling: WalCeilingPolicy,
488    ) -> Result<Self, SqliteError> {
489        Self::sqlite_read_only_with_pool_config(
490            path,
491            PoolConfig {
492                wal_ceiling,
493                ..PoolConfig::default()
494            },
495            max_readers,
496        )
497    }
498
499    /// Open an existing live database with read-only SQLite connections.
500    ///
501    /// Ordinary WAL locking and change detection remain enabled, so a writable
502    /// shared-memory sidecar is accepted. No schema preparation is performed.
503    pub fn sqlite_read_only_live_with_max_readers_and_wal_ceiling(
504        path: impl AsRef<Path>,
505        max_readers: Option<usize>,
506        wal_ceiling: WalCeilingPolicy,
507    ) -> Result<Self, SqliteError> {
508        Self::sqlite_read_only_with_pool_constructor(
509            path,
510            PoolConfig {
511                wal_ceiling,
512                ..PoolConfig::default()
513            },
514            max_readers,
515            ConnectionPool::new_read_only_live,
516        )
517    }
518
519    fn sqlite_read_only_with_pool_config(
520        path: impl AsRef<Path>,
521        pool_config: PoolConfig,
522        max_readers: Option<usize>,
523    ) -> Result<Self, SqliteError> {
524        Self::sqlite_read_only_with_pool_constructor(
525            path,
526            pool_config,
527            max_readers,
528            ConnectionPool::new,
529        )
530    }
531
532    fn sqlite_read_only_with_pool_constructor(
533        path: impl AsRef<Path>,
534        pool_config: PoolConfig,
535        max_readers: Option<usize>,
536        open_pool: fn(PoolConfig) -> Result<ConnectionPool, SqliteError>,
537    ) -> Result<Self, SqliteError> {
538        crate::extension::ensure_extensions_loaded();
539        let resolved = path.as_ref().to_path_buf();
540        let mut config = PoolConfig {
541            path: Some(resolved.clone()),
542            read_only: true,
543            write_queue_enabled: Some(false),
544            ..pool_config
545        };
546        if let Some(max_readers) = max_readers {
547            config.max_readers = max_readers;
548        }
549        // Both pool constructors open the writer slot with `SQLITE_OPEN_READ_ONLY`
550        // (no `SQLITE_OPEN_CREATE`) and set `PRAGMA query_only = ON` on it, so a
551        // missing path is rejected instead of created, and any write attempt is
552        // rejected at the SQLite level regardless of which code path reaches the
553        // writer.
554        let pool = open_pool(config)?;
555        Ok(Self {
556            pool: Arc::new(pool),
557            is_file_backed: true,
558            path: Some(resolved),
559            vector_tables_ready: Default::default(),
560            text_tables_ready: Default::default(),
561            notes_seq_repair_runs: AtomicUsize::new(0),
562            store_schemas: std::array::from_fn(|_| Arc::new(StoreSchemaGate::default())),
563        })
564    }
565
566    /// In-memory SQLite database (for tests).
567    ///
568    /// All data is lost when the backend is dropped. The pool degrades to
569    /// single-connection mode since in-memory databases cannot be shared
570    /// across multiple connections.
571    pub fn memory() -> Result<Self, SqliteError> {
572        crate::extension::ensure_extensions_loaded();
573        let config = PoolConfig {
574            path: None,
575            ..PoolConfig::default()
576        };
577        let pool = ConnectionPool::new(config)?;
578        Ok(Self {
579            pool: Arc::new(pool),
580            is_file_backed: false,
581            path: None,
582            vector_tables_ready: Default::default(),
583            text_tables_ready: Default::default(),
584            notes_seq_repair_runs: AtomicUsize::new(0),
585            store_schemas: std::array::from_fn(|_| Arc::new(StoreSchemaGate::default())),
586        })
587    }
588
589    /// Apply a service's schema plan (run migrations).
590    ///
591    /// Each migration in the plan's `sqlite` list is applied idempotently,
592    /// including when another opener commits it first. Already-applied
593    /// migrations are skipped after taking the SQLite write lock. The
594    /// `_schema_versions` table tracks which migrations have been run.
595    pub fn apply_schema(
596        &self,
597        plan: &crate::migrations::ServiceSchemaPlan,
598    ) -> Result<(), SqliteError> {
599        let admission = self.pool.write_admission();
600        crate::migrations::apply_schema_plan_with_admission(
601            &mut self.pool.migration_transactions(),
602            plan,
603            &admission,
604        )
605    }
606
607    /// Apply pack-auxiliary DDL statements.
608    ///
609    /// Executes the full plan in one transaction, applying each DDL statement
610    /// idempotently via `execute_batch`. Each statement MUST be self-contained
611    /// and use `CREATE TABLE IF NOT EXISTS` (or equivalent idempotent DDL) so
612    /// that calling this method more than once does not fail.
613    ///
614    /// Pack auxiliary tables are NOT tracked in `_schema_versions` — they are
615    /// non-versioned. Use `apply_schema` with a `ServiceSchemaPlan` when version
616    /// tracking is needed.
617    ///
618    /// Plans declaring nullable-column upgrades must use
619    /// [`Self::apply_pack_ddl_statements_with_columns`]. The runtime supplies
620    /// the SQL slice and column metadata separately because its `SchemaPlan`
621    /// type lives above this crate in the dependency chain.
622    pub fn apply_pack_ddl_statements(
623        &self,
624        statements: &[&'static str],
625    ) -> Result<(), SqliteError> {
626        self.apply_pack_ddl_statements_with_columns(statements, &[])
627    }
628
629    /// Apply a pack's nullable-column upgrades and idempotent SQL atomically.
630    ///
631    /// Missing columns are added only to existing tables; the full SQL plan
632    /// creates fresh tables. Existing columns and the final schema must match
633    /// the declarations. Schema inspection, additions, and SQL all run under
634    /// one writer transaction, including rollback if any later step fails.
635    pub fn apply_pack_ddl_statements_with_columns(
636        &self,
637        statements: &[&'static str],
638        additions: &[khive_types::PackColumnAddition],
639    ) -> Result<(), SqliteError> {
640        let writer = self.pool.writer()?;
641        writer.transaction(|conn| {
642            pack_schema::add_missing_columns(conn, additions)?;
643            for &stmt in statements {
644                conn.execute_batch(stmt)?;
645            }
646            pack_schema::validate_columns(conn, additions)?;
647            Ok(())
648        })
649    }
650
651    /// Validate a pack's declared columns without applying SQL or acquiring a writer.
652    /// Read-only hosts use this before exposing verbs that require these columns.
653    pub fn validate_pack_schema_columns(
654        &self,
655        additions: &[khive_types::PackColumnAddition],
656    ) -> Result<(), SqliteError> {
657        if additions.is_empty() {
658            return Ok(());
659        }
660        let reader = self.pool.reader()?;
661        pack_schema::validate_columns(reader.conn(), additions)
662    }
663
664    /// Prepare the core schema for runtime boot.
665    ///
666    /// Writable backends acquire the canonical database-GC owner before the
667    /// writer, apply the ordinary versioned prefix, and may finish V21 only
668    /// through its zero-legacy-reference fast path. A legacy V20 database
669    /// remains at V20 for the async host's application-assisted attachment
670    /// cutover; this method alone is not a serving boot gate.
671    /// Read-only backends perform a query-only compatibility check and require
672    /// the snapshot to be at this build's exact latest schema version.
673    pub fn prepare_core_schema(&self) -> Result<u32, SqliteError> {
674        if self.is_read_only() {
675            let reader = self.pool.reader()?;
676            crate::migrations::validate_schema_is_current(reader.conn())
677        } else {
678            let latest = crate::migrations::MIGRATIONS
679                .last()
680                .map(|migration| migration.version)
681                .unwrap_or(0);
682            {
683                let reader = self.pool.reader()?;
684                let current = crate::migrations::read_schema_version(reader.conn())?;
685                if current >= latest {
686                    return crate::migrations::validate_schema_is_current(reader.conn());
687                }
688            }
689            let owner = crate::stores::blob::acquire_database_gc_owner_for_path_blocking(
690                self.pool.canonical_path().map(Path::to_path_buf),
691            )
692            .map_err(|error| {
693                SqliteError::InvalidData(format!(
694                    "failed to acquire database GC owner before schema preparation: {error}"
695                ))
696            })?;
697            self.run_core_migrations(&owner)
698        }
699    }
700
701    /// Inspect the coordinated V21 attachment cutover state.
702    pub fn attachment_cutover_status(
703        &self,
704    ) -> Result<crate::migrations::AttachmentCutoverStatus, SqliteError> {
705        let reader = self.pool.reader()?;
706        crate::migrations::attachment_cutover_status(reader.conn())
707    }
708
709    fn require_attachment_cutover_owner(
710        &self,
711        owner: &crate::stores::blob::DatabaseGcOwnerGuard,
712    ) -> Result<(), SqliteError> {
713        let sql = self.sql();
714        let backend_path = sql.database_path();
715        if owner.database_path() != backend_path.as_deref() {
716            return Err(SqliteError::InvalidData(format!(
717                "attachment cutover GC owner targets {:?}, but this backend is {:?}",
718                owner.database_path(),
719                backend_path.as_deref()
720            )));
721        }
722        Ok(())
723    }
724
725    /// Commit resumable V21 stage 1 while the caller owns this database's GC
726    /// protocol. The owner must remain live through verified application
727    /// backfill and finalization.
728    pub fn stage_attachment_cutover(
729        &self,
730        owner: &crate::stores::blob::DatabaseGcOwnerGuard,
731    ) -> Result<(), SqliteError> {
732        self.require_attachment_cutover_owner(owner)?;
733        if self.is_read_only() {
734            return Err(SqliteError::InvalidData(
735                "cannot stage attachment cutover on a read-only backend".into(),
736            ));
737        }
738        let mut writer = self.pool.writer()?;
739        let admission = self.pool.write_admission();
740        crate::migrations::stage_attachment_cutover_with_admission(writer.conn_mut(), &admission)
741    }
742
743    /// Atomically publish a verified batch of pack-owned attachment roles.
744    pub fn apply_verified_attachments(
745        &self,
746        owner: &crate::stores::blob::DatabaseGcOwnerGuard,
747        attachments: &[khive_storage::Attachment],
748    ) -> Result<(), SqliteError> {
749        self.require_attachment_cutover_owner(owner)?;
750        if self.is_read_only() {
751            return Err(SqliteError::InvalidData(
752                "cannot apply verified attachments on a read-only backend".into(),
753            ));
754        }
755        let mut writer = self.pool.writer()?;
756        let tx = writer
757            .conn_mut()
758            .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
759        let admission = self.pool.write_admission();
760        if let Err(error) = admission.check() {
761            let rollback = tx.rollback();
762            return Err(crate::migrations::capacity_refusal_after_rollback(
763                writer.conn(),
764                rollback,
765                error,
766                "verified attachment publication",
767            ));
768        }
769        for attachment in attachments {
770            attachment
771                .validate()
772                .map_err(|error| SqliteError::InvalidData(error.to_string()))?;
773            crate::migrations::apply_generic_verified_attachment(
774                &tx,
775                &attachment.record_uuid.to_string(),
776                attachment.substrate.as_str(),
777                &attachment.role,
778                &attachment.content_ref,
779                attachment.media_type.as_deref(),
780                attachment.size_bytes,
781                attachment.created_at,
782            )?;
783        }
784        tx.commit()?;
785        Ok(())
786    }
787
788    /// Atomically swap GC liveness/fences to attachments, remove the legacy
789    /// entity column, and record V21 while the canonical owner is held.
790    pub fn finalize_attachment_cutover(
791        &self,
792        owner: &crate::stores::blob::DatabaseGcOwnerGuard,
793    ) -> Result<(), SqliteError> {
794        self.require_attachment_cutover_owner(owner)?;
795        if self.is_read_only() {
796            return Err(SqliteError::InvalidData(
797                "cannot finalize attachment cutover on a read-only backend".into(),
798            ));
799        }
800        let mut writer = self.pool.writer()?;
801        let admission = self.pool.write_admission();
802        crate::migrations::finalize_attachment_cutover_with_admission(writer.conn_mut(), &admission)
803    }
804
805    /// Get an EntityStore. Applies the entities DDL if not already present.
806    ///
807    /// Idempotent — safe to call multiple times.
808    pub fn entities(&self) -> Result<Arc<dyn khive_storage::EntityStore>, SqliteError> {
809        self.entities_for_namespace("local")
810    }
811
812    /// Get an EntityStore. The namespace parameter is validated (non-empty) and
813    /// the entities schema is applied, but the store itself is unscoped — namespace
814    /// is the caller's responsibility on each query/delete call.
815    pub fn entities_for_namespace(
816        &self,
817        namespace: &str,
818    ) -> Result<Arc<dyn khive_storage::EntityStore>, SqliteError> {
819        if namespace.trim().is_empty() {
820            return Err(SqliteError::InvalidData(
821                "entities namespace must be non-empty".to_string(),
822            ));
823        }
824        self.ensure_store_schema(StoreSchemaKind::Entities)?;
825
826        Ok(Arc::new(entity::SqlEntityStore::new(
827            Arc::clone(&self.pool),
828            self.is_file_backed,
829        )))
830    }
831
832    /// Get the role-keyed attachment store.
833    ///
834    /// Unlike the legacy capability accessors, this does not install DDL on
835    /// demand. The coordinated V21 core cutover owns creation of the table,
836    /// reference fences, GC liveness swap, and removal of the legacy entity
837    /// column as one boot-gated operation.
838    pub fn attachments(&self) -> Result<Arc<dyn khive_storage::AttachmentStore>, SqliteError> {
839        Ok(Arc::new(attachment::SqlAttachmentStore::new(
840            Arc::clone(&self.pool),
841            self.is_file_backed,
842        )))
843    }
844
845    /// Get a GraphStore for the default namespace.
846    ///
847    /// Creates the `graph_edges` table (with indexes) if it does not already
848    /// exist. Idempotent — safe to call multiple times.
849    pub fn graph(&self) -> Result<Arc<dyn khive_storage::GraphStore>, SqliteError> {
850        self.graph_for_namespace("local")
851    }
852
853    /// Get a GraphStore scoped to a namespace.
854    pub fn graph_for_namespace(
855        &self,
856        namespace: &str,
857    ) -> Result<Arc<dyn khive_storage::GraphStore>, SqliteError> {
858        if namespace.trim().is_empty() {
859            return Err(SqliteError::InvalidData(
860                "graph namespace must be non-empty".to_string(),
861            ));
862        }
863        self.ensure_store_schema(StoreSchemaKind::Graph)?;
864
865        Ok(Arc::new(
866            graph::SqlGraphStore::new_scoped(
867                Arc::clone(&self.pool),
868                self.is_file_backed,
869                namespace.trim().to_string(),
870            )
871            .with_index_repair(crate::stores::index_repair::IndexRepairContext::new(
872                Arc::clone(&self.pool),
873                self.store_schemas.clone(),
874                crate::stores::index_repair::IndexReadKind::Graph,
875            )),
876        ))
877    }
878
879    fn ensure_store_schema(&self, kind: StoreSchemaKind) -> Result<(), SqliteError> {
880        if self.is_read_only()
881            || self.store_schemas[kind as usize]
882                .ready
883                .load(Ordering::Acquire)
884        {
885            return Ok(());
886        }
887        let writer = self.constructor_writer()?;
888        self.ensure_store_schema_with_writer(kind, writer.conn())
889    }
890
891    fn ensure_store_schema_with_writer(
892        &self,
893        kind: StoreSchemaKind,
894        conn: &rusqlite::Connection,
895    ) -> Result<(), SqliteError> {
896        self.store_schemas[kind as usize].ensure(conn, kind.initializer())?;
897        Ok(())
898    }
899
900    fn constructor_writer(
901        &self,
902    ) -> Result<crate::pool::PooledAutocommitWriteUnit<'_>, SqliteError> {
903        let context = khive_storage::capture_request_read_context();
904        let Some(operation) = context.store_acquisition_operation() else {
905            return self.pool.autocommit_write_unit();
906        };
907        let writer = self
908            .pool
909            .writer_until_for_admitted_operation(|| context.blocking_stop_reason().is_some())?
910            .ok_or_else(|| {
911                SqliteError::RequestReadStopped(khive_storage::StorageError::Timeout {
912                    operation: operation.into(),
913                })
914            })?;
915        writer.admit_autocommit()
916    }
917
918    /// Get a NoteStore. Applies the notes DDL if not already present.
919    ///
920    /// Idempotent — safe to call multiple times.
921    pub fn notes(&self) -> Result<Arc<dyn khive_storage::NoteStore>, SqliteError> {
922        self.notes_for_namespace("local")
923    }
924
925    /// Get a NoteStore. The namespace parameter is validated (non-empty) and
926    /// the notes schema is applied, but the store itself is unscoped — namespace
927    /// is the caller's responsibility on each query/delete call.
928    pub fn notes_for_namespace(
929        &self,
930        namespace: &str,
931    ) -> Result<Arc<dyn khive_storage::NoteStore>, SqliteError> {
932        if namespace.trim().is_empty() {
933            return Err(SqliteError::InvalidData(
934                "notes namespace must be non-empty".to_string(),
935            ));
936        }
937        if !self.is_read_only()
938            && (!self.store_schemas[StoreSchemaKind::Notes as usize]
939                .ready
940                .load(Ordering::Acquire)
941                || self.notes_seq_repair_runs.load(Ordering::Relaxed) == 0)
942        {
943            let writer = self.constructor_writer()?;
944            self.ensure_store_schema_with_writer(StoreSchemaKind::Notes, writer.conn())?;
945
946            // The anti-join repair is a full `notes` scan -- gate it to run at
947            // most once per backend/pool. `try_writer()` blocks for exclusive
948            // access to the single writer connection for this whole function,
949            // so this load-then-run-then-store is race-free: no other caller on
950            // this pool can observe or advance `notes_seq_repair_runs` while we
951            // hold the writer guard (khive #827).
952            if self.notes_seq_repair_runs.load(Ordering::Relaxed) == 0 {
953                note::repair_notes_seq(writer.conn())?;
954                self.notes_seq_repair_runs.fetch_add(1, Ordering::Relaxed);
955            }
956        }
957
958        Ok(Arc::new(
959            note::SqlNoteStore::new(Arc::clone(&self.pool), self.is_file_backed).with_index_repair(
960                crate::stores::index_repair::IndexRepairContext::new(
961                    Arc::clone(&self.pool),
962                    self.store_schemas.clone(),
963                    crate::stores::index_repair::IndexReadKind::Notes,
964                ),
965            ),
966        ))
967    }
968
969    /// How many times the lazy `notes_seq` anti-join repair has actually
970    /// executed against this backend's pool. Exposed for regression tests
971    /// asserting the repair runs at most once per backend for the process's
972    /// lifetime, not once per `notes_for_namespace` call (khive #827).
973    pub fn notes_seq_repair_run_count(&self) -> usize {
974        self.notes_seq_repair_runs.load(Ordering::Relaxed)
975    }
976
977    /// Get an EventStore for the default namespace.
978    ///
979    /// Creates the `events` table (with indexes) if it does not already exist.
980    /// Idempotent — safe to call multiple times.
981    pub fn events(&self) -> Result<Arc<dyn khive_storage::EventStore>, SqliteError> {
982        self.events_for_namespace("local")
983    }
984
985    /// Get an EventStore scoped to a namespace.
986    pub fn events_for_namespace(
987        &self,
988        namespace: &str,
989    ) -> Result<Arc<dyn khive_storage::EventStore>, SqliteError> {
990        if namespace.trim().is_empty() {
991            return Err(SqliteError::InvalidData(
992                "events namespace must be non-empty".to_string(),
993            ));
994        }
995        self.ensure_store_schema(StoreSchemaKind::Events)?;
996
997        Ok(Arc::new(event::SqlEventStore::new_scoped(
998            Arc::clone(&self.pool),
999            self.is_file_backed,
1000            namespace.trim().to_string(),
1001        )))
1002    }
1003
1004    /// Get the agent-process store (ADR-142 §1). Applies the agents DDL if not
1005    /// already present. Idempotent — safe to call multiple times. Unlike the
1006    /// other stores here, agent-process records are not namespace-scoped, so
1007    /// there is no `_for_namespace` variant.
1008    pub fn agents(&self) -> Result<Arc<dyn khive_storage::AgentStore>, SqliteError> {
1009        self.ensure_store_schema(StoreSchemaKind::Agents)?;
1010
1011        Ok(Arc::new(agents::SqlAgentStore::new(
1012            Arc::clone(&self.pool),
1013            self.is_file_backed,
1014        )))
1015    }
1016
1017    /// Get a VectorStore for a specific embedding model, scoped to the default namespace.
1018    ///
1019    /// Creates the vec0 virtual table if it does not already exist. The `model_key`
1020    /// must contain only ASCII alphanumeric/underscore characters. The `embedding_model`
1021    /// is the canonical display name stored in each vector row.
1022    pub fn vectors(
1023        &self,
1024        model_key: &str,
1025        embedding_model: &str,
1026        dimensions: usize,
1027    ) -> Result<Arc<dyn khive_storage::VectorStore>, SqliteError> {
1028        self.vectors_for_namespace(model_key, embedding_model, dimensions, "local")
1029    }
1030
1031    /// Get a VectorStore for a specific embedding model with a default namespace.
1032    ///
1033    /// Creates the vec0 virtual table if it does not already exist. The `namespace`
1034    /// is a default for trait methods that lack a per-call namespace parameter
1035    /// (count, delete, info). Access control is enforced at the runtime layer.
1036    ///
1037    /// The `model_key` must contain only ASCII alphanumeric/underscore characters.
1038    /// The `embedding_model` is the canonical display name stored in the `embedding_model`
1039    /// column of each vector row (e.g. `"all-minilm-l6-v2"`).
1040    pub fn vectors_for_namespace(
1041        &self,
1042        model_key: &str,
1043        embedding_model: &str,
1044        dimensions: usize,
1045        namespace: &str,
1046    ) -> Result<Arc<dyn khive_storage::VectorStore>, SqliteError> {
1047        validate_vector_model_key(model_key)?;
1048        if namespace.trim().is_empty() {
1049            return Err(SqliteError::InvalidData(
1050                "vector store namespace must be non-empty".to_string(),
1051            ));
1052        }
1053        self.ensure_vector_tables(&[(model_key, dimensions)])?;
1054        Ok(Arc::new(vectors::SqliteVecStore::new(
1055            Arc::clone(&self.pool),
1056            self.is_file_backed,
1057            model_key.to_string(),
1058            embedding_model.to_string(),
1059            dimensions,
1060            namespace.trim().to_string(),
1061        )?))
1062    }
1063
1064    /// Ensure all requested vector tables with one schema-writer acquisition.
1065    /// Read-only backends inspect the same tables using one reader instead.
1066    ///
1067    /// A writable backend prepares each model's table once: the first call that
1068    /// names it validates or creates the table under the writer and records the
1069    /// model only after every step succeeded. Later calls for a recorded model
1070    /// return without taking the writer, and a failed call records nothing, so
1071    /// the next call repeats the check.
1072    pub fn ensure_vector_tables(&self, models: &[(&str, usize)]) -> Result<(), SqliteError> {
1073        for (model_key, _) in models {
1074            validate_vector_model_key(model_key)?;
1075        }
1076        if models.is_empty() {
1077            return Ok(());
1078        }
1079
1080        // Ensure sqlite-vec is registered before creating vec0 tables.
1081        crate::extension::ensure_extensions_loaded();
1082
1083        // Direct store construction may precede migrations. Leave the
1084        // provenance sidecar to its versioned migration; vector operations
1085        // tolerate its absence until the ledger advances.
1086
1087        if self.is_read_only() {
1088            // Snapshot inspection must not check schema through the pool's
1089            // query-only writer slot: even a SELECT there is a writer-class
1090            // acquisition and violates ADR-028 A2's write-free lifecycle.
1091            let reader = self.pool.reader()?;
1092            for (model_key, _) in models {
1093                let table = format!("vec_{model_key}");
1094                if !sqlite_table_exists(reader.conn(), &table)? {
1095                    return Err(SqliteError::InvalidData(format!(
1096                        "read-only database has no vector table '{table}'; create and populate it in \
1097                         a writable copy before opening the snapshot"
1098                    )));
1099                }
1100                validate_vector_table_columns(reader.conn(), &table)?;
1101            }
1102            return Ok(());
1103        }
1104
1105        let pending = self.unprepared_vector_tables(models);
1106        if pending.is_empty() {
1107            return Ok(());
1108        }
1109        let writer = self.constructor_writer()?;
1110        // A concurrent caller may have prepared these tables while this one
1111        // waited for the writer.
1112        let pending = self.unprepared_vector_tables(&pending);
1113        if pending.is_empty() {
1114            return Ok(());
1115        }
1116
1117        // Detect old-schema vec0 tables that predate the `field` column.
1118        // Use pragma_table_info to check columns directly; substring matching on the
1119        // CREATE DDL is fragile (a model_key containing "field" would false-match).
1120        for (model_key, _) in &pending {
1121            let table = format!("vec_{model_key}");
1122            // V17 migration (vector_embedding_model_tag_preserving_rebuild) adds
1123            // `field` and `embedding_model` to all pre-existing vec0 tables at
1124            // migration time.  If this table still lacks either column post-migration
1125            // that indicates the database was not migrated — return a hard error
1126            // rather than silently dropping data.
1127            if sqlite_table_exists(writer.conn(), &table)? {
1128                validate_vector_table_columns(writer.conn(), &table)?;
1129            }
1130        }
1131
1132        // Ensure the _embedding_models registry table exists.
1133        // This is a no-op when the table already exists. Running it here ensures
1134        // the registry is present for any caller that opens a vector store without
1135        // first calling run_migrations() (e.g., tests that create stores directly).
1136        // Production callers are expected to call run_migrations() at startup, which
1137        // creates the registry via V14; this is a belt-and-suspenders fallback.
1138        // Schema is defined in `migrations::EMBEDDING_MODELS_DDL` (single source of
1139        // truth) to prevent the two copies from silently drifting.
1140        writer
1141            .conn()
1142            .execute_batch(crate::migrations::EMBEDDING_MODELS_DDL)?;
1143
1144        // Same guarantee for the ANN write log: vector write paths append to it
1145        // in the same transaction as the vec0 mutation, so it must exist in any
1146        // database that hosts vec_* tables.
1147        writer
1148            .conn()
1149            .execute_batch(crate::migrations::ANN_WRITE_LOG_DDL)?;
1150        writer
1151            .conn()
1152            .execute_batch(crate::migrations::ANN_WRITE_LOG_MODEL_SEQ_INDEX_DDL)?;
1153        writer
1154            .conn()
1155            .execute_batch(crate::migrations::ANN_CONSUMER_PENDING_DDL)?;
1156        // Create missing vec0 tables without changing existing vector data.
1157        for (model_key, dimensions) in &pending {
1158            let ddl = format!(
1159                "CREATE VIRTUAL TABLE IF NOT EXISTS vec_{} USING vec0(\
1160                 subject_id TEXT PRIMARY KEY, \
1161                 namespace TEXT NOT NULL, \
1162                 kind TEXT NOT NULL, \
1163                 field TEXT NOT NULL, \
1164                 embedding_model TEXT NOT NULL, \
1165                 embedding float[{}] distance_metric=cosine\
1166                 )",
1167                model_key, dimensions
1168            );
1169            writer.conn().execute_batch(&ddl)?;
1170        }
1171        // Record the models only now that every step above succeeded, while the
1172        // writer is still held so a waiting caller re-checks against this set.
1173        self.vector_tables_ready.lock().extend(
1174            pending
1175                .iter()
1176                .map(|(model_key, _)| (*model_key).to_string()),
1177        );
1178        Ok(())
1179    }
1180
1181    /// The requested vector models this backend has not yet validated or created.
1182    fn unprepared_vector_tables<'a>(&self, models: &[(&'a str, usize)]) -> Vec<(&'a str, usize)> {
1183        let ready = self.vector_tables_ready.lock();
1184        models
1185            .iter()
1186            .filter(|(model_key, _)| !ready.contains(*model_key))
1187            .copied()
1188            .collect()
1189    }
1190
1191    /// Register an embedding model in the `_embedding_models` registry table.
1192    ///
1193    /// Idempotent: if a row with the same `canonical_key` already exists, updates its
1194    /// status back to `'active'` without changing other fields.
1195    pub fn register_embedding_model(
1196        &self,
1197        engine_name: &str,
1198        model_id: &str,
1199        key_version: &str,
1200        dimensions: u32,
1201    ) -> Result<(), SqliteError> {
1202        let writer = self.pool.autocommit_write_unit()?;
1203        writer
1204            .conn()
1205            .execute_batch(crate::migrations::EMBEDDING_MODELS_DDL)
1206            .map_err(|error| {
1207                SqliteError::write(error, khive_storage::error::SqliteWriteStage::Statement)
1208            })?;
1209
1210        let now = chrono::Utc::now().timestamp_micros();
1211        let canonical_key =
1212            format!("{engine_name}:{model_id}:{key_version}:{dimensions}").into_bytes();
1213        let id = uuid::Uuid::new_v4();
1214        writer
1215            .conn()
1216            .execute(
1217                "INSERT INTO _embedding_models \
1218             (id, engine_name, model_id, key_version, dim, output_dim, status, \
1219              activated_at, superseded_at, superseded_by, canonical_key, created_at) \
1220             VALUES (?1, ?2, ?3, ?4, ?5, NULL, 'active', ?6, NULL, NULL, ?7, ?8) \
1221             ON CONFLICT(canonical_key) DO UPDATE SET \
1222                status = 'active', \
1223                activated_at = COALESCE(_embedding_models.activated_at, excluded.activated_at)",
1224                rusqlite::params![
1225                    id.as_bytes().as_slice(),
1226                    engine_name,
1227                    model_id,
1228                    key_version,
1229                    dimensions as i64,
1230                    now,
1231                    canonical_key,
1232                    now,
1233                ],
1234            )
1235            .map_err(|error| {
1236                SqliteError::write(error, khive_storage::error::SqliteWriteStage::Statement)
1237            })?;
1238        Ok(())
1239    }
1240
1241    /// Get a SparseStore for a specific model key, scoped to the default namespace.
1242    ///
1243    /// Creates the sparse table if it does not already exist.
1244    pub fn sparse(
1245        &self,
1246        model_key: &str,
1247    ) -> Result<Arc<dyn khive_storage::SparseStore>, SqliteError> {
1248        self.sparse_for_namespace(model_key, "local")
1249    }
1250
1251    /// Get a SparseStore for a specific model key with an explicit default namespace.
1252    ///
1253    /// The `model_key` must contain only ASCII alphanumeric/underscore characters.
1254    pub fn sparse_for_namespace(
1255        &self,
1256        model_key: &str,
1257        namespace: &str,
1258    ) -> Result<Arc<dyn khive_storage::SparseStore>, SqliteError> {
1259        if model_key.is_empty()
1260            || !model_key
1261                .chars()
1262                .all(|c| c.is_ascii_alphanumeric() || c == '_')
1263        {
1264            return Err(SqliteError::InvalidData(format!(
1265                "invalid model_key '{}': must be non-empty and contain only alphanumeric/underscore characters",
1266                model_key
1267            )));
1268        }
1269        if namespace.trim().is_empty() {
1270            return Err(SqliteError::InvalidData(
1271                "sparse store namespace must be non-empty".to_string(),
1272            ));
1273        }
1274
1275        if self.is_read_only() {
1276            let table = format!("sparse_{model_key}");
1277            let reader = self.pool.reader()?;
1278            if !sqlite_table_exists(reader.conn(), &table)? {
1279                return Err(SqliteError::InvalidData(format!(
1280                    "read-only database has no sparse table '{table}'; create and populate it in \
1281                     a writable copy before opening the snapshot"
1282                )));
1283            }
1284        } else {
1285            let writer = self.pool.autocommit_write_unit()?;
1286            sparse::ensure_sparse_schema(writer.conn(), model_key)
1287                .map_err(SqliteError::Rusqlite)?;
1288        }
1289
1290        Ok(Arc::new(sparse::SqliteSparseStore::new(
1291            Arc::clone(&self.pool),
1292            self.is_file_backed,
1293            model_key.to_string(),
1294            namespace.trim().to_string(),
1295        )?))
1296    }
1297
1298    /// Get a TextSearch for a specific table key.
1299    ///
1300    /// Creates the FTS5 virtual table if it does not already exist. Uses the
1301    /// `trigram` tokenizer by default (CJK-safe).
1302    ///
1303    /// The `table_key` must contain only ASCII alphanumeric/underscore characters.
1304    pub fn text(&self, table_key: &str) -> Result<Arc<dyn khive_storage::TextSearch>, SqliteError> {
1305        self.text_with_tokenizer(table_key, "trigram")
1306    }
1307
1308    /// Get a TextSearch with an explicit FTS5 tokenizer.
1309    ///
1310    /// Use when you need a tokenizer other than the default `trigram` — for
1311    /// example `unicode61` for Latin-only corpora.
1312    ///
1313    /// Both `table_key` and `tokenizer` must contain only ASCII
1314    /// alphanumeric/underscore characters.
1315    pub fn text_with_tokenizer(
1316        &self,
1317        table_key: &str,
1318        tokenizer: &str,
1319    ) -> Result<Arc<dyn khive_storage::TextSearch>, SqliteError> {
1320        if table_key.is_empty()
1321            || !table_key
1322                .chars()
1323                .all(|c| c.is_ascii_alphanumeric() || c == '_')
1324        {
1325            return Err(SqliteError::InvalidData(format!(
1326                "invalid table_key '{}': must be non-empty and contain only \
1327                 alphanumeric/underscore characters",
1328                table_key
1329            )));
1330        }
1331        // `text::rowid_map_table`/`text::rowid_map_state_table` name a
1332        // table's sidecar map `{table}_rowids` and that map's own completion
1333        // marker `{table}_rowids_state` — a `table_key` ending in either
1334        // reserved suffix (e.g. "entities_rowids" or "entities_rowids_state")
1335        // would resolve to the exact sidecar table name another key's own
1336        // map or marker already reserves. `CREATE VIRTUAL TABLE IF NOT
1337        // EXISTS` would then silently accept that ordinary (non-FTS5) table
1338        // as if it were this key's FTS table, and every later point
1339        // read/write against it would fail against the wrong schema.
1340        if table_key.ends_with("_rowids") || table_key.ends_with("_rowids_state") {
1341            return Err(SqliteError::InvalidData(format!(
1342                "invalid table_key '{}': must not end in '_rowids' or '_rowids_state' — those \
1343                 suffixes are reserved for a text table's own rowid-map sidecar and its \
1344                 completion-marker state table (see text::rowid_map_table, \
1345                 text::rowid_map_state_table)",
1346                table_key
1347            )));
1348        }
1349        if tokenizer.is_empty()
1350            || !tokenizer
1351                .chars()
1352                .all(|c| c.is_ascii_alphanumeric() || c == '_')
1353        {
1354            return Err(SqliteError::InvalidData(format!(
1355                "invalid tokenizer '{}': must be non-empty and contain only \
1356                 alphanumeric/underscore characters",
1357                tokenizer
1358            )));
1359        }
1360
1361        let ddl = format!(
1362            "CREATE VIRTUAL TABLE IF NOT EXISTS fts_{} USING fts5(\
1363             subject_id UNINDEXED, \
1364             kind UNINDEXED, \
1365             title, \
1366             body, \
1367             tags UNINDEXED, \
1368             namespace UNINDEXED, \
1369             metadata UNINDEXED, \
1370             updated_at UNINDEXED, \
1371             record_kind, \
1372             tokenize = '{}'\
1373             )",
1374            table_key, tokenizer
1375        );
1376        let table = format!("fts_{table_key}");
1377        if self.is_read_only() {
1378            let reader = self.pool.reader()?;
1379            if !sqlite_table_exists(reader.conn(), &table)? {
1380                return Err(SqliteError::InvalidData(format!(
1381                    "read-only database has no text-search table '{table}'; create and populate \
1382                     it in a writable copy before opening the snapshot"
1383                )));
1384            }
1385            // A read-only connection cannot create, backfill, or reconcile
1386            // the rowid-map sidecar: a snapshot taken before this migration
1387            // shipped can have the FTS table without its map at all, and a
1388            // snapshot taken mid-backfill (a crash, or a copy made between
1389            // the map's creation and its completion marker being written) can
1390            // have a map table that is only partially populated. Trusting an
1391            // unproven map here would silently hide live rows the map
1392            // doesn't yet know about — only a map with a durable completion
1393            // marker (`ROWID_MAP_BACKFILL_COMPLETE`) is safe to join against
1394            // read-only; anything else falls back to the pre-map scan
1395            // predicates rather than constructing a store whose get/delete
1396            // would either fail against a nonexistent sidecar or silently
1397            // miss rows an unreconciled map doesn't cover.
1398            let map = text::rowid_map_table(&table);
1399            let map_exists = sqlite_table_exists(reader.conn(), &map)?;
1400            let marker_present = if map_exists {
1401                let state = text::rowid_map_state_table(&table);
1402                sqlite_table_exists(reader.conn(), &state)?
1403                    && reader.conn().query_row(
1404                        &format!(
1405                            "SELECT EXISTS(SELECT 1 FROM {state} WHERE key = 'backfill' AND value = ?1)"
1406                        ),
1407                        rusqlite::params![text::ROWID_MAP_BACKFILL_COMPLETE],
1408                        |row| row.get(0),
1409                    )?
1410            } else {
1411                false
1412            };
1413            if !map_exists || !marker_present {
1414                warn_scan_fallback_once(&table);
1415                return Ok(Arc::new(text::Fts5TextSearch::new_scan_fallback(
1416                    Arc::clone(&self.pool),
1417                    self.is_file_backed,
1418                    table_key.to_string(),
1419                )));
1420            }
1421        } else if !self.text_tables_ready.lock().contains(&table) {
1422            let writer = self.pool.autocommit_write_unit()?;
1423            // A concurrent caller may have proven this table while this one
1424            // waited for the writer.
1425            if !self.text_tables_ready.lock().contains(&table) {
1426                writer.conn().execute_batch(&ddl)?;
1427                writer.conn().execute_batch(&text::rowid_map_ddl(&table))?;
1428                let marker_durable = ensure_fts_rowid_map_backfilled(
1429                    writer.conn(),
1430                    &table,
1431                    &self.pool.write_admission(),
1432                )?;
1433                // Record the table only now that every step succeeded, while
1434                // the writer is still held.
1435                if marker_durable {
1436                    self.text_tables_ready.lock().insert(table.clone());
1437                }
1438            }
1439        }
1440
1441        Ok(Arc::new(text::Fts5TextSearch::new(
1442            Arc::clone(&self.pool),
1443            self.is_file_backed,
1444            table_key.to_string(),
1445        )))
1446    }
1447
1448    /// Get a `BlobStore` rooted per khive#292's precedence chain:
1449    /// `KHIVE_BLOB_ROOT` env var > `config_root` (a caller-resolved
1450    /// `khive.toml` override — `khive-db` has no TOML parser of its own) >
1451    /// beside this backend's database directory. `floor_bytes` overrides the
1452    /// default fail-closed free-space floor (`None` keeps the default, 2% of
1453    /// the volume and at least 5 GiB; ADR-111 Amendment 6). Errors if none of
1454    /// the three roots apply — e.g. an in-memory
1455    /// backend with no override and no env var has nowhere to default to.
1456    pub fn blob_store(
1457        &self,
1458        config_root: Option<&Path>,
1459        floor_bytes: Option<u64>,
1460    ) -> Result<Arc<dyn khive_storage::BlobStore>, SqliteError> {
1461        Ok(Arc::new(self.fs_blob_store(config_root, floor_bytes)?))
1462    }
1463
1464    /// The concrete store behind [`Self::blob_store`], so its resolved floor
1465    /// can be read back.
1466    pub(crate) fn fs_blob_store(
1467        &self,
1468        config_root: Option<&Path>,
1469        floor_bytes: Option<u64>,
1470    ) -> Result<blob::FsBlobStore, SqliteError> {
1471        let root = blob::resolve_blob_root(self.data_dir().as_deref(), config_root)?;
1472        let floor = blob::FsBlobStore::resolve_floor(&root, floor_bytes);
1473        blob::FsBlobStore::new(root, floor)
1474    }
1475
1476    /// Resolve the filesystem blob root exactly like [`Self::blob_store`] but
1477    /// require it to exist instead of creating it. Snapshot runtimes wrap the
1478    /// returned capability so its read methods remain available while every
1479    /// mutator is refused.
1480    pub fn blob_store_read_only(
1481        &self,
1482        config_root: Option<&Path>,
1483        floor_bytes: Option<u64>,
1484    ) -> Result<Arc<dyn khive_storage::BlobStore>, SqliteError> {
1485        let root = blob::resolve_blob_root(self.data_dir().as_deref(), config_root)?;
1486        let floor = blob::FsBlobStore::resolve_floor(&root, floor_bytes);
1487        Ok(Arc::new(blob::FsBlobStore::open_existing(root, floor)?))
1488    }
1489
1490    /// Is this a file-backed backend?
1491    pub fn is_file_backed(&self) -> bool {
1492        self.is_file_backed
1493    }
1494
1495    /// Whether this backend was opened with SQLite's read-only/query-only
1496    /// contract, explicitly or after filesystem-mode detection.
1497    pub fn is_read_only(&self) -> bool {
1498        self.pool.config().read_only
1499    }
1500
1501    /// Return the directory containing the backend's database file, or `None`
1502    /// for an in-memory backend.
1503    pub fn data_dir(&self) -> Option<std::path::PathBuf> {
1504        self.path.as_ref()?.parent().map(|p| p.to_path_buf())
1505    }
1506
1507    /// Root directory for this database's ANN segment tree, or `None` for an
1508    /// in-memory backend. Derived from the database file name itself
1509    /// (`<db-file>.ann/` beside the file), so two databases sharing a parent
1510    /// directory can never adopt each other's segments or UUID maps. The
1511    /// suffix is appended at the `OsString` byte level — a lossy UTF-8
1512    /// conversion would collapse distinct non-UTF-8 filenames into one
1513    /// replacement-character root, breaking exactly that isolation.
1514    pub fn ann_root(&self) -> Option<std::path::PathBuf> {
1515        ann_root_for(self.path.as_ref()?)
1516    }
1517
1518    /// Access the underlying pool (escape hatch).
1519    pub fn pool(&self) -> &ConnectionPool {
1520        &self.pool
1521    }
1522
1523    /// Durable owner evidence for this backend, captured at pool construction.
1524    /// The caller must choose the effective main backend before comparing it.
1525    pub fn database_owner_identity(
1526        &self,
1527    ) -> Result<DatabaseOwnerIdentity, DatabaseOwnerIdentityError> {
1528        self.pool.database_owner_identity()
1529    }
1530
1531    /// Refuse a secondary or copied database as the supplied database owner.
1532    pub fn verify_database_owner(
1533        &self,
1534        expected: &DatabaseOwnerIdentity,
1535    ) -> Result<(), DatabaseOwnerIdentityError> {
1536        self.pool.verify_database_owner(expected)
1537    }
1538
1539    /// Clone the underlying pool Arc.
1540    pub fn pool_arc(&self) -> Arc<ConnectionPool> {
1541        Arc::clone(&self.pool)
1542    }
1543}
1544
1545/// `<db-file>.ann` sibling of a database file, appended at the `OsString`
1546/// byte level: a lossy UTF-8 conversion would collapse distinct non-UTF-8
1547/// filenames into one replacement-character root, breaking the per-database
1548/// segment isolation that `ann_root` exists to guarantee.
1549fn ann_root_for(path: &std::path::Path) -> Option<std::path::PathBuf> {
1550    let mut file = path.file_name()?.to_os_string();
1551    file.push(".ann");
1552    path.parent().map(|p| p.join(file))
1553}
1554
1555#[cfg(test)]
1556#[path = "backend/store_accessor_tests.rs"]
1557mod store_accessor_tests;
1558
1559#[cfg(test)]
1560#[path = "backend/store_accessor_index_tests.rs"]
1561mod store_accessor_index_tests;
1562
1563#[cfg(test)]
1564#[path = "backend_tests.rs"]
1565mod tests;
1566
1567#[cfg(test)]
1568#[path = "backend_admission_tests.rs"]
1569mod admission_tests;
1570
1571#[cfg(test)]
1572mod live_readonly_tests;