Skip to main content

paper/core/index/
connection.rs

1use anyhow::{Context, Result};
2use rusqlite::OptionalExtension;
3use std::path::Path;
4use std::sync::Once;
5
6static REGISTER_VEC: Once = Once::new();
7
8/// Register `sqlite-vec` as a SQLite auto-extension exactly once per process.
9///
10/// The C extension is compiled statically into the binary (no runtime `.dylib`
11/// download, no `load_extension` call), which keeps `paper` a single binary.
12#[allow(clippy::missing_transmute_annotations)]
13pub fn register_sqlite_vec() {
14    REGISTER_VEC.call_once(|| unsafe {
15        // SAFETY: `sqlite_vec::sqlite3_vec_init` has the SQLite extension entry
16        // point signature, `sqlite3_extension_init`:
17        //     int (*sqlite3_extension_init)(sqlite3 *db, char **pzErrMsg,
18        //                                   const sqlite3_api_routines *pThunk)
19        // `sqlite3_auto_extension` registers entry points of type
20        // `void (*)(void)`. The sqlite-vec crate documents transmuting
21        // `sqlite3_vec_init` to `void (*)(void)` as the supported way to register
22        // a statically-linked extension. Both are `extern "C"` fn pointers of the
23        // same width, so the transmute is sound: SQLite invokes the entry point
24        // with the extension-init argument layout that `sqlite3_vec_init`
25        // expects. Registration runs once per process via `REGISTER_VEC`.
26        rusqlite::ffi::sqlite3_auto_extension(Some(std::mem::transmute(
27            sqlite_vec::sqlite3_vec_init as *const (),
28        )));
29    });
30}
31
32/// Thin wrapper around `rusqlite::Connection` that guarantees the sqlite-vec
33/// extension is available and applies Paper's pragmas.
34pub struct Connection {
35    conn: rusqlite::Connection,
36}
37
38impl Connection {
39    /// Open an existing database file, running `PRAGMA quick_check` to verify
40    /// integrity before any work proceeds. If the file doesn't exist, it is
41    /// created with creation-time pragmas (page_size, etc.).
42    ///
43    /// `quick_check` is O(database size) and only runs on the first open after
44    /// creation; subsequent opens skip it since WAL mode guarantees crash
45    /// safety (I4: avoids per-search latency on large indexes).
46    pub fn open(path: &Path) -> Result<Self> {
47        register_sqlite_vec();
48        let existed = path.exists();
49        if let Some(parent) = path.parent() {
50            std::fs::create_dir_all(parent)
51                .with_context(|| format!("creating db directory {}", parent.display()))?;
52        }
53        let conn = rusqlite::Connection::open(path)
54            .with_context(|| format!("opening {}", path.display()))?;
55        if !existed {
56            // Fresh database: quick_check is instant on an empty file.
57            let quick_check: String = conn.query_row("PRAGMA quick_check", [], |row| row.get(0))?;
58            if quick_check != "ok" {
59                anyhow::bail!("database quick_check failed: {quick_check}");
60            }
61        }
62        Self::finish(conn, !existed) // set page_size only on fresh files
63    }
64
65    /// Open a database file that will be created if it doesn't exist. Runs
66    /// `PRAGMA quick_check` on the new/empty file (always "ok") and applies
67    /// creation-time pragmas like `page_size`.
68    ///
69    /// This is a convenience alias for `open`; both now handle fresh files.
70    pub fn open_or_create(path: &Path) -> Result<Self> {
71        Self::open(path)
72    }
73
74    pub fn open_in_memory() -> Result<Self> {
75        register_sqlite_vec();
76        let conn = rusqlite::Connection::open_in_memory()?;
77        Self::finish(conn, false) // in-memory DB cannot set page_size
78    }
79
80    /// Apply Paper's standard pragmas. `allow_page_size` controls whether
81    /// `PRAGMA page_size = 8192` is executed (only on fresh files, never on
82    /// existing databases — changing it requires VACUUM).
83    fn finish(conn: rusqlite::Connection, allow_page_size: bool) -> Result<Self> {
84        conn.busy_timeout(std::time::Duration::from_secs(5))?;
85        // WAL lets searches run while the indexer writes.
86        conn.execute_batch(
87            "PRAGMA journal_mode = WAL;
88             PRAGMA synchronous = NORMAL;
89             PRAGMA foreign_keys = ON;
90             PRAGMA mmap_size = 268435456;
91             PRAGMA wal_autocheckpoint = 1000;",
92        )?;
93        if allow_page_size {
94            // page_size must be set before any tables are created; it only takes
95            // effect on a fresh file. On an existing DB this is a no-op.
96            conn.execute_batch("PRAGMA page_size = 8192;")?;
97        }
98        Ok(Self { conn })
99    }
100
101    pub fn inner(&self) -> &rusqlite::Connection {
102        &self.conn
103    }
104
105    pub fn inner_mut(&mut self) -> &mut rusqlite::Connection {
106        &mut self.conn
107    }
108
109    pub fn execute_batch(&self, sql: &str) -> Result<()> {
110        self.conn.execute_batch(sql)?;
111        Ok(())
112    }
113
114    /// Convenience for scalar queries used by doctor/tests.
115    pub fn query_i64(&self, sql: &str) -> Result<i64> {
116        Ok(self.conn.query_row(sql, [], |row| row.get(0))?)
117    }
118
119    pub fn query_row<T, P: rusqlite::Params>(
120        &self,
121        sql: &str,
122        params: P,
123        f: impl FnOnce(&rusqlite::Row<'_>) -> rusqlite::Result<T>,
124    ) -> Result<T> {
125        Ok(self.conn.query_row(sql, params, f)?)
126    }
127
128    /// Get the embedding model ID stored in the index, if any.
129    pub fn stored_embedding_model(&self) -> Result<Option<String>> {
130        let model: Option<String> = self
131            .conn
132            .query_row(
133                "SELECT value FROM index_meta WHERE key = 'embedding_model'",
134                [],
135                |row| row.get(0),
136            )
137            .optional()?;
138        Ok(model)
139    }
140}