second-brain-core 0.13.0

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
use std::cell::Cell;
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Condvar, Mutex, MutexGuard};
use std::thread;
use std::time::{Duration, Instant};

use anyhow::{Context, Result, anyhow, ensure};
use rusqlite::{Connection, ErrorCode, OptionalExtension};
use zeroize::Zeroizing;

use crate::keys::StoreKey;

pub(super) const BUSY_TIMEOUT: Duration = Duration::from_secs(5);

/// Opens one connection with the pragmas every connection in the store needs.
/// `busy_timeout` is armed before any pragma: switching journal_mode takes a
/// brief exclusive lock, so two processes opening the same fresh database
/// concurrently can otherwise hit a hard SQLITE_BUSY with no busy handler
/// installed yet to retry it.
pub(super) fn open_connection(path: &Path, key: Option<&StoreKey>) -> Result<Connection> {
    let conn = Connection::open(path)
        .with_context(|| format!("opening sqlite db at {}", path.display()))?;
    conn.busy_timeout(BUSY_TIMEOUT)
        .context("arming busy_timeout")?;
    if let Some(key) = key {
        apply_key(&conn, key, path)?;
    }
    set_wal_mode(&conn)?;
    conn.pragma_update(None, "synchronous", "NORMAL")
        .context("setting synchronous=NORMAL")?;
    Ok(conn)
}

/// Plain SQLite ignores `PRAGMA key` without an error, which would write
/// a keyed store's pages in plaintext, so a build without SQLCipher is
/// refused before the key is set.
pub(super) fn apply_key(conn: &Connection, key: &StoreKey, path: &Path) -> Result<()> {
    require_sqlcipher(conn)
        .with_context(|| format!("opening the encrypted store at {}", path.display()))?;
    let pragma = Zeroizing::new(format!("PRAGMA key = \"{}\";", key.sql_literal().as_str()));
    conn.execute_batch(&pragma)
        .context("setting the store key")?;
    match conn.query_row("SELECT count(*) FROM sqlite_master", [], |r| {
        r.get::<_, i64>(0)
    }) {
        Ok(_) => Ok(()),
        Err(e) if e.sqlite_error_code() == Some(ErrorCode::NotADatabase) => Err(anyhow!(
            "the store key does not open {}; it is plaintext or under another key",
            path.display()
        )),
        Err(e) => Err(e).with_context(|| format!("reading {} under the store key", path.display())),
    }
}

pub(super) fn require_sqlcipher(conn: &Connection) -> Result<()> {
    let version: Option<String> = conn
        .query_row("PRAGMA cipher_version", [], |r| r.get(0))
        .optional()
        .context("reading cipher_version")?;
    ensure!(
        version.is_some_and(|v| !v.is_empty()),
        "this build has no SQLCipher; build second-brain-core with the `encryption` feature"
    );
    Ok(())
}

/// SQLCipher answers `PRAGMA page_size` on a keyed connection with its
/// `cipher_page_size` setting, as text.
pub(crate) fn page_size(conn: &Connection) -> Result<i64> {
    let value: rusqlite::types::Value = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?;
    match value {
        rusqlite::types::Value::Integer(n) => Ok(n),
        rusqlite::types::Value::Text(text) => text
            .trim()
            .parse()
            .with_context(|| format!("PRAGMA page_size answered {text:?}")),
        other => Err(anyhow!("PRAGMA page_size answered {other:?}")),
    }
}

/// `busy_timeout`'s built-in retry loop does not cover the exclusive lock
/// SQLite takes while switching journal_mode to WAL: two connections racing
/// to make that switch on the same fresh database can still see a bare
/// SQLITE_BUSY from the pragma itself, confirmed against this rusqlite
/// version by a concurrent-open regression test. Retry it by hand, bounded
/// by the same window `busy_timeout` uses, instead of trusting the driver to
/// cover a case it does not.
fn set_wal_mode(conn: &Connection) -> Result<()> {
    let deadline = Instant::now() + BUSY_TIMEOUT;
    loop {
        match conn.pragma_update(None, "journal_mode", "WAL") {
            Ok(()) => return Ok(()),
            Err(err) if is_database_busy(&err) && Instant::now() < deadline => {
                thread::sleep(Duration::from_millis(20));
            }
            Err(err) => return Err(err).context("setting journal_mode=WAL"),
        }
    }
}

fn is_database_busy(err: &rusqlite::Error) -> bool {
    err.sqlite_error_code() == Some(ErrorCode::DatabaseBusy)
}

/// A sane default reader pool size: enough connections that concurrent
/// readers stop queueing behind each other, bounded so a store never opens
/// more file descriptors than the machine has cores to use them with.
pub(super) fn default_pool_size() -> usize {
    std::thread::available_parallelism()
        .map(|n| n.get())
        .unwrap_or(4)
        .clamp(2, 16)
}

thread_local! {
    static READ_DEADLINE: Cell<Option<Instant>> = const { Cell::new(None) };
}

/// Bounds the wait for a pooled read connection on this thread until the
/// guard drops: a read that finds the pool empty waits at most until
/// `deadline`, then opens a connection of its own, which closes on drop.
/// That open is bounded by `BUSY_TIMEOUT`, not by the deadline, and if it
/// fails the read waits for the pool as an unbounded read would.
/// Thread-local because every store read takes a connection through
/// `reader()` and a recall's reads all run on the one thread that holds the
/// guard; the guard is not `Send`, so it cannot be dropped on another.
pub struct ReadDeadline {
    previous: Option<Instant>,
    not_send: PhantomData<*const ()>,
}

impl ReadDeadline {
    pub fn set(deadline: Instant) -> Self {
        ReadDeadline {
            previous: READ_DEADLINE.replace(Some(deadline)),
            not_send: PhantomData,
        }
    }
}

impl Drop for ReadDeadline {
    fn drop(&mut self) {
        READ_DEADLINE.set(self.previous);
    }
}

/// Fixed-size pool of read connections to the same file: `checkout` waits
/// for a connection to come back rather than ever pooling one past `size`.
/// WAL gives one writer and many concurrent readers across separate
/// connections, so each pooled connection is a full physical connection,
/// not a wrapper over the writer's.
pub(super) struct ReaderPool {
    path: PathBuf,
    size: usize,
    inner: Mutex<PoolState>,
    returned: Condvar,
}

struct PoolState {
    free: Vec<Connection>,
    key: Option<Arc<StoreKey>>,
    /// Connections opened past a read deadline that are still open; they
    /// hold the file as much as pooled ones do.
    spares: usize,
    /// While set, no checkout takes a connection or opens a spare, so the
    /// file can be swapped once every connection has come back.
    draining: bool,
}

impl ReaderPool {
    pub(super) fn open(path: &Path, size: usize, key: Option<Arc<StoreKey>>) -> Result<Self> {
        let free = (0..size)
            .map(|_| open_connection(path, key.as_deref()))
            .collect::<Result<Vec<_>>>()?;
        Ok(Self {
            path: path.to_path_buf(),
            size,
            inner: Mutex::new(PoolState {
                free,
                key,
                spares: 0,
                draining: false,
            }),
            returned: Condvar::new(),
        })
    }

    pub(super) fn path(&self) -> &Path {
        &self.path
    }

    pub(super) fn key(&self) -> Option<Arc<StoreKey>> {
        self.lock().key.clone()
    }

    /// A read waits while the pool drains for a swap, even past its
    /// deadline: a spare opened then would keep the old file open.
    pub(super) fn checkout(&self) -> PooledConn<'_> {
        let mut state = self.lock();
        if let Some(deadline) = READ_DEADLINE.get() {
            let wait = deadline.saturating_duration_since(Instant::now());
            state = self
                .returned
                .wait_timeout_while(state, wait, |s| s.free.is_empty() || s.draining)
                .unwrap_or_else(|e| e.into_inner())
                .0;
            if state.free.is_empty() && !state.draining {
                state.spares += 1;
                let key = state.key.clone();
                drop(state);
                match open_connection(&self.path, key.as_deref()) {
                    Ok(conn) => {
                        return PooledConn {
                            conn: Some(conn),
                            pool: self,
                            spare: true,
                        };
                    }
                    Err(e) => {
                        tracing::warn!("no read connection to spare past a deadline: {e:#}");
                        state = self.lock();
                        state.spares -= 1;
                        self.returned.notify_all();
                    }
                }
            }
        }
        let mut state = self
            .returned
            .wait_while(state, |s| s.free.is_empty() || s.draining)
            .unwrap_or_else(|e| e.into_inner());
        PooledConn {
            conn: state.free.pop(),
            pool: self,
            spare: false,
        }
    }

    fn lock(&self) -> MutexGuard<'_, PoolState> {
        self.inner.lock().unwrap_or_else(|e| e.into_inner())
    }

    /// Stops new checkouts, waits up to `wait` for every connection, pooled
    /// or spare, to come back, and then holds them until the guard drops.
    /// None, with checkouts resumed, when one stays out past `wait`.
    pub(super) fn exclusive(&self, wait: Duration) -> Option<Exclusive<'_>> {
        let mut state = self.lock();
        state.draining = true;
        let (mut state, _) = self
            .returned
            .wait_timeout_while(state, wait, |s| s.free.len() < self.size || s.spares > 0)
            .unwrap_or_else(|e| e.into_inner());
        if state.free.len() < self.size || state.spares > 0 {
            state.draining = false;
            self.returned.notify_all();
            return None;
        }
        Some(Exclusive { pool: self, state })
    }
}

pub(super) struct Exclusive<'a> {
    pool: &'a ReaderPool,
    state: MutexGuard<'a, PoolState>,
}

impl Exclusive<'_> {
    pub(super) fn close_all(&mut self) {
        self.state.free.clear();
    }

    /// Opens a full pool on the file under `key`, or fails its reads.
    pub(super) fn refill(mut self, key: Option<Arc<StoreKey>>) -> Result<()> {
        let opened = (0..self.pool.size)
            .map(|_| open_connection(&self.pool.path, key.as_deref()))
            .collect::<Result<Vec<_>>>();
        match opened {
            Ok(conns) => {
                self.state.free = conns;
                self.state.key = key;
                Ok(())
            }
            Err(e) => self.fail_reads(e),
        }
    }

    /// Fills the pool with placeholder connections that fail every read, so
    /// no checkout waits forever on a store that could not reopen; returns
    /// `cause`.
    pub(super) fn fail_reads(mut self, cause: anyhow::Error) -> Result<()> {
        let placeholders = (0..self.pool.size)
            .map(|_| unusable_connection())
            .collect::<Result<Vec<_>>>();
        match placeholders {
            Ok(conns) => {
                self.state.free = conns;
                Err(cause)
            }
            Err(p) => Err(cause.context(format!("{p:#}"))),
        }
    }
}

impl Drop for Exclusive<'_> {
    fn drop(&mut self) {
        self.state.draining = false;
        self.pool.returned.notify_all();
    }
}

/// A connection that answers every statement with an error, for a store
/// whose file could not be reopened.
pub(super) fn unusable_connection() -> Result<Connection> {
    let conn = Connection::open_in_memory()?;
    conn.pragma_update(None, "query_only", true)?;
    Ok(conn)
}

/// A connection from the pool, or a spare opened past a read deadline,
/// which closes on drop instead of joining the pool.
pub(crate) struct PooledConn<'a> {
    conn: Option<Connection>,
    pool: &'a ReaderPool,
    spare: bool,
}

impl std::ops::Deref for PooledConn<'_> {
    type Target = Connection;

    fn deref(&self) -> &Connection {
        self.conn.as_ref().expect("taken only in Drop")
    }
}

impl Drop for PooledConn<'_> {
    fn drop(&mut self) {
        let Some(conn) = self.conn.take() else {
            return;
        };
        if self.spare {
            drop(conn);
            self.pool.lock().spares -= 1;
        } else {
            self.pool.lock().free.push(conn);
        }
        self.pool.returned.notify_all();
    }
}

#[cfg(test)]
mod tests {
    use std::sync::mpsc;
    use std::time::Duration;

    use super::*;

    fn pool(size: usize) -> (tempfile::TempDir, ReaderPool) {
        let dir = tempfile::tempdir().unwrap();
        let pool = ReaderPool::open(&dir.path().join("brain.db"), size, None).unwrap();
        (dir, pool)
    }

    #[test]
    fn a_read_past_its_deadline_opens_a_spare_connection_instead_of_waiting() {
        let (_dir, pool) = pool(1);
        let held = pool.checkout();
        let (done, answered) = mpsc::channel();
        thread::scope(|scope| {
            scope.spawn(|| {
                let _bound = ReadDeadline::set(Instant::now() + Duration::from_millis(50));
                let spare = pool.checkout();
                let one: i64 = spare.query_row("SELECT 1", [], |r| r.get(0)).unwrap();
                done.send(one).unwrap();
            });
            let answer = answered.recv_timeout(Duration::from_secs(10));
            drop(held);
            assert_eq!(
                answer,
                Ok(1),
                "the bounded read waited for the held connection"
            );
        });
        assert_eq!(
            pool.lock().free.len(),
            1,
            "only the pooled connection came back"
        );
        assert_eq!(pool.lock().spares, 0);
    }

    #[test]
    fn a_read_without_a_deadline_waits_for_a_pooled_connection() {
        let (_dir, pool) = pool(1);
        let held = pool.checkout();
        let (done, answered) = mpsc::channel();
        thread::scope(|scope| {
            scope.spawn(|| {
                let conn = pool.checkout();
                done.send(conn.is_autocommit()).unwrap();
            });
            assert!(answered.recv_timeout(Duration::from_millis(100)).is_err());
            drop(held);
            assert!(answered.recv_timeout(Duration::from_secs(10)).is_ok());
        });
    }

    #[test]
    fn a_deadline_guard_restores_the_one_before_it() {
        let outer = Instant::now() + Duration::from_secs(60);
        let _outer = ReadDeadline::set(outer);
        {
            let _inner = ReadDeadline::set(Instant::now());
        }
        assert_eq!(READ_DEADLINE.get(), Some(outer));
    }
}