second_brain_core/sqlite_store/
pool.rs1use std::cell::Cell;
2use std::marker::PhantomData;
3use std::path::{Path, PathBuf};
4use std::sync::{Arc, Condvar, Mutex, MutexGuard};
5use std::thread;
6use std::time::{Duration, Instant};
7
8use anyhow::{Context, Result, anyhow, ensure};
9use rusqlite::{Connection, ErrorCode, OptionalExtension};
10use zeroize::Zeroizing;
11
12use crate::keys::StoreKey;
13
14pub(super) const BUSY_TIMEOUT: Duration = Duration::from_secs(5);
15
16pub(super) fn open_connection(path: &Path, key: Option<&StoreKey>) -> Result<Connection> {
22 let conn = Connection::open(path)
23 .with_context(|| format!("opening sqlite db at {}", path.display()))?;
24 conn.busy_timeout(BUSY_TIMEOUT)
25 .context("arming busy_timeout")?;
26 if let Some(key) = key {
27 apply_key(&conn, key, path)?;
28 }
29 set_wal_mode(&conn)?;
30 conn.pragma_update(None, "synchronous", "NORMAL")
31 .context("setting synchronous=NORMAL")?;
32 Ok(conn)
33}
34
35pub(super) fn apply_key(conn: &Connection, key: &StoreKey, path: &Path) -> Result<()> {
39 require_sqlcipher(conn)
40 .with_context(|| format!("opening the encrypted store at {}", path.display()))?;
41 let pragma = Zeroizing::new(format!("PRAGMA key = \"{}\";", key.sql_literal().as_str()));
42 conn.execute_batch(&pragma)
43 .context("setting the store key")?;
44 match conn.query_row("SELECT count(*) FROM sqlite_master", [], |r| {
45 r.get::<_, i64>(0)
46 }) {
47 Ok(_) => Ok(()),
48 Err(e) if e.sqlite_error_code() == Some(ErrorCode::NotADatabase) => Err(anyhow!(
49 "the store key does not open {}; it is plaintext or under another key",
50 path.display()
51 )),
52 Err(e) => Err(e).with_context(|| format!("reading {} under the store key", path.display())),
53 }
54}
55
56pub(super) fn require_sqlcipher(conn: &Connection) -> Result<()> {
57 let version: Option<String> = conn
58 .query_row("PRAGMA cipher_version", [], |r| r.get(0))
59 .optional()
60 .context("reading cipher_version")?;
61 ensure!(
62 version.is_some_and(|v| !v.is_empty()),
63 "this build has no SQLCipher; build second-brain-core with the `encryption` feature"
64 );
65 Ok(())
66}
67
68pub(crate) fn page_size(conn: &Connection) -> Result<i64> {
71 let value: rusqlite::types::Value = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?;
72 match value {
73 rusqlite::types::Value::Integer(n) => Ok(n),
74 rusqlite::types::Value::Text(text) => text
75 .trim()
76 .parse()
77 .with_context(|| format!("PRAGMA page_size answered {text:?}")),
78 other => Err(anyhow!("PRAGMA page_size answered {other:?}")),
79 }
80}
81
82fn set_wal_mode(conn: &Connection) -> Result<()> {
90 let deadline = Instant::now() + BUSY_TIMEOUT;
91 loop {
92 match conn.pragma_update(None, "journal_mode", "WAL") {
93 Ok(()) => return Ok(()),
94 Err(err) if is_database_busy(&err) && Instant::now() < deadline => {
95 thread::sleep(Duration::from_millis(20));
96 }
97 Err(err) => return Err(err).context("setting journal_mode=WAL"),
98 }
99 }
100}
101
102fn is_database_busy(err: &rusqlite::Error) -> bool {
103 err.sqlite_error_code() == Some(ErrorCode::DatabaseBusy)
104}
105
106pub(super) fn default_pool_size() -> usize {
110 std::thread::available_parallelism()
111 .map(|n| n.get())
112 .unwrap_or(4)
113 .clamp(2, 16)
114}
115
116thread_local! {
117 static READ_DEADLINE: Cell<Option<Instant>> = const { Cell::new(None) };
118}
119
120pub struct ReadDeadline {
129 previous: Option<Instant>,
130 not_send: PhantomData<*const ()>,
131}
132
133impl ReadDeadline {
134 pub fn set(deadline: Instant) -> Self {
135 ReadDeadline {
136 previous: READ_DEADLINE.replace(Some(deadline)),
137 not_send: PhantomData,
138 }
139 }
140}
141
142impl Drop for ReadDeadline {
143 fn drop(&mut self) {
144 READ_DEADLINE.set(self.previous);
145 }
146}
147
148pub(super) struct ReaderPool {
154 path: PathBuf,
155 size: usize,
156 inner: Mutex<PoolState>,
157 returned: Condvar,
158}
159
160struct PoolState {
161 free: Vec<Connection>,
162 key: Option<Arc<StoreKey>>,
163 spares: usize,
166 draining: bool,
169}
170
171impl ReaderPool {
172 pub(super) fn open(path: &Path, size: usize, key: Option<Arc<StoreKey>>) -> Result<Self> {
173 let free = (0..size)
174 .map(|_| open_connection(path, key.as_deref()))
175 .collect::<Result<Vec<_>>>()?;
176 Ok(Self {
177 path: path.to_path_buf(),
178 size,
179 inner: Mutex::new(PoolState {
180 free,
181 key,
182 spares: 0,
183 draining: false,
184 }),
185 returned: Condvar::new(),
186 })
187 }
188
189 pub(super) fn path(&self) -> &Path {
190 &self.path
191 }
192
193 pub(super) fn key(&self) -> Option<Arc<StoreKey>> {
194 self.lock().key.clone()
195 }
196
197 pub(super) fn checkout(&self) -> PooledConn<'_> {
200 let mut state = self.lock();
201 if let Some(deadline) = READ_DEADLINE.get() {
202 let wait = deadline.saturating_duration_since(Instant::now());
203 state = self
204 .returned
205 .wait_timeout_while(state, wait, |s| s.free.is_empty() || s.draining)
206 .unwrap_or_else(|e| e.into_inner())
207 .0;
208 if state.free.is_empty() && !state.draining {
209 state.spares += 1;
210 let key = state.key.clone();
211 drop(state);
212 match open_connection(&self.path, key.as_deref()) {
213 Ok(conn) => {
214 return PooledConn {
215 conn: Some(conn),
216 pool: self,
217 spare: true,
218 };
219 }
220 Err(e) => {
221 tracing::warn!("no read connection to spare past a deadline: {e:#}");
222 state = self.lock();
223 state.spares -= 1;
224 self.returned.notify_all();
225 }
226 }
227 }
228 }
229 let mut state = self
230 .returned
231 .wait_while(state, |s| s.free.is_empty() || s.draining)
232 .unwrap_or_else(|e| e.into_inner());
233 PooledConn {
234 conn: state.free.pop(),
235 pool: self,
236 spare: false,
237 }
238 }
239
240 fn lock(&self) -> MutexGuard<'_, PoolState> {
241 self.inner.lock().unwrap_or_else(|e| e.into_inner())
242 }
243
244 pub(super) fn exclusive(&self, wait: Duration) -> Option<Exclusive<'_>> {
248 let mut state = self.lock();
249 state.draining = true;
250 let (mut state, _) = self
251 .returned
252 .wait_timeout_while(state, wait, |s| s.free.len() < self.size || s.spares > 0)
253 .unwrap_or_else(|e| e.into_inner());
254 if state.free.len() < self.size || state.spares > 0 {
255 state.draining = false;
256 self.returned.notify_all();
257 return None;
258 }
259 Some(Exclusive { pool: self, state })
260 }
261}
262
263pub(super) struct Exclusive<'a> {
264 pool: &'a ReaderPool,
265 state: MutexGuard<'a, PoolState>,
266}
267
268impl Exclusive<'_> {
269 pub(super) fn close_all(&mut self) {
270 self.state.free.clear();
271 }
272
273 pub(super) fn refill(mut self, key: Option<Arc<StoreKey>>) -> Result<()> {
275 let opened = (0..self.pool.size)
276 .map(|_| open_connection(&self.pool.path, key.as_deref()))
277 .collect::<Result<Vec<_>>>();
278 match opened {
279 Ok(conns) => {
280 self.state.free = conns;
281 self.state.key = key;
282 Ok(())
283 }
284 Err(e) => self.fail_reads(e),
285 }
286 }
287
288 pub(super) fn fail_reads(mut self, cause: anyhow::Error) -> Result<()> {
292 let placeholders = (0..self.pool.size)
293 .map(|_| unusable_connection())
294 .collect::<Result<Vec<_>>>();
295 match placeholders {
296 Ok(conns) => {
297 self.state.free = conns;
298 Err(cause)
299 }
300 Err(p) => Err(cause.context(format!("{p:#}"))),
301 }
302 }
303}
304
305impl Drop for Exclusive<'_> {
306 fn drop(&mut self) {
307 self.state.draining = false;
308 self.pool.returned.notify_all();
309 }
310}
311
312pub(super) fn unusable_connection() -> Result<Connection> {
315 let conn = Connection::open_in_memory()?;
316 conn.pragma_update(None, "query_only", true)?;
317 Ok(conn)
318}
319
320pub(crate) struct PooledConn<'a> {
323 conn: Option<Connection>,
324 pool: &'a ReaderPool,
325 spare: bool,
326}
327
328impl std::ops::Deref for PooledConn<'_> {
329 type Target = Connection;
330
331 fn deref(&self) -> &Connection {
332 self.conn.as_ref().expect("taken only in Drop")
333 }
334}
335
336impl Drop for PooledConn<'_> {
337 fn drop(&mut self) {
338 let Some(conn) = self.conn.take() else {
339 return;
340 };
341 if self.spare {
342 drop(conn);
343 self.pool.lock().spares -= 1;
344 } else {
345 self.pool.lock().free.push(conn);
346 }
347 self.pool.returned.notify_all();
348 }
349}
350
351#[cfg(test)]
352mod tests {
353 use std::sync::mpsc;
354 use std::time::Duration;
355
356 use super::*;
357
358 fn pool(size: usize) -> (tempfile::TempDir, ReaderPool) {
359 let dir = tempfile::tempdir().unwrap();
360 let pool = ReaderPool::open(&dir.path().join("brain.db"), size, None).unwrap();
361 (dir, pool)
362 }
363
364 #[test]
365 fn a_read_past_its_deadline_opens_a_spare_connection_instead_of_waiting() {
366 let (_dir, pool) = pool(1);
367 let held = pool.checkout();
368 let (done, answered) = mpsc::channel();
369 thread::scope(|scope| {
370 scope.spawn(|| {
371 let _bound = ReadDeadline::set(Instant::now() + Duration::from_millis(50));
372 let spare = pool.checkout();
373 let one: i64 = spare.query_row("SELECT 1", [], |r| r.get(0)).unwrap();
374 done.send(one).unwrap();
375 });
376 let answer = answered.recv_timeout(Duration::from_secs(10));
377 drop(held);
378 assert_eq!(
379 answer,
380 Ok(1),
381 "the bounded read waited for the held connection"
382 );
383 });
384 assert_eq!(
385 pool.lock().free.len(),
386 1,
387 "only the pooled connection came back"
388 );
389 assert_eq!(pool.lock().spares, 0);
390 }
391
392 #[test]
393 fn a_read_without_a_deadline_waits_for_a_pooled_connection() {
394 let (_dir, pool) = pool(1);
395 let held = pool.checkout();
396 let (done, answered) = mpsc::channel();
397 thread::scope(|scope| {
398 scope.spawn(|| {
399 let conn = pool.checkout();
400 done.send(conn.is_autocommit()).unwrap();
401 });
402 assert!(answered.recv_timeout(Duration::from_millis(100)).is_err());
403 drop(held);
404 assert!(answered.recv_timeout(Duration::from_secs(10)).is_ok());
405 });
406 }
407
408 #[test]
409 fn a_deadline_guard_restores_the_one_before_it() {
410 let outer = Instant::now() + Duration::from_secs(60);
411 let _outer = ReadDeadline::set(outer);
412 {
413 let _inner = ReadDeadline::set(Instant::now());
414 }
415 assert_eq!(READ_DEADLINE.get(), Some(outer));
416 }
417}