second_brain_core/sqlite_store/
compaction.rs1use std::ffi::CString;
2use std::os::unix::ffi::OsStrExt;
3use std::os::unix::fs::MetadataExt;
4use std::path::{Path, PathBuf};
5
6use anyhow::{Context, Result, bail};
7use rusqlite::{OptionalExtension, params};
8use serde::Serialize;
9
10use super::SqliteStore;
11use super::pool::BUSY_TIMEOUT;
12
13const COMPACTED_KEY: &str = "stored_redaction_v1_compacted";
14const VACUUM_BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600);
15const SPACE_MARGIN: u64 = 16 * 1024 * 1024;
16
17#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
20pub struct VacuumSpace {
21 pub path: PathBuf,
22 pub needed_bytes: u64,
23 pub free_bytes: u64,
24}
25
26impl VacuumSpace {
27 pub fn fits(&self) -> bool {
28 self.free_bytes >= self.needed_bytes
29 }
30}
31
32impl SqliteStore {
33 pub fn vacuum(&self) -> Result<(u64, u64)> {
36 self.vacuum_then(false)
37 }
38
39 pub fn vacuum_clearing_wal(&self) -> Result<(u64, u64)> {
44 self.vacuum_then(true)
45 }
46
47 fn vacuum_then(&self, truncate_wal: bool) -> Result<(u64, u64)> {
48 let conn = self.conn();
49 let size = |sql: &str| -> Result<u64> {
50 let page_size: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?;
51 let pages: i64 = conn.query_row(sql, [], |r| r.get(0))?;
52 Ok((page_size * pages) as u64)
53 };
54 let before = size("PRAGMA page_count")?;
55 conn.busy_timeout(VACUUM_BUSY_TIMEOUT)?;
56 let vacuumed = conn.execute_batch("VACUUM;").and_then(|()| {
57 if !truncate_wal {
58 return Ok(0);
59 }
60 conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| {
61 r.get::<_, i64>(0)
62 })
63 });
64 conn.busy_timeout(BUSY_TIMEOUT)?;
65 if vacuumed.context("vacuuming the store")? != 0 {
66 bail!("the WAL checkpoint after VACUUM was blocked by readers");
67 }
68 let after = size("PRAGMA page_count")?;
69 Ok((before, after))
70 }
71
72 pub fn vacuum_space(&self) -> Result<VacuumSpace> {
75 let Some(db) = self.db_path()? else {
76 return Ok(VacuumSpace {
77 path: PathBuf::new(),
78 needed_bytes: 0,
79 free_bytes: 0,
80 });
81 };
82 let live = self.live_bytes()?;
83 let copy = live + live / 16 + SPACE_MARGIN;
84 let db_dir = db
85 .parent()
86 .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
87 let temp_dir = sqlite_temp_dir();
88 let mut needs = vec![(db_dir.clone(), copy)];
89 if same_filesystem(&db_dir, &temp_dir)? {
90 needs[0].1 += copy;
91 } else {
92 needs.push((temp_dir, copy));
93 }
94 let mut tightest: Option<VacuumSpace> = None;
95 for (path, needed_bytes) in needs {
96 let free_bytes = fs4::available_space(&path)
97 .with_context(|| format!("reading free space at {}", path.display()))?;
98 let space = VacuumSpace {
99 path,
100 needed_bytes,
101 free_bytes,
102 };
103 let headroom = |s: &VacuumSpace| s.free_bytes as i128 - s.needed_bytes as i128;
104 if tightest
105 .as_ref()
106 .is_none_or(|t| headroom(&space) < headroom(t))
107 {
108 tightest = Some(space);
109 }
110 }
111 Ok(tightest.expect("the database directory is always checked"))
112 }
113
114 pub fn stored_redaction_compacted(&self) -> Result<bool> {
115 let value: Option<String> = self
116 .reader()
117 .query_row(
118 "SELECT value FROM index_meta WHERE key = ?1",
119 params![COMPACTED_KEY],
120 |r| r.get(0),
121 )
122 .optional()?;
123 Ok(value.as_deref() == Some("done"))
124 }
125
126 pub fn mark_stored_redaction_compacted(&self) -> Result<()> {
127 self.conn().execute(
128 "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, 'done')",
129 params![COMPACTED_KEY],
130 )?;
131 Ok(())
132 }
133
134 fn db_path(&self) -> Result<Option<PathBuf>> {
135 let file: String = self
136 .reader()
137 .query_row("PRAGMA database_list", [], |r| r.get(2))?;
138 Ok((!file.is_empty()).then(|| PathBuf::from(file)))
139 }
140
141 fn live_bytes(&self) -> Result<u64> {
142 let conn = self.reader();
143 let page_size: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?;
144 let pages: i64 = conn.query_row("PRAGMA page_count", [], |r| r.get(0))?;
145 let free: i64 = conn.query_row("PRAGMA freelist_count", [], |r| r.get(0))?;
146 Ok((pages - free).max(0) as u64 * page_size as u64)
147 }
148}
149
150fn sqlite_temp_dir() -> PathBuf {
153 let from_env = ["SQLITE_TMPDIR", "TMPDIR"]
154 .into_iter()
155 .filter_map(|var| std::env::var_os(var).map(PathBuf::from));
156 let fixed = ["/var/tmp", "/usr/tmp", "/tmp", "."]
157 .into_iter()
158 .map(PathBuf::from);
159 from_env
160 .chain(fixed)
161 .find(|dir| usable_temp_dir(dir))
162 .unwrap_or_else(|| PathBuf::from("."))
163}
164
165fn usable_temp_dir(dir: &Path) -> bool {
166 if !dir.is_dir() {
167 return false;
168 }
169 let Ok(path) = CString::new(dir.as_os_str().as_bytes()) else {
170 return false;
171 };
172 unsafe { libc::access(path.as_ptr(), libc::W_OK | libc::X_OK) == 0 }
173}
174
175fn same_filesystem(a: &Path, b: &Path) -> Result<bool> {
176 let dev = |p: &Path| {
177 std::fs::metadata(p)
178 .map(|m| m.dev())
179 .with_context(|| format!("reading {}", p.display()))
180 };
181 Ok(dev(a)? == dev(b)?)
182}
183
184#[cfg(test)]
185mod tests {
186 use super::*;
187
188 #[test]
189 fn an_in_memory_store_needs_no_space() {
190 let store = SqliteStore::in_memory("m".to_string()).unwrap();
191 assert!(store.vacuum_space().unwrap().fits());
192 }
193
194 #[test]
195 fn a_shortfall_does_not_fit() {
196 let space = VacuumSpace {
197 path: PathBuf::from("/"),
198 needed_bytes: 10,
199 free_bytes: 9,
200 };
201 assert!(!space.fits());
202 }
203
204 #[test]
205 fn the_temp_dir_is_one_sqlite_can_write() {
206 assert!(usable_temp_dir(&sqlite_temp_dir()));
207 }
208}