second_brain_core/sqlite_store/
compaction.rs1use std::ffi::CString;
2use std::os::unix::ffi::OsStrExt;
3use std::os::unix::fs::MetadataExt;
4use std::path::{Path, PathBuf};
5
6use anyhow::{Context, Result, bail};
7use rusqlite::{OptionalExtension, params};
8use serde::Serialize;
9
10use super::SqliteStore;
11use super::encrypt_migration::RewriteTo;
12use super::pool::BUSY_TIMEOUT;
13
14const COMPACTED_KEY: &str = "stored_redaction_v1_compacted";
15const VACUUM_BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600);
16pub(super) const SPACE_MARGIN: u64 = 16 * 1024 * 1024;
17
18#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
21pub struct VacuumSpace {
22 pub path: PathBuf,
23 pub needed_bytes: u64,
24 pub free_bytes: u64,
25}
26
27impl VacuumSpace {
28 pub fn fits(&self) -> bool {
29 self.free_bytes >= self.needed_bytes
30 }
31}
32
33impl SqliteStore {
34 pub fn vacuum(&self) -> Result<(u64, u64)> {
37 self.vacuum_then(false)
38 }
39
40 pub fn vacuum_clearing_wal(&self) -> Result<(u64, u64)> {
45 self.vacuum_then(true)
46 }
47
48 fn vacuum_then(&self, truncate_wal: bool) -> Result<(u64, u64)> {
51 let conn = self.conn();
52 if self.is_encrypted() {
53 drop(conn);
54 let rewrite = self.rewrite_under_key(RewriteTo::CurrentKey)?;
55 return Ok((rewrite.bytes_before, rewrite.bytes_after));
56 }
57 let size = |sql: &str| -> Result<u64> {
58 let page_size = super::pool::page_size(&conn)?;
59 let pages: i64 = conn.query_row(sql, [], |r| r.get(0))?;
60 Ok((page_size * pages) as u64)
61 };
62 let before = size("PRAGMA page_count")?;
63 conn.busy_timeout(VACUUM_BUSY_TIMEOUT)?;
64 let vacuumed = conn
67 .execute_batch("PRAGMA temp_store = FILE; VACUUM; PRAGMA temp_store = DEFAULT;")
68 .and_then(|()| {
69 if !truncate_wal {
70 return Ok(0);
71 }
72 conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| {
73 r.get::<_, i64>(0)
74 })
75 });
76 conn.busy_timeout(BUSY_TIMEOUT)?;
77 if vacuumed.context("vacuuming the store")? != 0 {
78 bail!("the WAL checkpoint after VACUUM was blocked by readers");
79 }
80 let after = size("PRAGMA page_count")?;
81 Ok((before, after))
82 }
83
84 pub fn vacuum_space(&self) -> Result<VacuumSpace> {
87 let Some(db) = self.file_path()? else {
88 return Ok(VacuumSpace {
89 path: PathBuf::new(),
90 needed_bytes: 0,
91 free_bytes: 0,
92 });
93 };
94 let live = self.live_bytes()?;
95 let copy = live + live / 16 + SPACE_MARGIN;
96 let db_dir = db
97 .parent()
98 .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
99 if self.is_encrypted() {
100 return self.rewrite_space();
101 }
102 let temp_dir = sqlite_temp_dir();
103 let mut needs = vec![(db_dir.clone(), copy)];
104 if same_filesystem(&db_dir, &temp_dir)? {
105 needs[0].1 += copy;
106 } else {
107 needs.push((temp_dir, copy));
108 }
109 let mut tightest: Option<VacuumSpace> = None;
110 for (path, needed_bytes) in needs {
111 let free_bytes = fs4::available_space(&path)
112 .with_context(|| format!("reading free space at {}", path.display()))?;
113 let space = VacuumSpace {
114 path,
115 needed_bytes,
116 free_bytes,
117 };
118 let headroom = |s: &VacuumSpace| s.free_bytes as i128 - s.needed_bytes as i128;
119 if tightest
120 .as_ref()
121 .is_none_or(|t| headroom(&space) < headroom(t))
122 {
123 tightest = Some(space);
124 }
125 }
126 Ok(tightest.expect("the database directory is always checked"))
127 }
128
129 pub fn stored_redaction_compacted(&self) -> Result<bool> {
130 let value: Option<String> = self
131 .reader()
132 .query_row(
133 "SELECT value FROM index_meta WHERE key = ?1",
134 params![COMPACTED_KEY],
135 |r| r.get(0),
136 )
137 .optional()?;
138 Ok(value.as_deref() == Some("done"))
139 }
140
141 pub fn mark_stored_redaction_compacted(&self) -> Result<()> {
142 self.conn().execute(
143 "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, 'done')",
144 params![COMPACTED_KEY],
145 )?;
146 Ok(())
147 }
148
149 pub(super) fn live_bytes(&self) -> Result<u64> {
150 let conn = self.reader();
151 let page_size = super::pool::page_size(&conn)?;
152 let pages: i64 = conn.query_row("PRAGMA page_count", [], |r| r.get(0))?;
153 let free: i64 = conn.query_row("PRAGMA freelist_count", [], |r| r.get(0))?;
154 Ok((pages - free).max(0) as u64 * page_size as u64)
155 }
156}
157
158fn sqlite_temp_dir() -> PathBuf {
161 let from_env = ["SQLITE_TMPDIR", "TMPDIR"]
162 .into_iter()
163 .filter_map(|var| std::env::var_os(var).map(PathBuf::from));
164 let fixed = ["/var/tmp", "/usr/tmp", "/tmp", "."]
165 .into_iter()
166 .map(PathBuf::from);
167 from_env
168 .chain(fixed)
169 .find(|dir| usable_temp_dir(dir))
170 .unwrap_or_else(|| PathBuf::from("."))
171}
172
173fn usable_temp_dir(dir: &Path) -> bool {
174 if !dir.is_dir() {
175 return false;
176 }
177 let Ok(path) = CString::new(dir.as_os_str().as_bytes()) else {
178 return false;
179 };
180 unsafe { libc::access(path.as_ptr(), libc::W_OK | libc::X_OK) == 0 }
181}
182
183fn same_filesystem(a: &Path, b: &Path) -> Result<bool> {
184 let dev = |p: &Path| {
185 std::fs::metadata(p)
186 .map(|m| m.dev())
187 .with_context(|| format!("reading {}", p.display()))
188 };
189 Ok(dev(a)? == dev(b)?)
190}
191
192#[cfg(test)]
193mod tests {
194 use super::*;
195
196 #[test]
197 fn an_in_memory_store_needs_no_space() {
198 let store = SqliteStore::in_memory("m".to_string()).unwrap();
199 assert!(store.vacuum_space().unwrap().fits());
200 }
201
202 #[test]
203 fn a_shortfall_does_not_fit() {
204 let space = VacuumSpace {
205 path: PathBuf::from("/"),
206 needed_bytes: 10,
207 free_bytes: 9,
208 };
209 assert!(!space.fits());
210 }
211
212 #[test]
213 fn the_temp_dir_is_one_sqlite_can_write() {
214 assert!(usable_temp_dir(&sqlite_temp_dir()));
215 }
216}