Skip to main content

second_brain_core/sqlite_store/
compaction.rs

1use std::ffi::CString;
2use std::os::unix::ffi::OsStrExt;
3use std::os::unix::fs::MetadataExt;
4use std::path::{Path, PathBuf};
5
6use anyhow::{Context, Result, bail};
7use rusqlite::{OptionalExtension, params};
8use serde::Serialize;
9
10use super::SqliteStore;
11use super::encrypt_migration::RewriteTo;
12use super::pool::BUSY_TIMEOUT;
13
14const COMPACTED_KEY: &str = "stored_redaction_v1_compacted";
15const VACUUM_BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600);
16pub(super) const SPACE_MARGIN: u64 = 16 * 1024 * 1024;
17
18/// Free space against what a VACUUM of this store needs, on the filesystem
19/// with the least room to spare.
20#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
21pub struct VacuumSpace {
22    pub path: PathBuf,
23    pub needed_bytes: u64,
24    pub free_bytes: u64,
25}
26
27impl VacuumSpace {
28    pub fn fits(&self) -> bool {
29        self.free_bytes >= self.needed_bytes
30    }
31}
32
33impl SqliteStore {
34    /// `VACUUM` cannot run inside a transaction and needs the writer alone;
35    /// returns (bytes before, bytes after).
36    pub fn vacuum(&self) -> Result<(u64, u64)> {
37        self.vacuum_then(false)
38    }
39
40    /// `vacuum`, then a checkpoint that truncates the WAL, because the frames
41    /// VACUUM appended, and older ones past them, would otherwise keep every
42    /// page's previous contents on disk. Errors when readers keep the
43    /// checkpoint from finishing, though the VACUUM itself went through.
44    pub fn vacuum_clearing_wal(&self) -> Result<(u64, u64)> {
45        self.vacuum_then(true)
46    }
47
48    /// A store never goes from keyed back to plaintext, so one found
49    /// plaintext under the writer stays so for the VACUUM.
50    fn vacuum_then(&self, truncate_wal: bool) -> Result<(u64, u64)> {
51        let conn = self.conn();
52        if self.is_encrypted() {
53            drop(conn);
54            let rewrite = self.rewrite_under_key(RewriteTo::CurrentKey)?;
55            return Ok((rewrite.bytes_before, rewrite.bytes_after));
56        }
57        let size = |sql: &str| -> Result<u64> {
58            let page_size = super::pool::page_size(&conn)?;
59            let pages: i64 = conn.query_row(sql, [], |r| r.get(0))?;
60            Ok((page_size * pages) as u64)
61        };
62        let before = size("PRAGMA page_count")?;
63        conn.busy_timeout(VACUUM_BUSY_TIMEOUT)?;
64        // SQLCipher builds default temp_store to memory, which would hold
65        // VACUUM's whole copy of the store in RAM.
66        let vacuumed = conn
67            .execute_batch("PRAGMA temp_store = FILE; VACUUM; PRAGMA temp_store = DEFAULT;")
68            .and_then(|()| {
69                if !truncate_wal {
70                    return Ok(0);
71                }
72                conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| {
73                    r.get::<_, i64>(0)
74                })
75            });
76        conn.busy_timeout(BUSY_TIMEOUT)?;
77        if vacuumed.context("vacuuming the store")? != 0 {
78            bail!("the WAL checkpoint after VACUUM was blocked by readers");
79        }
80        let after = size("PRAGMA page_count")?;
81        Ok((before, after))
82    }
83
84    /// VACUUM writes a full copy of the live pages to SQLite's temp
85    /// directory and then the same again into the WAL, so both need room.
86    pub fn vacuum_space(&self) -> Result<VacuumSpace> {
87        let Some(db) = self.file_path()? else {
88            return Ok(VacuumSpace {
89                path: PathBuf::new(),
90                needed_bytes: 0,
91                free_bytes: 0,
92            });
93        };
94        let live = self.live_bytes()?;
95        let copy = live + live / 16 + SPACE_MARGIN;
96        let db_dir = db
97            .parent()
98            .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
99        if self.is_encrypted() {
100            return self.rewrite_space();
101        }
102        let temp_dir = sqlite_temp_dir();
103        let mut needs = vec![(db_dir.clone(), copy)];
104        if same_filesystem(&db_dir, &temp_dir)? {
105            needs[0].1 += copy;
106        } else {
107            needs.push((temp_dir, copy));
108        }
109        let mut tightest: Option<VacuumSpace> = None;
110        for (path, needed_bytes) in needs {
111            let free_bytes = fs4::available_space(&path)
112                .with_context(|| format!("reading free space at {}", path.display()))?;
113            let space = VacuumSpace {
114                path,
115                needed_bytes,
116                free_bytes,
117            };
118            let headroom = |s: &VacuumSpace| s.free_bytes as i128 - s.needed_bytes as i128;
119            if tightest
120                .as_ref()
121                .is_none_or(|t| headroom(&space) < headroom(t))
122            {
123                tightest = Some(space);
124            }
125        }
126        Ok(tightest.expect("the database directory is always checked"))
127    }
128
129    pub fn stored_redaction_compacted(&self) -> Result<bool> {
130        let value: Option<String> = self
131            .reader()
132            .query_row(
133                "SELECT value FROM index_meta WHERE key = ?1",
134                params![COMPACTED_KEY],
135                |r| r.get(0),
136            )
137            .optional()?;
138        Ok(value.as_deref() == Some("done"))
139    }
140
141    pub fn mark_stored_redaction_compacted(&self) -> Result<()> {
142        self.conn().execute(
143            "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, 'done')",
144            params![COMPACTED_KEY],
145        )?;
146        Ok(())
147    }
148
149    pub(super) fn live_bytes(&self) -> Result<u64> {
150        let conn = self.reader();
151        let page_size = super::pool::page_size(&conn)?;
152        let pages: i64 = conn.query_row("PRAGMA page_count", [], |r| r.get(0))?;
153        let free: i64 = conn.query_row("PRAGMA freelist_count", [], |r| r.get(0))?;
154        Ok((pages - free).max(0) as u64 * page_size as u64)
155    }
156}
157
158/// The directory SQLite's unix VFS picks for temp files, in its own order:
159/// the first candidate that is a directory this process can write and enter.
160fn sqlite_temp_dir() -> PathBuf {
161    let from_env = ["SQLITE_TMPDIR", "TMPDIR"]
162        .into_iter()
163        .filter_map(|var| std::env::var_os(var).map(PathBuf::from));
164    let fixed = ["/var/tmp", "/usr/tmp", "/tmp", "."]
165        .into_iter()
166        .map(PathBuf::from);
167    from_env
168        .chain(fixed)
169        .find(|dir| usable_temp_dir(dir))
170        .unwrap_or_else(|| PathBuf::from("."))
171}
172
173fn usable_temp_dir(dir: &Path) -> bool {
174    if !dir.is_dir() {
175        return false;
176    }
177    let Ok(path) = CString::new(dir.as_os_str().as_bytes()) else {
178        return false;
179    };
180    unsafe { libc::access(path.as_ptr(), libc::W_OK | libc::X_OK) == 0 }
181}
182
183fn same_filesystem(a: &Path, b: &Path) -> Result<bool> {
184    let dev = |p: &Path| {
185        std::fs::metadata(p)
186            .map(|m| m.dev())
187            .with_context(|| format!("reading {}", p.display()))
188    };
189    Ok(dev(a)? == dev(b)?)
190}
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195
196    #[test]
197    fn an_in_memory_store_needs_no_space() {
198        let store = SqliteStore::in_memory("m".to_string()).unwrap();
199        assert!(store.vacuum_space().unwrap().fits());
200    }
201
202    #[test]
203    fn a_shortfall_does_not_fit() {
204        let space = VacuumSpace {
205            path: PathBuf::from("/"),
206            needed_bytes: 10,
207            free_bytes: 9,
208        };
209        assert!(!space.fits());
210    }
211
212    #[test]
213    fn the_temp_dir_is_one_sqlite_can_write() {
214        assert!(usable_temp_dir(&sqlite_temp_dir()));
215    }
216}