Skip to main content

second_brain_core/sqlite_store/
compaction.rs

1use std::ffi::CString;
2use std::os::unix::ffi::OsStrExt;
3use std::os::unix::fs::MetadataExt;
4use std::path::{Path, PathBuf};
5
6use anyhow::{Context, Result, bail};
7use rusqlite::{OptionalExtension, params};
8use serde::Serialize;
9
10use super::SqliteStore;
11use super::pool::BUSY_TIMEOUT;
12
13const COMPACTED_KEY: &str = "stored_redaction_v1_compacted";
14const VACUUM_BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600);
15const SPACE_MARGIN: u64 = 16 * 1024 * 1024;
16
17/// Free space against what a VACUUM of this store needs, on the filesystem
18/// with the least room to spare.
19#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
20pub struct VacuumSpace {
21    pub path: PathBuf,
22    pub needed_bytes: u64,
23    pub free_bytes: u64,
24}
25
26impl VacuumSpace {
27    pub fn fits(&self) -> bool {
28        self.free_bytes >= self.needed_bytes
29    }
30}
31
32impl SqliteStore {
33    /// `VACUUM` cannot run inside a transaction and needs the writer alone;
34    /// returns (bytes before, bytes after).
35    pub fn vacuum(&self) -> Result<(u64, u64)> {
36        self.vacuum_then(false)
37    }
38
39    /// `vacuum`, then a checkpoint that truncates the WAL, because the frames
40    /// VACUUM appended, and older ones past them, would otherwise keep every
41    /// page's previous contents on disk. Errors when readers keep the
42    /// checkpoint from finishing, though the VACUUM itself went through.
43    pub fn vacuum_clearing_wal(&self) -> Result<(u64, u64)> {
44        self.vacuum_then(true)
45    }
46
47    fn vacuum_then(&self, truncate_wal: bool) -> Result<(u64, u64)> {
48        let conn = self.conn();
49        let size = |sql: &str| -> Result<u64> {
50            let page_size: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?;
51            let pages: i64 = conn.query_row(sql, [], |r| r.get(0))?;
52            Ok((page_size * pages) as u64)
53        };
54        let before = size("PRAGMA page_count")?;
55        conn.busy_timeout(VACUUM_BUSY_TIMEOUT)?;
56        let vacuumed = conn.execute_batch("VACUUM;").and_then(|()| {
57            if !truncate_wal {
58                return Ok(0);
59            }
60            conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| {
61                r.get::<_, i64>(0)
62            })
63        });
64        conn.busy_timeout(BUSY_TIMEOUT)?;
65        if vacuumed.context("vacuuming the store")? != 0 {
66            bail!("the WAL checkpoint after VACUUM was blocked by readers");
67        }
68        let after = size("PRAGMA page_count")?;
69        Ok((before, after))
70    }
71
72    /// VACUUM writes a full copy of the live pages to SQLite's temp
73    /// directory and then the same again into the WAL, so both need room.
74    pub fn vacuum_space(&self) -> Result<VacuumSpace> {
75        let Some(db) = self.db_path()? else {
76            return Ok(VacuumSpace {
77                path: PathBuf::new(),
78                needed_bytes: 0,
79                free_bytes: 0,
80            });
81        };
82        let live = self.live_bytes()?;
83        let copy = live + live / 16 + SPACE_MARGIN;
84        let db_dir = db
85            .parent()
86            .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
87        let temp_dir = sqlite_temp_dir();
88        let mut needs = vec![(db_dir.clone(), copy)];
89        if same_filesystem(&db_dir, &temp_dir)? {
90            needs[0].1 += copy;
91        } else {
92            needs.push((temp_dir, copy));
93        }
94        let mut tightest: Option<VacuumSpace> = None;
95        for (path, needed_bytes) in needs {
96            let free_bytes = fs4::available_space(&path)
97                .with_context(|| format!("reading free space at {}", path.display()))?;
98            let space = VacuumSpace {
99                path,
100                needed_bytes,
101                free_bytes,
102            };
103            let headroom = |s: &VacuumSpace| s.free_bytes as i128 - s.needed_bytes as i128;
104            if tightest
105                .as_ref()
106                .is_none_or(|t| headroom(&space) < headroom(t))
107            {
108                tightest = Some(space);
109            }
110        }
111        Ok(tightest.expect("the database directory is always checked"))
112    }
113
114    pub fn stored_redaction_compacted(&self) -> Result<bool> {
115        let value: Option<String> = self
116            .reader()
117            .query_row(
118                "SELECT value FROM index_meta WHERE key = ?1",
119                params![COMPACTED_KEY],
120                |r| r.get(0),
121            )
122            .optional()?;
123        Ok(value.as_deref() == Some("done"))
124    }
125
126    pub fn mark_stored_redaction_compacted(&self) -> Result<()> {
127        self.conn().execute(
128            "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, 'done')",
129            params![COMPACTED_KEY],
130        )?;
131        Ok(())
132    }
133
134    fn db_path(&self) -> Result<Option<PathBuf>> {
135        let file: String = self
136            .reader()
137            .query_row("PRAGMA database_list", [], |r| r.get(2))?;
138        Ok((!file.is_empty()).then(|| PathBuf::from(file)))
139    }
140
141    fn live_bytes(&self) -> Result<u64> {
142        let conn = self.reader();
143        let page_size: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?;
144        let pages: i64 = conn.query_row("PRAGMA page_count", [], |r| r.get(0))?;
145        let free: i64 = conn.query_row("PRAGMA freelist_count", [], |r| r.get(0))?;
146        Ok((pages - free).max(0) as u64 * page_size as u64)
147    }
148}
149
150/// The directory SQLite's unix VFS picks for temp files, in its own order:
151/// the first candidate that is a directory this process can write and enter.
152fn sqlite_temp_dir() -> PathBuf {
153    let from_env = ["SQLITE_TMPDIR", "TMPDIR"]
154        .into_iter()
155        .filter_map(|var| std::env::var_os(var).map(PathBuf::from));
156    let fixed = ["/var/tmp", "/usr/tmp", "/tmp", "."]
157        .into_iter()
158        .map(PathBuf::from);
159    from_env
160        .chain(fixed)
161        .find(|dir| usable_temp_dir(dir))
162        .unwrap_or_else(|| PathBuf::from("."))
163}
164
165fn usable_temp_dir(dir: &Path) -> bool {
166    if !dir.is_dir() {
167        return false;
168    }
169    let Ok(path) = CString::new(dir.as_os_str().as_bytes()) else {
170        return false;
171    };
172    unsafe { libc::access(path.as_ptr(), libc::W_OK | libc::X_OK) == 0 }
173}
174
175fn same_filesystem(a: &Path, b: &Path) -> Result<bool> {
176    let dev = |p: &Path| {
177        std::fs::metadata(p)
178            .map(|m| m.dev())
179            .with_context(|| format!("reading {}", p.display()))
180    };
181    Ok(dev(a)? == dev(b)?)
182}
183
184#[cfg(test)]
185mod tests {
186    use super::*;
187
188    #[test]
189    fn an_in_memory_store_needs_no_space() {
190        let store = SqliteStore::in_memory("m".to_string()).unwrap();
191        assert!(store.vacuum_space().unwrap().fits());
192    }
193
194    #[test]
195    fn a_shortfall_does_not_fit() {
196        let space = VacuumSpace {
197            path: PathBuf::from("/"),
198            needed_bytes: 10,
199            free_bytes: 9,
200        };
201        assert!(!space.fits());
202    }
203
204    #[test]
205    fn the_temp_dir_is_one_sqlite_can_write() {
206        assert!(usable_temp_dir(&sqlite_temp_dir()));
207    }
208}