use std::path::{Path, PathBuf};
use std::time::SystemTime;
use rusqlite::{Connection, OptionalExtension, params};
use crate::model::{Icon, Item, ItemKey, Source};
pub const DEFAULT_EXTENSIONS: &[&str] = &[
"md",
"txt",
"rst",
"org",
"adoc",
"tex",
"log",
"csv",
"tsv",
"json",
"toml",
"yaml",
"yml",
"ini",
"conf",
"cfg",
"xml",
"html",
"css",
"scss",
"js",
"ts",
"jsx",
"tsx",
"rs",
"py",
"go",
"rb",
"sh",
"bash",
"zsh",
"fish",
"c",
"h",
"cpp",
"hpp",
"java",
"kt",
"swift",
"sql",
"lua",
"vim",
"nix",
"dockerfile",
"gradle",
"properties",
"env",
];
#[derive(Debug, Clone, Copy)]
pub struct Limits {
pub max_index_bytes: u64,
pub max_file_bytes: u64,
}
impl Default for Limits {
fn default() -> Self {
Self {
max_index_bytes: 512 * 1024 * 1024,
max_file_bytes: 2 * 1024 * 1024,
}
}
}
const MAX_TEXT_CHARS: usize = 200_000;
const MAX_DOCUMENTS: usize = 200_000;
const RESULTS: usize = 8;
const BATCH: usize = 400;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("could not open the content index")]
Open(#[source] rusqlite::Error),
#[error("content index query failed")]
Query(#[source] rusqlite::Error),
}
pub struct Content {
connection: Connection,
extensions: Vec<String>,
limits: Limits,
path: PathBuf,
buffer: Vec<u8>,
}
impl std::fmt::Debug for Content {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Content").finish_non_exhaustive()
}
}
impl Content {
pub fn open(path: &Path, extensions: Vec<String>, limits: Limits) -> Result<Self, Error> {
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let connection = Connection::open(path).map_err(Error::Open)?;
let _ = connection.pragma_update(None, "journal_mode", "WAL");
let _ = connection.pragma_update(None, "synchronous", "NORMAL");
connection
.execute_batch(
"CREATE TABLE IF NOT EXISTS documents (
path TEXT PRIMARY KEY,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
-- `path` is UNINDEXED: it is carried for retrieval, and the
-- path index already handles matching on names.
CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5 (
path UNINDEXED,
body,
tokenize = 'unicode61 remove_diacritics 2'
);",
)
.map_err(Error::Open)?;
let extensions = if extensions.is_empty() {
DEFAULT_EXTENSIONS.iter().map(|e| (*e).to_owned()).collect()
} else {
extensions
.iter()
.map(|e| e.trim_start_matches('.').to_lowercase())
.collect()
};
Ok(Self {
connection,
extensions,
limits,
path: path.to_path_buf(),
buffer: Vec::new(),
})
}
#[must_use]
pub fn is_full(&self) -> bool {
disk_usage(&self.path) >= self.limits.max_index_bytes
}
#[must_use]
pub fn accepts(&self, path: &Path) -> bool {
accepts(&self.extensions, path)
}
#[must_use]
pub fn len(&self) -> usize {
self.connection
.query_row("SELECT count(*) FROM documents", [], |row| {
row.get::<_, i64>(0)
})
.map_or(0, |count| count as usize)
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn index(&mut self, paths: &[PathBuf]) -> usize {
let mut indexed = 0;
let existing = self.len();
for chunk in paths.chunks(BATCH) {
if existing + indexed >= MAX_DOCUMENTS {
tracing::warn!(limit = MAX_DOCUMENTS, "content index full; stopping");
break;
}
if self.is_full() {
tracing::info!(
limit_mb = self.limits.max_index_bytes / (1024 * 1024),
"content index reached its size limit; stopping"
);
break;
}
let Content {
connection,
buffer,
limits,
extensions,
..
} = self;
let Ok(transaction) = connection.transaction() else {
break;
};
for path in chunk {
if !accepts(extensions, path) {
continue;
}
let Ok(metadata) = std::fs::metadata(path) else {
continue;
};
if !metadata.is_file() || metadata.len() > limits.max_file_bytes {
continue;
}
let size = metadata.len() as i64;
let mtime = metadata
.modified()
.ok()
.and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok())
.map_or(0, |age| age.as_secs() as i64);
if is_current(&transaction, path, mtime, size) {
continue;
}
if upsert(&transaction, buffer, path, mtime, size) {
indexed += 1;
}
}
if transaction.commit().is_err() {
break;
}
}
if indexed > 0 {
tracing::info!(indexed, total = self.len(), "content index updated");
}
indexed
}
pub fn prune_missing(&mut self) {
let Ok(mut statement) = self.connection.prepare("SELECT path FROM documents") else {
return;
};
let Ok(rows) = statement.query_map([], |row| row.get::<_, String>(0)) else {
return;
};
let gone: Vec<String> = rows
.flatten()
.filter(|path| !Path::new(path).exists())
.collect();
drop(statement);
for path in &gone {
let _ = self
.connection
.execute("DELETE FROM search WHERE path = ?1", params![path]);
let _ = self
.connection
.execute("DELETE FROM documents WHERE path = ?1", params![path]);
}
if !gone.is_empty() {
tracing::info!(removed = gone.len(), "pruned deleted documents");
}
}
pub fn search(&self, query: &str) -> Result<Vec<Item>, Error> {
let Some(expression) = fts_query(query) else {
return Ok(Vec::new());
};
let mut statement = self
.connection
.prepare(
"SELECT path,
snippet(search, 1, '', '', '…', 12) AS excerpt,
bm25(search) AS score
FROM search
WHERE search MATCH ?1
ORDER BY score
LIMIT ?2",
)
.map_err(Error::Query)?;
let rows = statement
.query_map(params![expression, RESULTS as i64], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, f64>(2)?,
))
})
.map_err(Error::Query)?;
Ok(rows
.flatten()
.filter_map(|(path, excerpt, score)| {
let path = PathBuf::from(path);
path.is_file().then(|| item(&path, &excerpt, score))
})
.collect())
}
}
fn files(path: &Path) -> [PathBuf; 3] {
let beside = |suffix: &str| {
let mut name = path.as_os_str().to_owned();
name.push(suffix);
PathBuf::from(name)
};
[beside("-wal"), beside("-shm"), path.to_path_buf()]
}
#[must_use]
pub fn disk_usage(path: &Path) -> u64 {
files(path)
.iter()
.filter_map(|file| std::fs::metadata(file).ok())
.map(|metadata| metadata.len())
.sum()
}
pub fn clear(path: &Path) -> std::io::Result<u64> {
let freed = disk_usage(path);
for file in files(path) {
match std::fs::remove_file(&file) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(error),
}
}
Ok(freed)
}
fn accepts(extensions: &[String], path: &Path) -> bool {
path.extension()
.and_then(|extension| extension.to_str())
.is_some_and(|extension| {
let extension = extension.to_lowercase();
extensions.contains(&extension)
})
}
fn is_current(connection: &Connection, path: &Path, mtime: i64, size: i64) -> bool {
let stored: Option<(i64, i64)> = connection
.query_row(
"SELECT mtime, size FROM documents WHERE path = ?1",
params![path.to_string_lossy()],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.optional()
.ok()
.flatten();
stored == Some((mtime, size))
}
fn upsert(
connection: &Connection,
buffer: &mut Vec<u8>,
path: &Path,
mtime: i64,
size: i64,
) -> bool {
let Some(text) = extract(buffer, path, size as u64) else {
return false;
};
let path_text = path.to_string_lossy().into_owned();
let removed = connection.execute("DELETE FROM search WHERE path = ?1", params![path_text]);
let inserted = connection.execute(
"INSERT INTO search (path, body) VALUES (?1, ?2)",
params![path_text, text],
);
let recorded = connection.execute(
"INSERT INTO documents (path, mtime, size) VALUES (?1, ?2, ?3)
ON CONFLICT(path) DO UPDATE SET mtime = excluded.mtime, size = excluded.size",
params![path_text, mtime, size],
);
removed.is_ok() && inserted.is_ok() && recorded.is_ok()
}
fn fts_query(query: &str) -> Option<String> {
let tokens: Vec<String> = query
.split_whitespace()
.filter(|token| token.chars().any(char::is_alphanumeric))
.map(|token| format!("\"{}\"", token.replace('"', "\"\"")))
.collect();
if tokens.is_empty() {
return None;
}
let last = tokens.len() - 1;
let mut expression = tokens;
expression[last].push('*');
Some(expression.join(" AND "))
}
fn extract(buffer: &mut Vec<u8>, path: &Path, size: u64) -> Option<String> {
use std::io::Read;
let file = std::fs::File::open(path).ok()?;
buffer.clear();
buffer.reserve(size as usize);
file.take(size).read_to_end(buffer).ok()?;
if buffer[..buffer.len().min(8192)].contains(&0) {
return None;
}
let lossy;
let text = match std::str::from_utf8(buffer) {
Ok(text) => text,
Err(_) => {
lossy = String::from_utf8_lossy(buffer);
&lossy
}
};
Some(match text.char_indices().nth(MAX_TEXT_CHARS) {
Some((cutoff, _)) => text[..cutoff].to_owned(),
None => text.to_owned(),
})
}
fn item(path: &Path, excerpt: &str, score: f64) -> Item {
let title = path.file_name().map_or_else(
|| path.display().to_string(),
|name| name.to_string_lossy().into_owned(),
);
let excerpt = excerpt.split_whitespace().collect::<Vec<_>>().join(" ");
let location = path
.parent()
.map(|parent| {
let text = parent.display().to_string();
dirs::home_dir()
.and_then(|home| {
text.strip_prefix(home.to_str()?)
.map(|rest| format!("~{rest}"))
})
.unwrap_or(text)
})
.unwrap_or_default();
Item {
key: ItemKey(format!("content:{}", path.display())),
id: 0,
title,
subtitle: if location.is_empty() {
excerpt
} else {
format!("{location} · {excerpt}")
},
icon: Some(Icon::Name("text-x-generic".to_owned())),
category_icon: Some(Icon::Name("edit-find-symbolic".to_owned())),
window: None,
source: Source::File {
path: path.to_path_buf(),
},
autocomplete: None,
score: (((-score) / 10.0).clamp(0.0, 1.0) as f32) * 3.5,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn index() -> Content {
let dir = std::env::temp_dir().join(format!("jump-content-test-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
Content::open(&dir.join("content.db"), Vec::new(), Limits::default()).expect("open index")
}
#[test]
fn quotes_tokens_so_fts_syntax_cannot_leak() {
let expression = fts_query("AND value").expect("expression");
assert!(expression.starts_with("\"AND\""));
assert!(expression.contains("AND \"value\"*"));
}
#[test]
fn embedded_quotes_are_escaped() {
let expression = fts_query("say \"hello\"").expect("expression");
assert!(expression.contains("\"\"hello\"\""));
}
#[test]
fn punctuation_only_queries_are_rejected() {
assert!(fts_query("!!! ???").is_none());
assert!(fts_query(" ").is_none());
}
#[test]
fn extension_filter_matches_case_insensitively() {
let content = index();
assert!(content.accepts(Path::new("/tmp/a.MD")));
assert!(content.accepts(Path::new("/tmp/a.rs")));
assert!(!content.accepts(Path::new("/tmp/a.png")));
}
#[test]
fn indexes_and_finds_by_content() {
let dir = std::env::temp_dir().join("jump-content-find");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("create dir");
let file = dir.join("notes.md");
std::fs::write(&file, "the quick brown fox jumps over the lazy dog").expect("write");
let mut content = Content::open(&dir.join("content.db"), Vec::new(), Limits::default())
.expect("open index");
assert_eq!(content.index(std::slice::from_ref(&file)), 1);
let results = content.search("brown fox").expect("search");
assert_eq!(results.len(), 1);
assert!(results[0].subtitle.contains("brown"));
assert_eq!(content.index(std::slice::from_ref(&file)), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn changed_files_are_reindexed_without_duplicating() {
let dir = std::env::temp_dir().join("jump-content-change");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("create dir");
let file = dir.join("doc.txt");
std::fs::write(&file, "original contents here").expect("write");
let mut content = Content::open(&dir.join("content.db"), Vec::new(), Limits::default())
.expect("open index");
content.index(std::slice::from_ref(&file));
std::fs::write(&file, "replacement contents entirely different now").expect("rewrite");
assert_eq!(content.index(std::slice::from_ref(&file)), 1);
assert_eq!(content.len(), 1, "the document is replaced, not duplicated");
assert!(content.search("original").expect("search").is_empty());
assert_eq!(content.search("replacement").expect("search").len(), 1);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn binary_files_are_skipped() {
let dir = std::env::temp_dir().join("jump-content-binary");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("create dir");
let file = dir.join("fake.txt");
std::fs::write(&file, [0x00, 0x01, 0x02, b'h', b'i']).expect("write");
let mut content = Content::open(&dir.join("content.db"), Vec::new(), Limits::default())
.expect("open index");
assert_eq!(content.index(&[file]), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn clearing_deletes_the_database_and_its_log() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("content.db");
assert_eq!(clear(&path).expect("clear nothing"), 0);
let file = dir.path().join("diary.md");
std::fs::write(&file, "words nobody else should find").expect("write");
let mut content = Content::open(&path, Vec::new(), Limits::default()).expect("open index");
assert_eq!(content.index(std::slice::from_ref(&file)), 1);
let [log, shared, database] = files(&path);
assert!(log.exists(), "WAL mode keeps a log beside the database");
let database_bytes = std::fs::metadata(&database).expect("database").len();
assert!(disk_usage(&path) > database_bytes);
drop(content);
std::fs::write(&log, b"left behind").expect("log");
std::fs::write(&shared, b"left behind").expect("shared memory");
let occupied = disk_usage(&path);
assert_eq!(clear(&path).expect("clear"), occupied);
for file in files(&path) {
assert!(!file.exists(), "{} survived", file.display());
}
assert_eq!(disk_usage(&path), 0);
assert!(file.exists());
let content = Content::open(&path, Vec::new(), Limits::default()).expect("reopen");
assert!(content.is_empty());
assert!(content.search("nobody").expect("search").is_empty());
}
#[test]
fn deleted_files_are_pruned() {
let dir = std::env::temp_dir().join("jump-content-prune");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("create dir");
let file = dir.join("temp.md");
std::fs::write(&file, "some indexed words").expect("write");
let mut content = Content::open(&dir.join("content.db"), Vec::new(), Limits::default())
.expect("open index");
content.index(std::slice::from_ref(&file));
assert_eq!(content.len(), 1);
std::fs::remove_file(&file).expect("remove");
content.prune_missing();
assert_eq!(content.len(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
}