1use std::path::{Path, PathBuf};
33use std::time::SystemTime;
34
35use rusqlite::{Connection, OptionalExtension, params};
36
37use crate::model::{Icon, Item, ItemKey, Source};
38
39pub const DEFAULT_EXTENSIONS: &[&str] = &[
44 "md",
45 "txt",
46 "rst",
47 "org",
48 "adoc",
49 "tex",
50 "log",
51 "csv",
52 "tsv",
53 "json",
54 "toml",
55 "yaml",
56 "yml",
57 "ini",
58 "conf",
59 "cfg",
60 "xml",
61 "html",
62 "css",
63 "scss",
64 "js",
65 "ts",
66 "jsx",
67 "tsx",
68 "rs",
69 "py",
70 "go",
71 "rb",
72 "sh",
73 "bash",
74 "zsh",
75 "fish",
76 "c",
77 "h",
78 "cpp",
79 "hpp",
80 "java",
81 "kt",
82 "swift",
83 "sql",
84 "lua",
85 "vim",
86 "nix",
87 "dockerfile",
88 "gradle",
89 "properties",
90 "env",
91];
92
93#[derive(Debug, Clone, Copy)]
95pub struct Limits {
96 pub max_index_bytes: u64,
98 pub max_file_bytes: u64,
104}
105
106impl Default for Limits {
107 fn default() -> Self {
108 Self {
109 max_index_bytes: 512 * 1024 * 1024,
110 max_file_bytes: 2 * 1024 * 1024,
111 }
112 }
113}
114
115const MAX_TEXT_CHARS: usize = 200_000;
117
118const MAX_DOCUMENTS: usize = 200_000;
121
122const RESULTS: usize = 8;
124
125const BATCH: usize = 400;
127
128#[derive(Debug, thiserror::Error)]
130pub enum Error {
131 #[error("could not open the content index")]
133 Open(#[source] rusqlite::Error),
134 #[error("content index query failed")]
136 Query(#[source] rusqlite::Error),
137}
138
139pub struct Content {
141 connection: Connection,
142 extensions: Vec<String>,
143 limits: Limits,
144 path: PathBuf,
145 buffer: Vec<u8>,
148}
149
150impl std::fmt::Debug for Content {
151 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
152 f.debug_struct("Content").finish_non_exhaustive()
153 }
154}
155
156impl Content {
157 pub fn open(path: &Path, extensions: Vec<String>, limits: Limits) -> Result<Self, Error> {
164 if let Some(parent) = path.parent() {
165 let _ = std::fs::create_dir_all(parent);
166 }
167
168 let connection = Connection::open(path).map_err(Error::Open)?;
169
170 let _ = connection.pragma_update(None, "journal_mode", "WAL");
173 let _ = connection.pragma_update(None, "synchronous", "NORMAL");
174
175 connection
176 .execute_batch(
177 "CREATE TABLE IF NOT EXISTS documents (
178 path TEXT PRIMARY KEY,
179 mtime INTEGER NOT NULL,
180 size INTEGER NOT NULL
181 );
182 -- `path` is UNINDEXED: it is carried for retrieval, and the
183 -- path index already handles matching on names.
184 CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5 (
185 path UNINDEXED,
186 body,
187 tokenize = 'unicode61 remove_diacritics 2'
188 );",
189 )
190 .map_err(Error::Open)?;
191
192 let extensions = if extensions.is_empty() {
193 DEFAULT_EXTENSIONS.iter().map(|e| (*e).to_owned()).collect()
194 } else {
195 extensions
196 .iter()
197 .map(|e| e.trim_start_matches('.').to_lowercase())
198 .collect()
199 };
200
201 Ok(Self {
202 connection,
203 extensions,
204 limits,
205 path: path.to_path_buf(),
206 buffer: Vec::new(),
207 })
208 }
209
210 #[must_use]
216 pub fn is_full(&self) -> bool {
217 disk_usage(&self.path) >= self.limits.max_index_bytes
218 }
219
220 #[must_use]
222 pub fn accepts(&self, path: &Path) -> bool {
223 accepts(&self.extensions, path)
224 }
225
226 #[must_use]
228 pub fn len(&self) -> usize {
229 self.connection
230 .query_row("SELECT count(*) FROM documents", [], |row| {
231 row.get::<_, i64>(0)
232 })
233 .map_or(0, |count| count as usize)
234 }
235
236 #[must_use]
238 pub fn is_empty(&self) -> bool {
239 self.len() == 0
240 }
241
242 pub fn index(&mut self, paths: &[PathBuf]) -> usize {
247 let mut indexed = 0;
248 let existing = self.len();
249
250 for chunk in paths.chunks(BATCH) {
255 if existing + indexed >= MAX_DOCUMENTS {
256 tracing::warn!(limit = MAX_DOCUMENTS, "content index full; stopping");
257 break;
258 }
259 if self.is_full() {
263 tracing::info!(
264 limit_mb = self.limits.max_index_bytes / (1024 * 1024),
265 "content index reached its size limit; stopping"
266 );
267 break;
268 }
269
270 let Content {
271 connection,
272 buffer,
273 limits,
274 extensions,
275 ..
276 } = self;
277 let Ok(transaction) = connection.transaction() else {
278 break;
279 };
280
281 for path in chunk {
282 if !accepts(extensions, path) {
283 continue;
284 }
285
286 let Ok(metadata) = std::fs::metadata(path) else {
287 continue;
288 };
289 if !metadata.is_file() || metadata.len() > limits.max_file_bytes {
290 continue;
291 }
292
293 let size = metadata.len() as i64;
294 let mtime = metadata
295 .modified()
296 .ok()
297 .and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok())
298 .map_or(0, |age| age.as_secs() as i64);
299
300 if is_current(&transaction, path, mtime, size) {
301 continue;
302 }
303 if upsert(&transaction, buffer, path, mtime, size) {
304 indexed += 1;
305 }
306 }
307
308 if transaction.commit().is_err() {
309 break;
310 }
311 }
312
313 if indexed > 0 {
314 tracing::info!(indexed, total = self.len(), "content index updated");
315 }
316 indexed
317 }
318
319 pub fn prune_missing(&mut self) {
321 let Ok(mut statement) = self.connection.prepare("SELECT path FROM documents") else {
322 return;
323 };
324 let Ok(rows) = statement.query_map([], |row| row.get::<_, String>(0)) else {
325 return;
326 };
327
328 let gone: Vec<String> = rows
329 .flatten()
330 .filter(|path| !Path::new(path).exists())
331 .collect();
332 drop(statement);
333
334 for path in &gone {
335 let _ = self
336 .connection
337 .execute("DELETE FROM search WHERE path = ?1", params![path]);
338 let _ = self
339 .connection
340 .execute("DELETE FROM documents WHERE path = ?1", params![path]);
341 }
342 if !gone.is_empty() {
343 tracing::info!(removed = gone.len(), "pruned deleted documents");
344 }
345 }
346
347 pub fn search(&self, query: &str) -> Result<Vec<Item>, Error> {
353 let Some(expression) = fts_query(query) else {
354 return Ok(Vec::new());
355 };
356
357 let mut statement = self
358 .connection
359 .prepare(
360 "SELECT path,
363 snippet(search, 1, '', '', '…', 12) AS excerpt,
364 bm25(search) AS score
365 FROM search
366 WHERE search MATCH ?1
367 ORDER BY score
368 LIMIT ?2",
369 )
370 .map_err(Error::Query)?;
371
372 let rows = statement
373 .query_map(params![expression, RESULTS as i64], |row| {
374 Ok((
375 row.get::<_, String>(0)?,
376 row.get::<_, String>(1)?,
377 row.get::<_, f64>(2)?,
378 ))
379 })
380 .map_err(Error::Query)?;
381
382 Ok(rows
383 .flatten()
384 .filter_map(|(path, excerpt, score)| {
385 let path = PathBuf::from(path);
386 path.is_file().then(|| item(&path, &excerpt, score))
389 })
390 .collect())
391 }
392}
393
394fn files(path: &Path) -> [PathBuf; 3] {
400 let beside = |suffix: &str| {
401 let mut name = path.as_os_str().to_owned();
402 name.push(suffix);
403 PathBuf::from(name)
404 };
405 [beside("-wal"), beside("-shm"), path.to_path_buf()]
406}
407
408#[must_use]
411pub fn disk_usage(path: &Path) -> u64 {
412 files(path)
413 .iter()
414 .filter_map(|file| std::fs::metadata(file).ok())
415 .map(|metadata| metadata.len())
416 .sum()
417}
418
419pub fn clear(path: &Path) -> std::io::Result<u64> {
431 let freed = disk_usage(path);
432 for file in files(path) {
433 match std::fs::remove_file(&file) {
434 Ok(()) => {}
435 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
436 Err(error) => return Err(error),
437 }
438 }
439 Ok(freed)
440}
441
442fn accepts(extensions: &[String], path: &Path) -> bool {
444 path.extension()
445 .and_then(|extension| extension.to_str())
446 .is_some_and(|extension| {
447 let extension = extension.to_lowercase();
448 extensions.contains(&extension)
449 })
450}
451
452fn is_current(connection: &Connection, path: &Path, mtime: i64, size: i64) -> bool {
454 let stored: Option<(i64, i64)> = connection
455 .query_row(
456 "SELECT mtime, size FROM documents WHERE path = ?1",
457 params![path.to_string_lossy()],
458 |row| Ok((row.get(0)?, row.get(1)?)),
459 )
460 .optional()
461 .ok()
462 .flatten();
463
464 stored == Some((mtime, size))
465}
466
467fn upsert(
469 connection: &Connection,
470 buffer: &mut Vec<u8>,
471 path: &Path,
472 mtime: i64,
473 size: i64,
474) -> bool {
475 let Some(text) = extract(buffer, path, size as u64) else {
476 return false;
477 };
478 let path_text = path.to_string_lossy().into_owned();
479
480 let removed = connection.execute("DELETE FROM search WHERE path = ?1", params![path_text]);
483 let inserted = connection.execute(
484 "INSERT INTO search (path, body) VALUES (?1, ?2)",
485 params![path_text, text],
486 );
487 let recorded = connection.execute(
488 "INSERT INTO documents (path, mtime, size) VALUES (?1, ?2, ?3)
489 ON CONFLICT(path) DO UPDATE SET mtime = excluded.mtime, size = excluded.size",
490 params![path_text, mtime, size],
491 );
492
493 removed.is_ok() && inserted.is_ok() && recorded.is_ok()
494}
495
496fn fts_query(query: &str) -> Option<String> {
502 let tokens: Vec<String> = query
503 .split_whitespace()
504 .filter(|token| token.chars().any(char::is_alphanumeric))
505 .map(|token| format!("\"{}\"", token.replace('"', "\"\"")))
506 .collect();
507
508 if tokens.is_empty() {
509 return None;
510 }
511
512 let last = tokens.len() - 1;
515 let mut expression = tokens;
516 expression[last].push('*');
517 Some(expression.join(" AND "))
518}
519
520fn extract(buffer: &mut Vec<u8>, path: &Path, size: u64) -> Option<String> {
527 use std::io::Read;
528
529 let file = std::fs::File::open(path).ok()?;
530 buffer.clear();
531 buffer.reserve(size as usize);
532 file.take(size).read_to_end(buffer).ok()?;
535
536 if buffer[..buffer.len().min(8192)].contains(&0) {
539 return None;
540 }
541
542 let lossy;
547 let text = match std::str::from_utf8(buffer) {
548 Ok(text) => text,
549 Err(_) => {
550 lossy = String::from_utf8_lossy(buffer);
551 &lossy
552 }
553 };
554
555 Some(match text.char_indices().nth(MAX_TEXT_CHARS) {
556 Some((cutoff, _)) => text[..cutoff].to_owned(),
557 None => text.to_owned(),
558 })
559}
560
561fn item(path: &Path, excerpt: &str, score: f64) -> Item {
563 let title = path.file_name().map_or_else(
564 || path.display().to_string(),
565 |name| name.to_string_lossy().into_owned(),
566 );
567
568 let excerpt = excerpt.split_whitespace().collect::<Vec<_>>().join(" ");
569
570 let location = path
574 .parent()
575 .map(|parent| {
576 let text = parent.display().to_string();
577 dirs::home_dir()
578 .and_then(|home| {
579 text.strip_prefix(home.to_str()?)
580 .map(|rest| format!("~{rest}"))
581 })
582 .unwrap_or(text)
583 })
584 .unwrap_or_default();
585
586 Item {
587 key: ItemKey(format!("content:{}", path.display())),
588 id: 0,
589 title,
590 subtitle: if location.is_empty() {
593 excerpt
594 } else {
595 format!("{location} · {excerpt}")
596 },
597 icon: Some(Icon::Name("text-x-generic".to_owned())),
598 category_icon: Some(Icon::Name("edit-find-symbolic".to_owned())),
599 window: None,
600 source: Source::File {
601 path: path.to_path_buf(),
602 },
603 autocomplete: None,
606 score: (((-score) / 10.0).clamp(0.0, 1.0) as f32) * 3.5,
607 }
608}
609
610#[cfg(test)]
611mod tests {
612 use super::*;
613
614 fn index() -> Content {
615 let dir = std::env::temp_dir().join(format!("jump-content-test-{}", std::process::id()));
619 let _ = std::fs::remove_dir_all(&dir);
620 Content::open(&dir.join("content.db"), Vec::new(), Limits::default()).expect("open index")
621 }
622
623 #[test]
624 fn quotes_tokens_so_fts_syntax_cannot_leak() {
625 let expression = fts_query("AND value").expect("expression");
627 assert!(expression.starts_with("\"AND\""));
628 assert!(expression.contains("AND \"value\"*"));
629 }
630
631 #[test]
632 fn embedded_quotes_are_escaped() {
633 let expression = fts_query("say \"hello\"").expect("expression");
634 assert!(expression.contains("\"\"hello\"\""));
635 }
636
637 #[test]
638 fn punctuation_only_queries_are_rejected() {
639 assert!(fts_query("!!! ???").is_none());
640 assert!(fts_query(" ").is_none());
641 }
642
643 #[test]
644 fn extension_filter_matches_case_insensitively() {
645 let content = index();
646 assert!(content.accepts(Path::new("/tmp/a.MD")));
647 assert!(content.accepts(Path::new("/tmp/a.rs")));
648 assert!(!content.accepts(Path::new("/tmp/a.png")));
649 }
650
651 #[test]
652 fn indexes_and_finds_by_content() {
653 let dir = std::env::temp_dir().join("jump-content-find");
654 let _ = std::fs::remove_dir_all(&dir);
655 std::fs::create_dir_all(&dir).expect("create dir");
656
657 let file = dir.join("notes.md");
658 std::fs::write(&file, "the quick brown fox jumps over the lazy dog").expect("write");
659
660 let mut content = Content::open(&dir.join("content.db"), Vec::new(), Limits::default())
661 .expect("open index");
662 assert_eq!(content.index(std::slice::from_ref(&file)), 1);
663
664 let results = content.search("brown fox").expect("search");
665 assert_eq!(results.len(), 1);
666 assert!(results[0].subtitle.contains("brown"));
667
668 assert_eq!(content.index(std::slice::from_ref(&file)), 0);
670
671 let _ = std::fs::remove_dir_all(&dir);
672 }
673
674 #[test]
675 fn changed_files_are_reindexed_without_duplicating() {
676 let dir = std::env::temp_dir().join("jump-content-change");
677 let _ = std::fs::remove_dir_all(&dir);
678 std::fs::create_dir_all(&dir).expect("create dir");
679
680 let file = dir.join("doc.txt");
681 std::fs::write(&file, "original contents here").expect("write");
682
683 let mut content = Content::open(&dir.join("content.db"), Vec::new(), Limits::default())
684 .expect("open index");
685 content.index(std::slice::from_ref(&file));
686
687 std::fs::write(&file, "replacement contents entirely different now").expect("rewrite");
690 assert_eq!(content.index(std::slice::from_ref(&file)), 1);
691 assert_eq!(content.len(), 1, "the document is replaced, not duplicated");
692
693 assert!(content.search("original").expect("search").is_empty());
694 assert_eq!(content.search("replacement").expect("search").len(), 1);
695
696 let _ = std::fs::remove_dir_all(&dir);
697 }
698
699 #[test]
700 fn binary_files_are_skipped() {
701 let dir = std::env::temp_dir().join("jump-content-binary");
702 let _ = std::fs::remove_dir_all(&dir);
703 std::fs::create_dir_all(&dir).expect("create dir");
704
705 let file = dir.join("fake.txt");
707 std::fs::write(&file, [0x00, 0x01, 0x02, b'h', b'i']).expect("write");
708
709 let mut content = Content::open(&dir.join("content.db"), Vec::new(), Limits::default())
710 .expect("open index");
711 assert_eq!(content.index(&[file]), 0);
712
713 let _ = std::fs::remove_dir_all(&dir);
714 }
715
716 #[test]
717 fn clearing_deletes_the_database_and_its_log() {
718 let dir = tempfile::tempdir().expect("tempdir");
719 let path = dir.path().join("content.db");
720
721 assert_eq!(clear(&path).expect("clear nothing"), 0);
723
724 let file = dir.path().join("diary.md");
725 std::fs::write(&file, "words nobody else should find").expect("write");
726 let mut content = Content::open(&path, Vec::new(), Limits::default()).expect("open index");
727 assert_eq!(content.index(std::slice::from_ref(&file)), 1);
728
729 let [log, shared, database] = files(&path);
732 assert!(log.exists(), "WAL mode keeps a log beside the database");
733 let database_bytes = std::fs::metadata(&database).expect("database").len();
734 assert!(disk_usage(&path) > database_bytes);
735
736 drop(content);
739 std::fs::write(&log, b"left behind").expect("log");
740 std::fs::write(&shared, b"left behind").expect("shared memory");
741
742 let occupied = disk_usage(&path);
743 assert_eq!(clear(&path).expect("clear"), occupied);
744 for file in files(&path) {
745 assert!(!file.exists(), "{} survived", file.display());
746 }
747 assert_eq!(disk_usage(&path), 0);
748 assert!(file.exists());
750
751 let content = Content::open(&path, Vec::new(), Limits::default()).expect("reopen");
753 assert!(content.is_empty());
754 assert!(content.search("nobody").expect("search").is_empty());
755 }
756
757 #[test]
758 fn deleted_files_are_pruned() {
759 let dir = std::env::temp_dir().join("jump-content-prune");
760 let _ = std::fs::remove_dir_all(&dir);
761 std::fs::create_dir_all(&dir).expect("create dir");
762
763 let file = dir.join("temp.md");
764 std::fs::write(&file, "some indexed words").expect("write");
765
766 let mut content = Content::open(&dir.join("content.db"), Vec::new(), Limits::default())
767 .expect("open index");
768 content.index(std::slice::from_ref(&file));
769 assert_eq!(content.len(), 1);
770
771 std::fs::remove_file(&file).expect("remove");
772 content.prune_missing();
773 assert_eq!(content.len(), 0);
774
775 let _ = std::fs::remove_dir_all(&dir);
776 }
777}