use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use grep_matcher::Matcher;
use grep_regex::{RegexMatcher, RegexMatcherBuilder};
use grep_searcher::{sinks::UTF8, BinaryDetection, SearcherBuilder};
use crate::config::schema::{FilesConfig, SearchConfig};
use crate::vault::walker::{self, WalkOptions};
const BATCH: usize = 32;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Hit {
pub path: PathBuf,
pub line: u64,
pub text: String,
pub span: (usize, usize),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SearchEvent {
Hits(Vec<Hit>),
Finished {
total: usize,
truncated: bool,
},
BadPattern(String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Query {
pub pattern: String,
pub regex: bool,
}
impl Query {
pub fn parse(input: &str, config: &SearchConfig) -> Self {
let trimmed = input.trim();
if trimmed.len() >= 2 && trimmed.starts_with('/') && trimmed.ends_with('/') {
return Query {
pattern: trimmed[1..trimmed.len() - 1].to_string(),
regex: true,
};
}
Query {
pattern: trimmed.to_string(),
regex: config.regex,
}
}
pub fn matcher(&self, config: &SearchConfig) -> Result<RegexMatcher, String> {
RegexMatcherBuilder::new()
.case_smart(config.smart_case)
.fixed_strings(!self.regex)
.line_terminator(Some(b'\n'))
.build(&self.pattern)
.map_err(|e| format!("{e}"))
}
}
#[derive(Debug)]
pub struct SearchHandle {
cancelled: Arc<AtomicBool>,
}
impl SearchHandle {
pub fn cancel(&self) {
self.cancelled.store(true, Ordering::Relaxed);
}
}
impl Drop for SearchHandle {
fn drop(&mut self) {
self.cancel();
}
}
pub fn spawn(
root: PathBuf,
query: Query,
search: SearchConfig,
files: FilesConfig,
walk: WalkOptions,
sink: impl Fn(SearchEvent) + Send + Sync + 'static,
) -> SearchHandle {
let cancelled = Arc::new(AtomicBool::new(false));
let flag = Arc::clone(&cancelled);
let sink = Arc::new(sink);
let thread_sink = Arc::clone(&sink);
let spawned = std::thread::Builder::new()
.name("perga-search".to_string())
.spawn(move || {
run(&root, &query, &search, &files, walk, &flag, &*thread_sink);
});
if spawned.is_err() {
sink(SearchEvent::Finished {
total: 0,
truncated: false,
});
}
SearchHandle { cancelled }
}
pub fn run(
root: &Path,
query: &Query,
search: &SearchConfig,
files: &FilesConfig,
walk: WalkOptions,
cancelled: &AtomicBool,
sink: &impl Fn(SearchEvent),
) {
if query.pattern.is_empty() {
sink(SearchEvent::Finished {
total: 0,
truncated: false,
});
return;
}
let matcher = match query.matcher(search) {
Ok(matcher) => matcher,
Err(e) => {
sink(SearchEvent::BadPattern(e));
return;
}
};
let mut searcher = SearcherBuilder::new()
.line_number(true)
.binary_detection(BinaryDetection::quit(0))
.build();
let mut batch: Vec<Hit> = Vec::with_capacity(BATCH);
let mut total = 0usize;
let mut truncated = false;
let paths = collect_paths(root, walk, files, search, cancelled);
'files: for relative in paths {
if cancelled.load(Ordering::Relaxed) {
return;
}
let absolute = root.join(&relative);
let mut hits: Vec<Hit> = Vec::new();
let outcome = searcher.search_path(
&matcher,
&absolute,
UTF8(|line, text| {
let span = matcher
.find(text.as_bytes())
.ok()
.flatten()
.map_or((0, 0), |m| (m.start(), m.end()));
hits.push(Hit {
path: relative.clone(),
line,
text: text.trim_end().to_string(),
span,
});
Ok(!cancelled.load(Ordering::Relaxed) && total + hits.len() < search.max_results)
}),
);
if let Err(e) = outcome {
tracing::debug!("skipping {} while searching: {e}", absolute.display());
}
total += hits.len();
batch.extend(hits);
if total >= search.max_results {
truncated = true;
}
if batch.len() >= BATCH || truncated {
sink(SearchEvent::Hits(std::mem::take(&mut batch)));
batch.reserve(BATCH);
}
if truncated {
break 'files;
}
}
if cancelled.load(Ordering::Relaxed) {
return;
}
if !batch.is_empty() {
sink(SearchEvent::Hits(batch));
}
sink(SearchEvent::Finished { total, truncated });
}
fn collect_paths(
root: &Path,
walk: WalkOptions,
files: &FilesConfig,
search: &SearchConfig,
cancelled: &AtomicBool,
) -> Vec<PathBuf> {
let paths = std::sync::Mutex::new(Vec::new());
walker::walk(root, walk, cancelled, &|event| {
if let walker::WalkEvent::Entries(entries) = event {
let mut paths = paths.lock().expect("the walk is the only writer");
paths.extend(
entries
.into_iter()
.filter(|entry| !entry.is_dir)
.filter(|entry| search.all_files || files.is_markdown(&entry.path))
.map(|entry| entry.path),
);
}
});
paths.into_inner().expect("the walk has finished")
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
fn vault() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/vault")
}
fn search(pattern: &str, config: SearchConfig) -> (Vec<Hit>, Option<(usize, bool)>) {
let hits = Mutex::new(Vec::new());
let finished = Mutex::new(None);
let query = Query::parse(pattern, &config);
run(
&vault(),
&query,
&config,
&FilesConfig::default(),
WalkOptions::default(),
&AtomicBool::new(false),
&|event| match event {
SearchEvent::Hits(batch) => hits.lock().unwrap().extend(batch),
SearchEvent::Finished { total, truncated } => {
*finished.lock().unwrap() = Some((total, truncated));
}
SearchEvent::BadPattern(e) => panic!("the pattern failed: {e}"),
},
);
(hits.into_inner().unwrap(), finished.into_inner().unwrap())
}
#[test]
fn a_literal_query_finds_its_lines() {
let (hits, finished) = search("Bearer tokens", SearchConfig::default());
assert_eq!(finished, Some((hits.len(), false)));
assert!(hits
.iter()
.any(|hit| hit.path == Path::new("docs/api/auth.md")));
let hit = &hits[0];
assert!(hit.text.contains("Bearer tokens"));
assert_eq!(&hit.text[hit.span.0..hit.span.1], "Bearer tokens");
assert!(hit.line > 0);
}
#[test]
fn a_literal_query_is_not_read_as_a_pattern() {
let (hits, _) = search("a.*b", SearchConfig::default());
assert!(hits.is_empty(), "{hits:?}");
}
#[test]
fn a_slash_wrapped_query_turns_on_regex_for_that_search() {
let query = Query::parse("/tok[e]n/", &SearchConfig::default());
assert!(query.regex);
assert_eq!(query.pattern, "tok[e]n");
let (hits, _) = search("/Bearer +tokens/", SearchConfig::default());
assert!(!hits.is_empty());
}
#[test]
fn smart_case_matches_either_case_until_the_query_says_otherwise() {
let (lower, _) = search("bearer", SearchConfig::default());
let (upper, _) = search("Bearer", SearchConfig::default());
assert!(!lower.is_empty());
assert_eq!(lower.len(), upper.len());
let (missing, _) = search("BEARER", SearchConfig::default());
assert!(missing.is_empty());
}
#[test]
fn an_invalid_regex_is_reported_rather_than_panicking() {
let reported = Mutex::new(None);
let config = SearchConfig {
regex: true,
..SearchConfig::default()
};
run(
&vault(),
&Query::parse("[unclosed", &config),
&config,
&FilesConfig::default(),
WalkOptions::default(),
&AtomicBool::new(false),
&|event| {
if let SearchEvent::BadPattern(e) = event {
*reported.lock().unwrap() = Some(e);
}
},
);
assert!(reported.into_inner().unwrap().is_some());
}
#[test]
fn the_result_cap_truncates_rather_than_running_on() {
let config = SearchConfig {
max_results: 3,
..SearchConfig::default()
};
let (hits, finished) = search("e", config);
assert!(hits.len() >= 3);
assert_eq!(finished.map(|(_, truncated)| truncated), Some(true));
}
#[test]
fn non_markdown_files_are_searched_only_when_asked_for() {
let (markdown_only, _) = search("notes/", SearchConfig::default());
assert!(!markdown_only
.iter()
.any(|hit| hit.path == Path::new(".gitignore")));
let config = SearchConfig {
all_files: true,
..SearchConfig::default()
};
let (everything, _) = search("notes/", config);
assert!(everything
.iter()
.any(|hit| hit.path == Path::new(".gitignore")));
}
#[test]
fn a_cancelled_search_stops_and_does_not_report_finishing() {
let cancelled = AtomicBool::new(false);
let finished = Mutex::new(false);
let config = SearchConfig::default();
run(
&vault(),
&Query::parse("e", &config),
&config,
&FilesConfig::default(),
WalkOptions::default(),
&cancelled,
&|event| match event {
SearchEvent::Hits(_) => cancelled.store(true, Ordering::Relaxed),
SearchEvent::Finished { .. } => *finished.lock().unwrap() = true,
SearchEvent::BadPattern(_) => {}
},
);
assert!(!finished.into_inner().unwrap());
}
#[test]
fn an_empty_query_finishes_without_reading_anything() {
let (hits, finished) = search(" ", SearchConfig::default());
assert!(hits.is_empty());
assert_eq!(finished, Some((0, false)));
}
}