use std::ops::{Bound, Range};
use std::path::Path;
use anyhow::{Context, Result, bail};
use tantivy::collector::{Count, TopDocs};
use tantivy::query::{
AllQuery, BooleanQuery, Occur, PhraseQuery, Query, QueryParser, RangeQuery, TermQuery,
};
use tantivy::schema::{IndexRecordOption, Value};
use tantivy::snippet::SnippetGenerator;
use tantivy::tokenizer::{TokenStream, Tokenizer};
use tantivy::{DocAddress, Index, IndexReader, Order, TantivyDocument, Term};
use crate::query::{self, Cmp};
use crate::{Fields, build_schema, register_tokenizers};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum SortMode {
#[default]
Relevance,
MtimeDesc,
MtimeAsc,
SizeDesc,
}
impl SortMode {
pub fn parse(name: Option<&str>) -> Self {
match name {
Some("mtime_desc") => Self::MtimeDesc,
Some("mtime_asc") => Self::MtimeAsc,
Some("size_desc") => Self::SizeDesc,
_ => Self::Relevance,
}
}
}
const PREVIEW_MAX_CHARS: usize = 1500;
const SNIPPET_MAX_CHARS: usize = 160;
const SNIPPET_SCAN_MAX_BYTES: usize = 128 * 1024;
pub struct SearchHit {
pub path: String,
pub score: f32,
pub snippet: String,
pub highlighted: Vec<Range<usize>>,
pub ext: String,
}
pub struct PreviewHit {
pub snippet: String,
pub highlighted: Vec<Range<usize>>,
pub size: u64,
pub mtime: i64,
pub ext: String,
}
pub struct SearchPage {
pub hits: Vec<SearchHit>,
pub total: usize,
}
pub struct Searcher {
reader: IndexReader,
parser: QueryParser,
fields: Fields,
}
impl Searcher {
pub fn open(index_dir: &Path) -> Result<Self> {
let index = Index::open_in_dir(index_dir)
.context("打不开索引目录,先跑 `dowse index <目录>` 建一次索引")?;
crate::meta::ensure_schema_version(index_dir)?;
register_tokenizers(&index);
let (_, fields) = build_schema();
let mut parser = QueryParser::for_index(&index, vec![fields.name, fields.content]);
parser.set_conjunction_by_default();
let reader = index.reader()?;
Ok(Self {
reader,
parser,
fields,
})
}
pub fn search(&self, query_str: &str, limit: usize) -> Result<Vec<SearchHit>> {
self.search_advanced(query_str, limit, None, SortMode::Relevance)
}
pub fn search_filtered(
&self,
query_str: &str,
limit: usize,
ext: Option<&str>,
) -> Result<Vec<SearchHit>> {
let group: Option<&[&str]> = ext.as_ref().map(std::slice::from_ref);
self.search_advanced(query_str, limit, group, SortMode::Relevance)
}
pub fn search_advanced(
&self,
query_str: &str,
limit: usize,
ext_group: Option<&[&str]>,
sort: SortMode,
) -> Result<Vec<SearchHit>> {
Ok(self
.search_paged(query_str, limit, 0, ext_group, sort)?
.hits)
}
pub fn search_paged(
&self,
query_str: &str,
limit: usize,
offset: usize,
ext_group: Option<&[&str]>,
sort: SortMode,
) -> Result<SearchPage> {
let (retrieval_query, snippet_query) = self.build_queries(query_str)?;
let query: Box<dyn Query> = match ext_group {
Some(exts) if !exts.is_empty() => {
let ext_should: Vec<(Occur, Box<dyn Query>)> = exts
.iter()
.map(|ext| {
let term =
Term::from_field_text(self.fields.ext, &ext.to_ascii_lowercase());
(
Occur::Should,
Box::new(TermQuery::new(term, IndexRecordOption::Basic))
as Box<dyn Query>,
)
})
.collect();
Box::new(BooleanQuery::new(vec![
(Occur::Must, retrieval_query),
(
Occur::Must,
Box::new(BooleanQuery::new(ext_should)) as Box<dyn Query>,
),
]))
}
_ => retrieval_query,
};
let searcher = self.reader.searcher();
let mut snippet_gen =
SnippetGenerator::create(&searcher, &snippet_query, self.fields.content)?;
snippet_gen.set_max_num_chars(SNIPPET_MAX_CHARS);
let (addrs, total): (Vec<(f32, DocAddress)>, usize) = match sort {
SortMode::Relevance => {
let (top, total) = searcher.search(
&query,
&(
TopDocs::with_limit(limit)
.and_offset(offset)
.order_by_score(),
Count,
),
)?;
(top, total)
}
SortMode::MtimeDesc => {
let (top, total) = searcher.search(
&query,
&(
TopDocs::with_limit(limit)
.and_offset(offset)
.order_by_fast_field::<i64>("mtime", Order::Desc),
Count,
),
)?;
(
top.into_iter().map(|(_, addr)| (0.0, addr)).collect(),
total,
)
}
SortMode::MtimeAsc => {
let (top, total) = searcher.search(
&query,
&(
TopDocs::with_limit(limit)
.and_offset(offset)
.order_by_fast_field::<i64>("mtime", Order::Asc),
Count,
),
)?;
(
top.into_iter().map(|(_, addr)| (0.0, addr)).collect(),
total,
)
}
SortMode::SizeDesc => {
let (top, total) = searcher.search(
&query,
&(
TopDocs::with_limit(limit)
.and_offset(offset)
.order_by_fast_field::<u64>("size", Order::Desc),
Count,
),
)?;
(
top.into_iter().map(|(_, addr)| (0.0, addr)).collect(),
total,
)
}
};
let mut hits = Vec::with_capacity(addrs.len());
for (score, addr) in addrs {
let doc: TantivyDocument = searcher.doc(addr)?;
let path = doc
.get_first(self.fields.path)
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_owned();
let content = doc
.get_first(self.fields.content)
.and_then(|v| v.as_str())
.unwrap_or_default();
let ext = doc
.get_first(self.fields.ext)
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_owned();
let (snippet, highlighted) =
snippet_with_fallback(&snippet_gen, content, SNIPPET_MAX_CHARS);
hits.push(SearchHit {
path,
score,
snippet,
highlighted,
ext,
});
}
Ok(SearchPage { hits, total })
}
fn build_queries(&self, query_str: &str) -> Result<(Box<dyn Query>, Box<dyn Query>)> {
let parsed = query::parse(query_str)?;
if !parsed.has_operators {
return Ok((
self.parser.parse_query(query_str)?,
self.parser.parse_query(query_str)?,
));
}
self.build_from_parsed(&parsed)
}
fn build_from_parsed(
&self,
parsed: &query::Parsed,
) -> Result<(Box<dyn Query>, Box<dyn Query>)> {
let mut group_queries: Vec<Box<dyn Query>> = Vec::new();
let mut content_only: Vec<(Occur, Box<dyn Query>)> = Vec::new();
for group in &parsed.groups {
let mut clauses: Vec<(Occur, Box<dyn Query>)> = Vec::new();
for term in &group.content {
clauses.push((Occur::Must, self.parser.parse_query(term)?));
content_only.push((Occur::Should, self.parser.parse_query(term)?));
}
for term in &group.excluded {
clauses.push((Occur::MustNot, self.parser.parse_query(term)?));
}
for path in &group.paths {
clauses.push((Occur::Must, self.build_path_query(path)?));
}
for bound in &group.mtimes {
clauses.push((Occur::Must, self.build_mtime_query(bound)));
}
for bound in &group.sizes {
clauses.push((Occur::Must, self.build_size_query(bound)));
}
let has_positive = clauses.iter().any(|(occ, _)| *occ != Occur::MustNot);
if !has_positive {
clauses.push((Occur::Must, Box::new(AllQuery)));
}
group_queries.push(Box::new(BooleanQuery::new(clauses)));
}
let retrieval: Box<dyn Query> = if group_queries.len() == 1 {
group_queries.pop().expect("已判定非空")
} else {
let shoulds = group_queries
.into_iter()
.map(|q| (Occur::Should, q))
.collect();
Box::new(BooleanQuery::new(shoulds))
};
let snippet: Box<dyn Query> = Box::new(BooleanQuery::new(content_only));
Ok((retrieval, snippet))
}
fn build_path_query(&self, path: &query::PathTerm) -> Result<Box<dyn Query>> {
let mut tokenizer = crate::tokenizer::MixedTokenizer::new();
let mut stream = tokenizer.token_stream(&path.operand);
let mut terms: Vec<Term> = Vec::new();
while stream.advance() {
terms.push(Term::from_field_text(
self.fields.path_text,
&stream.token().text,
));
}
if terms.is_empty() {
bail!(
"path: 操作数 \"{}\" 里没有可检索的内容(全是标点/空白?)",
path.operand
);
}
if path.phrase && terms.len() >= 2 {
return Ok(Box::new(PhraseQuery::new(terms)));
}
if terms.len() == 1 {
let term = terms.pop().expect("刚判过长度为 1");
return Ok(Box::new(TermQuery::new(term, IndexRecordOption::Basic)));
}
let clauses: Vec<(Occur, Box<dyn Query>)> = terms
.into_iter()
.map(|t| {
(
Occur::Must,
Box::new(TermQuery::new(t, IndexRecordOption::Basic)) as Box<dyn Query>,
)
})
.collect();
Ok(Box::new(BooleanQuery::new(clauses)))
}
fn build_mtime_query(&self, bound: &query::DateBound) -> Box<dyn Query> {
let field = self.fields.mtime;
let start = Term::from_field_i64(field, bound.start_ms);
let next = Term::from_field_i64(field, bound.next_ms);
let (lower, upper) = match bound.cmp {
Cmp::Gt => (Bound::Included(next), Bound::Unbounded),
Cmp::Ge => (Bound::Included(start), Bound::Unbounded),
Cmp::Lt => (Bound::Unbounded, Bound::Excluded(start)),
Cmp::Le => (Bound::Unbounded, Bound::Excluded(next)),
};
Box::new(RangeQuery::new(lower, upper))
}
fn build_size_query(&self, bound: &query::SizeBound) -> Box<dyn Query> {
let field = self.fields.size;
let term = Term::from_field_u64(field, bound.bytes);
let (lower, upper) = match bound.cmp {
Cmp::Gt => (Bound::Excluded(term), Bound::Unbounded),
Cmp::Ge => (Bound::Included(term), Bound::Unbounded),
Cmp::Lt => (Bound::Unbounded, Bound::Excluded(term)),
Cmp::Le => (Bound::Unbounded, Bound::Included(term)),
};
Box::new(RangeQuery::new(lower, upper))
}
pub fn num_docs(&self) -> u64 {
self.reader.searcher().num_docs()
}
pub fn count_under(&self, root: &Path) -> Result<u64> {
let exact = root.to_string_lossy().into_owned();
let mut prefix = exact.clone();
if !prefix.ends_with(std::path::MAIN_SEPARATOR) {
prefix.push(std::path::MAIN_SEPARATOR);
}
let upper = format!("{prefix}\u{10FFFF}");
let exact_query = TermQuery::new(
Term::from_field_text(self.fields.path, &exact),
IndexRecordOption::Basic,
);
let subtree_query = RangeQuery::new(
Bound::Included(Term::from_field_text(self.fields.path, &prefix)),
Bound::Excluded(Term::from_field_text(self.fields.path, &upper)),
);
let query = BooleanQuery::new(vec![
(Occur::Should, Box::new(exact_query) as Box<dyn Query>),
(Occur::Should, Box::new(subtree_query) as Box<dyn Query>),
]);
let searcher = self.reader.searcher();
Ok(searcher.search(&query, &Count)? as u64)
}
pub fn reload(&self) -> Result<()> {
self.reader.reload().context("索引 reader 重载失败")
}
pub fn preview(&self, path: &str, query_str: &str) -> Result<Option<PreviewHit>> {
let searcher = self.reader.searcher();
let path_term = Term::from_field_text(self.fields.path, path);
let path_query = TermQuery::new(path_term, IndexRecordOption::Basic);
let (_retrieval, content_query) = self.build_queries(query_str)?;
let combined = BooleanQuery::new(vec![
(Occur::Must, Box::new(path_query.clone()) as Box<dyn Query>),
(Occur::Must, content_query),
]);
let top_docs = searcher.search(&combined, &TopDocs::with_limit(1).order_by_score())?;
if let Some((_, addr)) = top_docs.into_iter().next() {
let mut snippet_gen =
SnippetGenerator::create(&searcher, &combined, self.fields.content)?;
snippet_gen.set_max_num_chars(PREVIEW_MAX_CHARS);
let doc: TantivyDocument = searcher.doc(addr)?;
let content = doc
.get_first(self.fields.content)
.and_then(|v| v.as_str())
.unwrap_or_default();
let (snippet, highlighted) =
snippet_with_fallback(&snippet_gen, content, PREVIEW_MAX_CHARS);
let (size, mtime, ext) = self.doc_meta(&doc);
return Ok(Some(PreviewHit {
snippet,
highlighted,
size,
mtime,
ext,
}));
}
let fallback = searcher.search(&path_query, &TopDocs::with_limit(1).order_by_score())?;
let Some((_, addr)) = fallback.into_iter().next() else {
return Ok(None);
};
let doc: TantivyDocument = searcher.doc(addr)?;
let content = doc
.get_first(self.fields.content)
.and_then(|v| v.as_str())
.unwrap_or_default();
let window: String = content.chars().take(PREVIEW_MAX_CHARS).collect();
let (size, mtime, ext) = self.doc_meta(&doc);
Ok(Some(PreviewHit {
snippet: window,
highlighted: vec![],
size,
mtime,
ext,
}))
}
fn doc_meta(&self, doc: &TantivyDocument) -> (u64, i64, String) {
let size = doc
.get_first(self.fields.size)
.and_then(|v| v.as_u64())
.unwrap_or(0);
let mtime = doc
.get_first(self.fields.mtime)
.and_then(|v| v.as_i64())
.unwrap_or(0);
let ext = doc
.get_first(self.fields.ext)
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_owned();
(size, mtime, ext)
}
}
pub fn normalize_ranges(mut ranges: Vec<Range<usize>>) -> Vec<Range<usize>> {
ranges.sort_by_key(|r| r.start);
let mut merged: Vec<Range<usize>> = Vec::with_capacity(ranges.len());
for r in ranges {
match merged.last_mut() {
Some(last) if r.start <= last.end => {
if r.end > last.end {
last.end = r.end;
}
}
_ => merged.push(r),
}
}
merged
}
fn truncate_scan_window(content: &str) -> &str {
if content.len() <= SNIPPET_SCAN_MAX_BYTES {
return content;
}
let mut end = SNIPPET_SCAN_MAX_BYTES;
while end > 0 && !content.is_char_boundary(end) {
end -= 1;
}
&content[..end]
}
fn snippet_with_fallback(
snippet_gen: &SnippetGenerator,
content: &str,
fallback_chars: usize,
) -> (String, Vec<Range<usize>>) {
let scan_window = truncate_scan_window(content);
let snippet = snippet_gen.snippet(scan_window);
if !snippet.highlighted().is_empty() {
return (
snippet.fragment().to_owned(),
normalize_ranges(snippet.highlighted().to_vec()),
);
}
let head: String = content.chars().take(fallback_chars).collect();
(head, Vec::new())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalize_ranges_empty() {
assert_eq!(normalize_ranges(vec![]), Vec::<Range<usize>>::new());
}
#[test]
fn normalize_ranges_out_of_order() {
let input = vec![10..13, 0..2, 5..7];
assert_eq!(normalize_ranges(input), vec![0..2, 5..7, 10..13]);
}
#[test]
fn normalize_ranges_fully_overlapping() {
let input = vec![0..9, 0..6];
assert_eq!(normalize_ranges(input), vec![0..9]);
}
#[test]
fn normalize_ranges_partially_overlapping() {
let input = vec![0..6, 3..9];
assert_eq!(normalize_ranges(input), vec![0..9]);
}
#[test]
fn normalize_ranges_adjacent() {
let input = vec![0..3, 3..6];
assert_eq!(normalize_ranges(input), vec![0..6]);
}
#[test]
fn search_jieba_overlapping_tokens_yields_sorted_nonoverlapping_ranges() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(
target_dir.path().join("note.md"),
"系统采用分布式限流器保护后端服务。",
)?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search("分布式", 10)?;
assert!(!hits.is_empty(), "应该能搜到刚建的文档");
for hit in &hits {
for w in hit.highlighted.windows(2) {
assert!(
w[0].end <= w[1].start,
"区间应互不重叠且按起点排序: {:?}",
hit.highlighted
);
}
for r in &hit.highlighted {
assert!(r.start <= r.end);
assert!(r.end <= hit.snippet.len());
}
}
Ok(())
}
#[test]
fn search_is_case_insensitive_for_latin() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("api.md"), "API design notes")?;
std::fs::write(target_dir.path().join("fs.md"), "File system layout")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
for (query, expected) in [
("api", "api.md"),
("API", "api.md"),
("file", "fs.md"),
("FILE", "fs.md"),
] {
let hits = searcher.search(query, 10)?;
assert_eq!(hits.len(), 1, "查询 {query:?} 应命中一篇");
assert!(
hits[0].path.ends_with(expected),
"查询 {query:?} 应命中 {expected},实际 {:?}",
hits[0].path
);
}
Ok(())
}
#[test]
fn search_hyphenated_string_matches_subwords() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("covid.md"), "covid-19 vaccine")?;
std::fs::write(
target_dir.path().join("marker.md"),
"glimmerfrost-9931-unique-marker",
)?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
for (query, expected) in [
("covid", "covid.md"),
("19", "covid.md"),
("glimmerfrost", "marker.md"),
("9931", "marker.md"),
("marker", "marker.md"),
] {
let hits = searcher.search(query, 10)?;
assert!(
hits.iter().any(|h| h.path.ends_with(expected)),
"子词查询 {query:?} 应命中 {expected},实际 {:?}",
hits.iter().map(|h| &h.path).collect::<Vec<_>>()
);
}
let full = searcher.search("glimmerfrost-9931-unique-marker", 10)?;
assert!(
full.iter().any(|h| h.path.ends_with("marker.md")),
"整串查询应命中 marker.md,实际 {:?}",
full.iter().map(|h| &h.path).collect::<Vec<_>>()
);
Ok(())
}
#[test]
fn search_mixed_cjk_latin_document() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(
target_dir.path().join("mixed.md"),
"用 GPT-4 写的 covid-19 报告",
)?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
for query in ["gpt", "covid", "报告"] {
let hits = searcher.search(query, 10)?;
assert!(
hits.iter().any(|h| h.path.ends_with("mixed.md")),
"混合文档查询 {query:?} 应命中 mixed.md,实际 {:?}",
hits.iter().map(|h| &h.path).collect::<Vec<_>>()
);
}
Ok(())
}
#[test]
fn search_mixed_document_highlight_ranges_are_valid() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(
target_dir.path().join("mixed.md"),
"这是一份关于 glimmerfrost-9931 的中文说明,包含 API 设计与 covid-19 数据。",
)?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search("glimmerfrost", 10)?;
assert_eq!(hits.len(), 1);
let hit = &hits[0];
assert!(!hit.highlighted.is_empty(), "命中词应该被标记出来");
for w in hit.highlighted.windows(2) {
assert!(w[0].end <= w[1].start, "区间应互不重叠且按起点排序");
}
for r in &hit.highlighted {
assert!(r.start <= r.end);
assert!(r.end <= hit.snippet.len());
assert!(hit.snippet.is_char_boundary(r.start));
assert!(hit.snippet.is_char_boundary(r.end));
assert_eq!(hit.snippet[r.start..r.end].to_lowercase(), "glimmerfrost");
}
Ok(())
}
#[test]
fn phrase_query_matches_adjacent_terms_under_sequential_positions() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(
target_dir.path().join("marker.md"),
"glimmerfrost-9931-unique-marker",
)?;
std::fs::write(
target_dir.path().join("cn.md"),
"系统采用分布式限流器保护后端服务。",
)?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let phrase = searcher.search("\"unique marker\"", 10)?;
assert!(
phrase.iter().any(|h| h.path.ends_with("marker.md")),
"短语 \"unique marker\" 应命中 marker.md,实际 {:?}",
phrase.iter().map(|h| &h.path).collect::<Vec<_>>()
);
let reversed = searcher.search("\"marker unique\"", 10)?;
assert!(
!reversed.iter().any(|h| h.path.ends_with("marker.md")),
"词序颠倒的短语不该命中,实际 {:?}",
reversed.iter().map(|h| &h.path).collect::<Vec<_>>()
);
let cn = searcher.search("\"分布式\"", 10)?;
assert!(
cn.iter().any(|h| h.path.ends_with("cn.md")),
"中文短语应命中 cn.md,实际 {:?}",
cn.iter().map(|h| &h.path).collect::<Vec<_>>()
);
Ok(())
}
#[test]
fn search_multi_word_query_defaults_to_and() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("both.md"), "限流中间件的实现细节")?;
std::fs::write(target_dir.path().join("one.md"), "限流方案对比笔记")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search("限流 中间件", 10)?;
assert_eq!(
hits.len(),
1,
"AND 语义下只有同时含两个词的文档命中: {hits:?}",
hits = hits.iter().map(|h| &h.path).collect::<Vec<_>>()
);
assert!(hits[0].path.ends_with("both.md"));
Ok(())
}
#[test]
fn search_filtered_by_ext_only_matches_that_extension() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("note.md"), "限流方案对比")?;
std::fs::write(target_dir.path().join("note.txt"), "限流方案对比")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search_filtered("限流", 10, Some("md"))?;
assert_eq!(
hits.len(),
1,
"ext 过滤后应该只剩 .md 那篇: {hits:?}",
hits = hits.iter().map(|h| &h.path).collect::<Vec<_>>()
);
assert!(hits[0].path.ends_with("note.md"));
assert_eq!(hits[0].ext, "md");
let unfiltered = searcher.search("限流", 10)?;
assert_eq!(unfiltered.len(), 2);
Ok(())
}
#[test]
fn search_advanced_ext_group_matches_only_group_members() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("note.md"), "限流方案")?;
std::fs::write(target_dir.path().join("main.rs"), "限流方案 fn main")?;
std::fs::write(target_dir.path().join("note.txt"), "限流方案")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let code_hits = searcher.search_advanced(
"限流",
10,
Some(crate::ext_groups::CODE),
SortMode::Relevance,
)?;
assert_eq!(
code_hits.len(),
1,
"代码分组应该只命中 .rs: {:?}",
code_hits.iter().map(|h| &h.path).collect::<Vec<_>>()
);
assert!(code_hits[0].path.ends_with("main.rs"));
let doc_hits = searcher.search_advanced(
"限流",
10,
Some(crate::ext_groups::DOC),
SortMode::Relevance,
)?;
assert_eq!(
doc_hits.len(),
2,
"文档分组应该命中 .md 和 .txt 两篇: {:?}",
doc_hits.iter().map(|h| &h.path).collect::<Vec<_>>()
);
Ok(())
}
#[test]
fn search_sorted_by_mtime_desc_orders_newest_first() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("old.md"), "笔记 alpha")?;
std::thread::sleep(std::time::Duration::from_millis(20));
std::fs::write(target_dir.path().join("new.md"), "笔记 beta")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search_advanced("笔记", 10, None, SortMode::MtimeDesc)?;
assert_eq!(hits.len(), 2);
assert!(
hits[0].path.ends_with("new.md"),
"mtime 降序,最新的应排第一: {:?}",
hits.iter().map(|h| &h.path).collect::<Vec<_>>()
);
assert!(hits[1].path.ends_with("old.md"));
Ok(())
}
#[test]
fn search_sorted_by_mtime_asc_orders_oldest_first() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("old.md"), "笔记 alpha")?;
std::thread::sleep(std::time::Duration::from_millis(20));
std::fs::write(target_dir.path().join("new.md"), "笔记 beta")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search_advanced("笔记", 10, None, SortMode::MtimeAsc)?;
assert_eq!(hits.len(), 2);
assert!(
hits[0].path.ends_with("old.md"),
"mtime 升序,最旧的应排第一: {:?}",
hits.iter().map(|h| &h.path).collect::<Vec<_>>()
);
assert!(hits[1].path.ends_with("new.md"));
Ok(())
}
#[test]
fn search_sorted_by_size_desc_orders_largest_first() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("small.md"), "笔记")?;
std::fs::write(target_dir.path().join("big.md"), "笔记".repeat(500))?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search_advanced("笔记", 10, None, SortMode::SizeDesc)?;
assert_eq!(hits.len(), 2);
assert!(
hits[0].path.ends_with("big.md"),
"size 降序,最大的应排第一: {:?}",
hits.iter().map(|h| &h.path).collect::<Vec<_>>()
);
assert!(hits[1].path.ends_with("small.md"));
Ok(())
}
#[test]
fn preview_returns_window_around_hit_with_normalized_ranges() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
let filler_before = "无关内容。".repeat(400);
let filler_after = "更多无关内容。".repeat(400);
let content = format!("{filler_before}这里是分布式限流器的核心实现。{filler_after}");
let doc_path = target_dir.path().join("long.md");
std::fs::write(&doc_path, &content)?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search("限流器", 10)?;
assert_eq!(hits.len(), 1);
let full_path = &hits[0].path;
let preview = searcher
.preview(full_path, "限流器")?
.expect("文档存在,preview 不应为 None");
assert!(!preview.snippet.is_empty());
assert!(
preview.snippet.chars().count() < content.chars().count(),
"预览窗口应该比全文短——验证确实做了截窗而不是整篇塞回来"
);
assert!(!preview.highlighted.is_empty(), "命中词应该被标记出来");
for w in preview.highlighted.windows(2) {
assert!(w[0].end <= w[1].start, "区间应互不重叠且按起点排序");
}
for r in &preview.highlighted {
assert!(r.end <= preview.snippet.len());
assert!(preview.snippet.is_char_boundary(r.start));
assert!(preview.snippet.is_char_boundary(r.end));
}
Ok(())
}
#[test]
fn count_under_only_counts_docs_within_root_prefix() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("a.md"), "内容")?;
std::fs::write(target_dir.path().join("b.md"), "内容")?;
let sibling_named_with_shared_prefix = target_dir.path().with_file_name(format!(
"{}-sibling",
target_dir.path().file_name().unwrap().to_string_lossy()
));
std::fs::create_dir_all(&sibling_named_with_shared_prefix)?;
std::fs::write(sibling_named_with_shared_prefix.join("c.md"), "内容")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
assert_eq!(searcher.count_under(target_dir.path())?, 2);
std::fs::remove_dir_all(&sibling_named_with_shared_prefix).ok();
Ok(())
}
#[test]
fn preview_unknown_path_returns_none() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("note.md"), "随便写点什么")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let preview = searcher.preview("C:\\不存在\\的路径.md", "什么")?;
assert!(preview.is_none());
Ok(())
}
#[test]
fn preview_falls_back_to_head_when_query_does_not_match_doc() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("note.md"), "这篇笔记不含查询词")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search("笔记", 10)?;
let path = &hits[0].path;
let preview = searcher
.preview(path, "完全不相关的词汇")?
.expect("路径存在,应该退回开头预览而不是 None");
assert!(!preview.snippet.is_empty());
assert!(preview.highlighted.is_empty(), "回退分支不应该产生假的高亮");
Ok(())
}
#[test]
fn search_path_operator_matches_by_directory_and_filename() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
let sub = target_dir.path().join("zzreportfolder");
std::fs::create_dir_all(&sub)?;
std::fs::write(sub.join("note.md"), "内容")?;
std::fs::write(target_dir.path().join("other.md"), "内容")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let by_dir = searcher.search("path:zzreportfolder", 10)?;
assert_eq!(
by_dir.len(),
1,
"path: 目录名应只中一篇: {by_dir:?}",
by_dir = by_dir.iter().map(|h| &h.path).collect::<Vec<_>>()
);
assert!(by_dir[0].path.ends_with("note.md"));
let by_name = searcher.search("path:other", 10)?;
assert_eq!(by_name.len(), 1);
assert!(by_name[0].path.ends_with("other.md"));
let combined = searcher.search("内容 path:zzreportfolder", 10)?;
assert_eq!(combined.len(), 1);
assert!(combined[0].path.ends_with("note.md"));
Ok(())
}
#[test]
fn search_path_operator_quoted_phrase_operand() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
let sub = target_dir.path().join("my docs");
std::fs::create_dir_all(&sub)?;
std::fs::write(sub.join("a.md"), "内容")?;
std::fs::write(target_dir.path().join("b.md"), "内容")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search("path:\"my docs\"", 10)?;
assert_eq!(hits.len(), 1, "带引号的 path 操作数应按整体匹配那一篇");
assert!(hits[0].path.ends_with("a.md"));
Ok(())
}
#[test]
fn search_mtime_operator_filters_by_date() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("note.md"), "笔记内容")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
assert_eq!(searcher.search("笔记 mtime:>2000-01-01", 10)?.len(), 1);
assert_eq!(searcher.search("笔记 mtime:>=2000-01", 10)?.len(), 1);
assert_eq!(
searcher.search("笔记 mtime:<2000-01-01", 10)?.len(),
0,
"2000 年之前的过滤应把当前文件排除"
);
assert_eq!(
searcher.search("笔记 mtime:>2099-01-01", 10)?.len(),
0,
"未来日期之后的过滤应无命中"
);
Ok(())
}
#[test]
fn search_size_operator_filters_by_bytes() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("small.md"), "小")?;
std::fs::write(target_dir.path().join("big.md"), "大".repeat(2000))?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let big = searcher.search("size:>1kb", 10)?;
assert_eq!(big.len(), 1, "size:>1kb 只应中大文件");
assert!(big[0].path.ends_with("big.md"));
let small = searcher.search("size:<1kb", 10)?;
assert_eq!(small.len(), 1, "size:<1kb 只应中小文件");
assert!(small[0].path.ends_with("small.md"));
Ok(())
}
#[test]
fn search_or_groups_and_within_group_and() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("a.md"), "alpha 内容")?;
std::fs::write(target_dir.path().join("b.md"), "beta 内容")?;
std::fs::write(target_dir.path().join("c.md"), "gamma 内容")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let either = searcher.search("alpha OR beta", 10)?;
assert_eq!(either.len(), 2, "alpha OR beta 应中两篇");
let one = searcher.search("alpha OR zzznotthere", 10)?;
assert_eq!(one.len(), 1);
assert!(one[0].path.ends_with("a.md"));
let grouped = searcher.search("alpha zzmissing OR beta", 10)?;
assert_eq!(grouped.len(), 1, "AND 组优先,只应中 beta 那篇");
assert!(grouped[0].path.ends_with("b.md"));
Ok(())
}
#[test]
fn search_exclusion_via_dash_and_not() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("keep.md"), "限流 中间件")?;
std::fs::write(target_dir.path().join("drop.md"), "限流 废弃")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
for query in ["限流 -废弃", "限流 NOT 废弃"] {
let hits = searcher.search(query, 10)?;
assert_eq!(hits.len(), 1, "查询 {query:?} 应排除含'废弃'那篇");
assert!(hits[0].path.ends_with("keep.md"), "查询 {query:?} 结果不对");
}
Ok(())
}
#[test]
fn search_invalid_operand_errors_not_silently_ignored() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("note.md"), "内容")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
assert!(searcher.search("mtime:>abc", 10).is_err());
assert!(searcher.search("size:>10tb", 10).is_err());
assert!(
searcher.search("mtime:2026-01-01", 10).is_err(),
"缺比较符应报错"
);
Ok(())
}
#[test]
fn plain_query_without_operators_is_unchanged_regression() -> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
std::fs::write(target_dir.path().join("both.md"), "限流中间件的实现")?;
std::fs::write(target_dir.path().join("one.md"), "限流方案对比")?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let and_hits = searcher.search("限流 中间件", 10)?;
assert_eq!(and_hits.len(), 1);
assert!(and_hits[0].path.ends_with("both.md"));
let phrase = searcher.search("\"限流中间件\"", 10)?;
assert!(phrase.iter().any(|h| h.path.ends_with("both.md")));
Ok(())
}
#[test]
fn search_snippet_scan_truncation_keeps_retrieval_and_falls_back_beyond_scan_window()
-> Result<()> {
let index_dir = tempfile::tempdir()?;
let target_dir = tempfile::Builder::new().prefix("dowse-test-").tempdir()?;
const SENTINEL: &str = "zzzsentinelprobe888";
let filler = "填充内容占位符文本。".repeat(20_000);
assert!(
filler.len() > SNIPPET_SCAN_MAX_BYTES,
"语料要真的超过截断阈值才能验证截断生效"
);
let big_content = format!("{filler}文档末尾出现了 {SENTINEL} 这个词。");
std::fs::write(target_dir.path().join("big.md"), &big_content)?;
let small_content = format!("{SENTINEL} 出现在这篇小文档的开头。");
std::fs::write(target_dir.path().join("small.md"), &small_content)?;
crate::rebuild_index(index_dir.path(), target_dir.path())?;
let searcher = Searcher::open(index_dir.path())?;
let hits = searcher.search(SENTINEL, 10)?;
assert_eq!(hits.len(), 2, "检索不该被摘要截断影响,两篇文档都要命中");
let big_hit = hits
.iter()
.find(|h| h.path.ends_with("big.md"))
.expect("大文档应该命中");
assert!(
big_hit.highlighted.is_empty(),
"命中词被截断窗口挡在外面,摘要应退回无高亮兜底"
);
assert!(!big_hit.snippet.is_empty(), "兜底摘要不应该是空字符串");
let small_hit = hits
.iter()
.find(|h| h.path.ends_with("small.md"))
.expect("小文档应该命中");
assert!(
!small_hit.highlighted.is_empty(),
"命中词在扫描窗口内,摘要应该正常高亮"
);
Ok(())
}
}