use serde::{Deserialize, Serialize};
use std::{
path::{Path, PathBuf},
sync::{Arc, Mutex, MutexGuard},
};
use tantivy::{
collector::TopDocs,
query::QueryParser,
schema::*,
snippet::SnippetGenerator,
tokenizer::{LowerCaser, TextAnalyzer},
Index, IndexReader, IndexWriter, TantivyDocument, TantivyError,
};
use tantivy_jieba::JiebaTokenizer;
use crate::workspace_fs::WorkspaceFs;
#[derive(Deserialize)]
pub struct SearchQuery {
pub q: String,
}
#[derive(Serialize, Debug)]
pub struct SearchResult {
pub file_path: String,
pub file_name: String,
pub title: String,
pub snippet: String,
}
pub struct SearchIndex {
index: Index,
reader: IndexReader,
writer: Arc<Mutex<IndexWriter>>,
field_path: Field,
field_file_name: Field,
field_title: Field,
field_content: Field,
start_dir: PathBuf,
workspace_fs: Arc<WorkspaceFs>,
}
impl SearchIndex {
fn empty(workspace_fs: Arc<WorkspaceFs>) -> tantivy::Result<Self> {
let mut schema_builder = Schema::builder();
let text_options = TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer("jieba")
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
.set_stored();
let field_path = schema_builder.add_text_field("path", STRING | STORED);
let field_file_name = schema_builder.add_text_field("file_name", text_options.clone());
let field_title = schema_builder.add_text_field("title", text_options.clone());
let field_content = schema_builder.add_text_field("content", text_options);
let schema = schema_builder.build();
let index = Index::create_in_ram(schema);
let analyzer = TextAnalyzer::builder(JiebaTokenizer {})
.filter(LowerCaser)
.build();
index.tokenizers().register("jieba", analyzer);
let writer = index.writer(50_000_000)?;
let reader = index.reader()?;
Ok(Self {
index,
reader,
writer: Arc::new(Mutex::new(writer)),
field_path,
field_file_name,
field_title,
field_content,
start_dir: workspace_fs.ambient_root().to_path_buf(),
workspace_fs,
})
}
pub fn new(start_dir: &Path) -> tantivy::Result<Self> {
Self::for_workspace(Arc::new(WorkspaceFs::new(start_dir.to_path_buf(), None)))
}
pub(crate) fn for_workspace(workspace_fs: Arc<WorkspaceFs>) -> tantivy::Result<Self> {
let search_index = Self::empty(workspace_fs)?;
search_index.index_workspace()?;
Ok(search_index)
}
pub fn new_single_file(start_dir: &Path, file_name: &str) -> tantivy::Result<Self> {
Self::for_workspace(Arc::new(WorkspaceFs::new(
start_dir.to_path_buf(),
Some(file_name),
)))
}
fn writer(&self) -> tantivy::Result<MutexGuard<'_, IndexWriter>> {
self.writer.lock().map_err(|err| {
TantivyError::SystemError(format!("search index writer mutex poisoned: {err}"))
})
}
fn index_workspace(&self) -> tantivy::Result<()> {
use rayon::prelude::*;
tracing::info!("indexing markdown files in {:?}", self.start_dir);
let paths: Vec<(String, PathBuf)> = self
.workspace_fs
.content_files(usize::MAX)
.into_iter()
.filter(|(rel, _)| rel.as_path().extension().is_some_and(|ext| ext == "md"))
.map(|(rel, path)| (rel.as_route(), path))
.collect();
let docs: Vec<TantivyDocument> = paths
.par_iter()
.filter_map(|(rel, path)| {
let content = self.workspace_fs.read_content_to_string(rel).ok()?;
Some(self.build_document(path, &content))
})
.collect();
{
let mut writer = self.writer()?;
for doc in docs {
writer.add_document(doc)?;
}
writer.commit()?;
}
self.reader.reload()?;
tracing::info!("indexing complete");
Ok(())
}
fn rel_path(&self, path: &Path) -> String {
path.strip_prefix(&self.start_dir)
.unwrap_or(path)
.to_string_lossy()
.to_string()
}
fn build_document(&self, path: &Path, content: &str) -> TantivyDocument {
let relative_path = self
.workspace_fs
.route_for_path(path)
.unwrap_or_else(|| self.rel_path(path));
let file_name = path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("")
.to_string();
let title = content
.lines()
.find(|line| line.starts_with('#'))
.map(|line| line.trim_start_matches('#').trim().to_string())
.unwrap_or_else(|| file_name.clone());
let mut doc = TantivyDocument::default();
doc.add_text(self.field_path, &relative_path);
doc.add_text(self.field_file_name, &file_name);
doc.add_text(self.field_title, &title);
doc.add_text(self.field_content, content);
doc
}
pub fn search(&self, query_str: &str, limit: usize) -> tantivy::Result<Vec<SearchResult>> {
let searcher = self.reader.searcher();
let query_parser = QueryParser::for_index(
&self.index,
vec![self.field_file_name, self.field_title, self.field_content],
);
let query = query_parser.parse_query(query_str)?;
let top_docs = searcher.search(&query, &TopDocs::with_limit(limit))?;
let mut results = Vec::new();
let snippet_generator = SnippetGenerator::create(&searcher, &query, self.field_content)?;
for (_score, doc_address) in top_docs {
let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
let file_path = retrieved_doc
.get_first(self.field_path)
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let file_name = retrieved_doc
.get_first(self.field_file_name)
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let title = retrieved_doc
.get_first(self.field_title)
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let snippet = snippet_generator.snippet_from_doc(&retrieved_doc);
let snippet_html = snippet.to_html();
results.push(SearchResult {
file_path,
file_name,
title,
snippet: snippet_html,
});
}
Ok(results)
}
pub fn update_file(&self, path: &Path) -> tantivy::Result<()> {
let authorized = match self.workspace_fs.resolve_content_input(path) {
Ok(path) => path,
Err(_) => return Ok(()),
};
if authorized.extension().is_none_or(|ext| ext != "md") {
return Ok(());
}
let relative_path = self
.workspace_fs
.route_for_path(&authorized)
.unwrap_or_else(|| self.rel_path(&authorized));
let content = match self.workspace_fs.read_content_to_string(&relative_path) {
Ok(content) => content,
Err(_) => return Ok(()),
};
let doc = self.build_document(&authorized, &content);
{
let mut writer = self.writer()?;
let term = Term::from_field_text(self.field_path, &relative_path);
writer.delete_term(term);
writer.add_document(doc)?;
writer.commit()?;
}
self.reader.reload()?;
tracing::debug!("updated index: {}", relative_path);
Ok(())
}
pub fn delete_file(&self, path: &Path) -> tantivy::Result<()> {
let relative_path = self.rel_path(path);
{
let mut writer = self.writer()?;
let term = Term::from_field_text(self.field_path, &relative_path);
writer.delete_term(term);
writer.commit()?;
}
self.reader.reload()?;
tracing::debug!("removed from index: {}", relative_path);
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::TempDir;
fn create_test_file(dir: &Path, name: &str, content: &str) -> std::io::Result<()> {
let file_path = dir.join(name);
fs::write(file_path, content)
}
#[test]
fn test_search_index_creation() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(dir_path, "test1.md", "# Test Title\nThis is test content.").unwrap();
create_test_file(dir_path, "test2.md", "# Another Title\nMore content here.").unwrap();
let index = SearchIndex::new(dir_path).unwrap();
assert!(index.reader.searcher().num_docs() >= 2);
}
#[test]
fn test_search_basic() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(
dir_path,
"rust.md",
"# Rust Programming\nRust is a systems programming language.",
)
.unwrap();
create_test_file(
dir_path,
"python.md",
"# Python Guide\nPython is easy to learn.",
)
.unwrap();
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("Rust", 10).unwrap();
assert_eq!(results.len(), 1);
assert!(results[0].title.contains("Rust"));
}
#[test]
fn test_search_chinese() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(
dir_path,
"chinese.md",
"# 中文测试\n这是一个中文搜索测试文档。",
)
.unwrap();
create_test_file(
dir_path,
"english.md",
"# English Test\nThis is an English document.",
)
.unwrap();
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("中文", 10).unwrap();
assert_eq!(results.len(), 1);
assert!(results[0].title.contains("中文"));
let results = index.search("测试", 10).unwrap();
assert_eq!(results.len(), 1);
}
#[test]
fn test_search_multi_field() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(
dir_path,
"hello.md",
"# Hello World\nContent about greetings.",
)
.unwrap();
create_test_file(dir_path, "world.md", "# Universe\nThe world is vast.").unwrap();
create_test_file(dir_path, "test.md", "# Testing\nAnother document.").unwrap();
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("greetings", 10).unwrap();
assert_eq!(results.len(), 1);
let results = index.search("Hello", 10).unwrap();
assert!(!results.is_empty());
}
#[test]
fn test_update_file() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
let file_path = dir_path.join("update.md");
create_test_file(dir_path, "update.md", "# Original Title\nOriginal content.").unwrap();
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("Original", 10).unwrap();
assert_eq!(results.len(), 1);
fs::write(&file_path, "# Updated Title\nUpdated content.").unwrap();
index.update_file(&file_path).unwrap();
let results = index.search("Updated", 10).unwrap();
assert_eq!(results.len(), 1);
let results = index.search("Original", 10).unwrap();
assert_eq!(results.len(), 0);
}
#[test]
fn test_delete_file() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
let file_path = dir_path.join("delete.md");
create_test_file(dir_path, "delete.md", "# Delete Me\nThis will be deleted.").unwrap();
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("Delete", 10).unwrap();
assert_eq!(results.len(), 1);
index.delete_file(&file_path).unwrap();
let results = index.search("Delete", 10).unwrap();
assert_eq!(results.len(), 0);
}
#[test]
fn test_search_snippet_generation() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(
dir_path,
"snippet.md",
"# Snippet Test\nThis document contains important information about snippets. Snippets are useful."
).unwrap();
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("snippets", 10).unwrap();
assert_eq!(results.len(), 1);
assert!(!results[0].snippet.is_empty());
assert!(results[0].snippet.contains("<b>") || results[0].snippet.contains("snippet"));
}
#[test]
fn test_ignore_non_markdown_files() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(dir_path, "test.md", "# Markdown\nThis is markdown.").unwrap();
create_test_file(dir_path, "test.txt", "# Not Markdown\nThis is text.").unwrap();
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("Markdown", 10).unwrap();
assert_eq!(results.len(), 1);
let results = index.search("text", 10).unwrap();
assert_eq!(results.len(), 0);
}
#[test]
fn test_empty_query() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(dir_path, "test.md", "# Test\nContent").unwrap();
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("", 10);
assert!(results.is_err() || results.unwrap().is_empty());
}
#[test]
fn test_title_extraction() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(dir_path, "with-title.md", "# Main Title\nContent").unwrap();
create_test_file(dir_path, "no-title.md", "Just content without heading").unwrap();
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("Main", 10).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Main Title");
let results = index.search("no-title", 10).unwrap();
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "no-title");
}
#[test]
fn test_search_limit() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
for i in 0..10 {
create_test_file(
dir_path,
&format!("file{}.md", i),
"# Document\nCommon content",
)
.unwrap();
}
let index = SearchIndex::new(dir_path).unwrap();
let results = index.search("Common", 5).unwrap();
assert_eq!(results.len(), 5);
let results = index.search("Common", 20).unwrap();
assert_eq!(results.len(), 10);
}
#[test]
fn test_subdirectory_relative_paths() {
let temp_dir = TempDir::new().unwrap();
let sub = temp_dir.path().join("notes").join("deep");
fs::create_dir_all(&sub).unwrap();
create_test_file(&sub, "nested.md", "# Nested\nDeep content here").unwrap();
let index = SearchIndex::new(temp_dir.path()).unwrap();
let results = index.search("Deep", 10).unwrap();
assert_eq!(results.len(), 1);
let path = &results[0].file_path;
assert!(
path.starts_with("notes"),
"expected relative path, got: {path}"
);
assert!(
path.contains("nested"),
"expected file name in path, got: {path}"
);
assert!(
!path.starts_with('/'),
"path should be relative, got: {path}"
);
}
#[test]
fn test_update_file_ignores_non_markdown() {
let temp_dir = TempDir::new().unwrap();
create_test_file(temp_dir.path(), "test.md", "# Original\nMarkdown").unwrap();
let index = SearchIndex::new(temp_dir.path()).unwrap();
let txt_path = temp_dir.path().join("notes.txt");
fs::write(&txt_path, "Some text content").unwrap();
index.update_file(&txt_path).unwrap();
let results = index.search("Some text content", 10).unwrap();
assert!(results.is_empty());
}
#[test]
fn test_update_file_no_extension() {
let temp_dir = TempDir::new().unwrap();
create_test_file(temp_dir.path(), "test.md", "# Doc\nContent").unwrap();
let index = SearchIndex::new(temp_dir.path()).unwrap();
let no_ext = temp_dir.path().join("README");
fs::write(&no_ext, "Plain text").unwrap();
index.update_file(&no_ext).unwrap();
let results = index.search("Plain text", 10).unwrap();
assert!(results.is_empty());
}
#[test]
fn test_index_directory_parallel_completeness() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
const N: usize = 50;
for i in 0..N {
let body_repeat = (i % 7) + 1;
let body = "lorem ipsum dolor sit amet ".repeat(body_repeat * 4);
let content =
format!("# Doc {i}\nmarker_token_{i} is unique to this file.\n\n{body}\n");
create_test_file(dir_path, &format!("file_{i:02}.md", i = i), &content).unwrap();
}
let index = SearchIndex::new(dir_path).unwrap();
assert_eq!(
index.reader.searcher().num_docs(),
N as u64,
"parallel indexing dropped documents"
);
for i in [0usize, 7, 23, 49] {
let needle = format!("marker_token_{i}");
let results = index.search(&needle, 10).unwrap();
assert_eq!(
results.len(),
1,
"expected exactly one hit for `{needle}`, got {}",
results.len()
);
}
let results = index.search("Doc", 100).unwrap();
assert_eq!(results.len(), N);
}
#[test]
fn test_writer_poison_returns_error_not_panic() {
let temp_dir = TempDir::new().unwrap();
create_test_file(temp_dir.path(), "seed.md", "# Seed\nseed content").unwrap();
let index = SearchIndex::new(temp_dir.path()).unwrap();
let writer_handle = Arc::clone(&index.writer);
let poisoner = std::thread::spawn(move || {
let _guard = writer_handle.lock().unwrap();
panic!("intentional poison for test");
});
let join_result = poisoner.join();
assert!(
join_result.is_err(),
"poisoner thread was supposed to panic"
);
let new_file = temp_dir.path().join("new.md");
fs::write(&new_file, "# After Poison\nshould not panic").unwrap();
let result = index.update_file(&new_file);
assert!(
result.is_err(),
"update_file should report poisoned writer as an error"
);
let delete_result = index.delete_file(&new_file);
assert!(
delete_result.is_err(),
"delete_file should report poisoned writer as an error"
);
}
#[test]
fn test_search_is_case_insensitive() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(dir_path, "doc.md", "# Heading\nThe Quick BrownFox jumps.").unwrap();
let index = SearchIndex::new(dir_path).unwrap();
assert_eq!(
index.search("quick", 10).unwrap().len(),
1,
"lower-case query must match mixed-case text"
);
assert_eq!(
index.search("QUICK", 10).unwrap().len(),
1,
"upper-case query must match too"
);
assert_eq!(index.search("brownfox", 10).unwrap().len(), 1);
}
#[test]
fn test_single_file_index_contains_exactly_one_doc() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(dir_path, "pinned.md", "# Pinned\nPinned content.").unwrap();
create_test_file(dir_path, "sibling1.md", "# Sibling One\nFirst sibling.").unwrap();
create_test_file(dir_path, "sibling2.md", "# Sibling Two\nSecond sibling.").unwrap();
let index = SearchIndex::new_single_file(dir_path, "pinned.md").unwrap();
assert_eq!(
index.reader.searcher().num_docs(),
1,
"single-file index must contain exactly one document"
);
}
#[test]
fn test_single_file_index_no_sibling_leakage() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
create_test_file(
dir_path,
"pinned.md",
"# Pinned Document\nuniquepinnedtoken lives here.",
)
.unwrap();
create_test_file(
dir_path,
"secret-sibling.md",
"# Secret Sibling\nuniquesiblingtoken must stay private.",
)
.unwrap();
let index = SearchIndex::new_single_file(dir_path, "pinned.md").unwrap();
let leaked = index.search("uniquesiblingtoken", 10).unwrap();
assert!(
leaked.is_empty(),
"single-file index leaked a sibling file: {leaked:?}"
);
let hits = index.search("uniquepinnedtoken", 10).unwrap();
assert_eq!(hits.len(), 1, "pinned file should be searchable");
assert_eq!(hits[0].file_path, "pinned.md");
}
#[test]
fn test_single_file_index_relative_path_is_file_name() {
let temp_dir = TempDir::new().unwrap();
let dir_path = temp_dir.path();
let file_path = dir_path.join("note.md");
create_test_file(dir_path, "note.md", "# Note\noriginaltoken here.").unwrap();
let index = SearchIndex::new_single_file(dir_path, "note.md").unwrap();
let hits = index.search("originaltoken", 10).unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].file_path, "note.md");
fs::write(&file_path, "# Note\nupdatedtoken here.").unwrap();
index.update_file(&file_path).unwrap();
assert_eq!(
index.reader.searcher().num_docs(),
1,
"update_file should keep the single-file index at one document"
);
assert!(index.search("originaltoken", 10).unwrap().is_empty());
assert_eq!(index.search("updatedtoken", 10).unwrap().len(), 1);
}
#[cfg(unix)]
#[test]
fn test_directory_index_rejects_outside_symlink_on_build_and_update() {
use std::os::unix::fs::symlink;
let workspace = TempDir::new().unwrap();
let outside = TempDir::new().unwrap();
let secret = outside.path().join("secret.md");
fs::write(&secret, "uniquesymlinksecrettoken").unwrap();
let link = workspace.path().join("linked-secret.md");
symlink(&secret, &link).unwrap();
let index = SearchIndex::new(workspace.path()).unwrap();
assert!(index
.search("uniquesymlinksecrettoken", 10)
.unwrap()
.is_empty());
index.update_file(&link).unwrap();
assert!(index
.search("uniquesymlinksecrettoken", 10)
.unwrap()
.is_empty());
}
}