use std::collections::HashSet;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use tantivy::collector::TopDocs;
use tantivy::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError};
use tantivy::directory::{
Directory, DirectoryLock, FileHandle, Lock, MmapDirectory, WatchCallback, WatchHandle, WritePtr,
};
use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
use tantivy::schema::{
Field, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Value, STORED, STRING,
};
use tantivy::tokenizer::TextAnalyzer;
use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, Term};
use crate::error::{MemdError, Result};
use crate::index::sparse::{SparseIndex, SparseSearchResult};
use crate::text::CodeTokenizer;
use crate::types::{ChunkId, TenantId};
const DEFAULT_WRITER_MEMORY_BYTES: usize = 50_000_000;
const SPARSE_CHUNK_OVERFETCH: usize = 8;
const SPARSE_FETCH_CEILING: usize = 1024;
#[derive(Clone, Debug)]
struct ReadOnlyDirectory(MmapDirectory);
impl Directory for ReadOnlyDirectory {
fn get_file_handle(
&self,
path: &Path,
) -> std::result::Result<Arc<dyn FileHandle>, OpenReadError> {
self.0.get_file_handle(path)
}
fn delete(&self, path: &Path) -> std::result::Result<(), DeleteError> {
self.0.delete(path)
}
fn exists(&self, path: &Path) -> std::result::Result<bool, OpenReadError> {
self.0.exists(path)
}
fn open_write(&self, path: &Path) -> std::result::Result<WritePtr, OpenWriteError> {
self.0.open_write(path)
}
fn atomic_read(&self, path: &Path) -> std::result::Result<Vec<u8>, OpenReadError> {
self.0.atomic_read(path)
}
fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> {
self.0.atomic_write(path, data)
}
fn sync_directory(&self) -> io::Result<()> {
self.0.sync_directory()
}
fn acquire_lock(&self, lock: &Lock) -> std::result::Result<DirectoryLock, LockError> {
match self.0.acquire_lock(lock) {
Err(LockError::IoError(_)) => Ok(DirectoryLock::from(Box::new(()))),
result => result,
}
}
fn watch(&self, watch_callback: WatchCallback) -> tantivy::Result<WatchHandle> {
self.0.watch(watch_callback)
}
}
fn escape_query_text(query: &str) -> String {
const SPECIAL_CHARS: &str = "\\+-&|!(){}[]^\"~*?:/";
let mut escaped = String::with_capacity(query.len() + 8);
for ch in query.chars() {
if SPECIAL_CHARS.contains(ch) {
escaped.push('\\');
}
escaped.push(ch);
}
escaped
}
fn plain_query_text(query: &str) -> String {
query
.chars()
.map(|ch| {
if ch.is_alphanumeric() || ch == '_' {
ch
} else {
' '
}
})
.collect()
}
fn parse_text_query(parser: &QueryParser, query: &str) -> Result<Box<dyn Query>> {
match parser.parse_query(query) {
Ok(parsed) => Ok(parsed),
Err(primary_err) => {
let escaped_query = escape_query_text(query);
if let Ok(parsed) = parser.parse_query(&escaped_query) {
return Ok(parsed);
}
let (lenient_query, parse_errors) = parser.parse_query_lenient(&escaped_query);
if parse_errors.is_empty() {
return Ok(lenient_query);
}
let plain_query = plain_query_text(query);
if !plain_query.trim().is_empty() {
if let Ok(parsed) = parser.parse_query(&plain_query) {
return Ok(parsed);
}
let (plain_lenient_query, plain_parse_errors) =
parser.parse_query_lenient(&plain_query);
if plain_parse_errors.is_empty() {
return Ok(plain_lenient_query);
}
}
Err(MemdError::ValidationError(format!(
"parse query: {}; lenient parser also reported syntax issues",
primary_err
)))
}
}
}
pub struct Bm25Index {
index: Index,
reader: IndexReader,
writer: Option<Mutex<IndexWriter>>,
#[allow(dead_code)]
schema: Schema,
tenant_field: Field,
chunk_field: Field,
sentence_field: Field,
text_field: Field,
dirty: AtomicBool,
}
struct Bm25SchemaFields {
schema: Schema,
tenant_field: Field,
chunk_field: Field,
sentence_field: Field,
text_field: Field,
}
fn build_schema_fields() -> Bm25SchemaFields {
let mut schema_builder = Schema::builder();
let tenant_field = schema_builder.add_text_field("tenant_id", STRING | STORED);
let chunk_field = schema_builder.add_text_field("chunk_id", STRING | STORED);
let sentence_field = schema_builder.add_u64_field("sentence_idx", STORED);
let text_indexing = TextFieldIndexing::default()
.set_tokenizer("code")
.set_index_option(IndexRecordOption::WithFreqsAndPositions);
let text_options = TextOptions::default()
.set_indexing_options(text_indexing)
.set_stored();
let text_field = schema_builder.add_text_field("text", text_options);
Bm25SchemaFields {
schema: schema_builder.build(),
tenant_field,
chunk_field,
sentence_field,
text_field,
}
}
impl Bm25Index {
pub fn new() -> Result<Self> {
Self::with_path(None)
}
pub fn with_path(path: Option<PathBuf>) -> Result<Self> {
let fields = build_schema_fields();
let index = match path {
Some(p) => {
std::fs::create_dir_all(&p)?;
let directory = MmapDirectory::open(&p)
.map_err(|e| MemdError::StorageError(format!("open index directory: {}", e)))?;
Index::open_or_create(directory, fields.schema.clone())
.map_err(|e| MemdError::StorageError(format!("open_or_create index: {}", e)))?
}
None => Index::create_in_ram(fields.schema.clone()),
};
let writer = index
.writer(DEFAULT_WRITER_MEMORY_BYTES)
.map_err(|e| MemdError::StorageError(format!("create writer: {}", e)))?;
Self::from_index(index, fields, Some(writer))
}
pub fn with_path_read_only(path: PathBuf) -> Result<Option<Self>> {
if !path.exists() {
tracing::debug!(
path = %path.display(),
"sparse index directory missing; read-only sparse search unavailable"
);
return Ok(None);
}
let fields = build_schema_fields();
let directory = MmapDirectory::open(&path)
.map_err(|e| MemdError::StorageError(format!("open index directory: {}", e)))?;
let index = Index::open(ReadOnlyDirectory(directory))
.map_err(|e| MemdError::StorageError(format!("open read-only index: {}", e)))?;
Ok(Some(Self::from_index(index, fields, None)?))
}
fn from_index(
index: Index,
fields: Bm25SchemaFields,
writer: Option<IndexWriter>,
) -> Result<Self> {
let code_tokenizer = TextAnalyzer::builder(CodeTokenizer::new()).build();
index.tokenizers().register("code", code_tokenizer);
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::OnCommitWithDelay)
.try_into()
.map_err(|e| MemdError::StorageError(format!("create reader: {}", e)))?;
Ok(Self {
index,
reader,
writer: writer.map(Mutex::new),
schema: fields.schema,
tenant_field: fields.tenant_field,
chunk_field: fields.chunk_field,
sentence_field: fields.sentence_field,
text_field: fields.text_field,
dirty: AtomicBool::new(false),
})
}
fn commit_if_dirty(&self) -> Result<()> {
let Some(writer) = self.writer.as_ref() else {
return Ok(());
};
let mut writer = writer
.lock()
.map_err(|e| MemdError::StorageError(format!("lock writer: {}", e)))?;
if self.dirty.swap(false, Ordering::AcqRel) {
writer
.commit()
.map_err(|e| MemdError::StorageError(format!("commit: {}", e)))?;
}
Ok(())
}
pub fn commit(&self) -> Result<()> {
self.commit_if_dirty()
}
pub fn total_docs(&self) -> Result<u64> {
let searcher = self.reader.searcher();
Ok(searcher.num_docs())
}
pub fn segment_count(&self) -> Result<usize> {
self.commit_if_dirty()?;
self.reader
.reload()
.map_err(|e| MemdError::StorageError(format!("reload reader: {}", e)))?;
let searcher = self.reader.searcher();
Ok(searcher.segment_readers().len())
}
pub fn merge_all_segments(&self) -> Result<()> {
self.commit_if_dirty()?;
let segment_ids = self
.index
.searchable_segment_ids()
.map_err(|e| MemdError::StorageError(format!("list segments: {e}")))?;
if segment_ids.len() <= 1 {
return Ok(());
}
let merge = {
let mut writer = self
.writer
.as_ref()
.ok_or_else(|| MemdError::ReadOnlyStore {
op: "bm25_merge_all_segments".to_string(),
})?
.lock()
.map_err(|e| MemdError::StorageError(format!("lock writer: {e}")))?;
writer.merge(&segment_ids)
};
merge
.wait()
.map_err(|e| MemdError::StorageError(format!("merge segments: {e}")))?;
self.reader
.reload()
.map_err(|e| MemdError::StorageError(format!("reload merged index: {e}")))?;
Ok(())
}
}
impl Default for Bm25Index {
fn default() -> Self {
Self::new().expect("failed to create default Bm25Index")
}
}
impl SparseIndex for Bm25Index {
fn insert(&self, tenant_id: &TenantId, chunk_id: &ChunkId, sentences: &[String]) -> Result<()> {
self.insert_batch(&[(tenant_id.clone(), chunk_id.clone(), sentences.to_vec())])
}
fn insert_batch(&self, items: &[(TenantId, ChunkId, Vec<String>)]) -> Result<()> {
if items.is_empty() {
return Ok(());
}
let Some(writer) = self.writer.as_ref() else {
return Err(MemdError::ReadOnlyStore {
op: "bm25_insert_batch".to_string(),
});
};
let writer = writer
.lock()
.map_err(|e| MemdError::StorageError(format!("lock writer: {}", e)))?;
for (tenant_id, chunk_id, sentences) in items {
for (idx, sentence) in sentences.iter().enumerate() {
let doc = doc!(
self.tenant_field => tenant_id.as_str(),
self.chunk_field => chunk_id.to_string(),
self.sentence_field => idx as u64,
self.text_field => sentence.clone(),
);
writer
.add_document(doc)
.map_err(|e| MemdError::StorageError(format!("add document: {}", e)))?;
}
}
self.dirty.store(true, Ordering::Release);
Ok(())
}
fn search(
&self,
tenant_id: &TenantId,
query: &str,
k: usize,
) -> Result<Vec<SparseSearchResult>> {
if k == 0 {
return Ok(Vec::new());
}
self.commit_if_dirty()?;
self.reader
.reload()
.map_err(|e| MemdError::StorageError(format!("reload reader: {}", e)))?;
let searcher = self.reader.searcher();
let tenant_term = Term::from_field_text(self.tenant_field, tenant_id.as_str());
let tenant_query: Box<dyn Query> =
Box::new(TermQuery::new(tenant_term, IndexRecordOption::Basic));
let query_parser = QueryParser::for_index(&self.index, vec![self.text_field]);
let text_query = parse_text_query(&query_parser, query)?;
let combined_query =
BooleanQuery::new(vec![(Occur::Must, tenant_query), (Occur::Must, text_query)]);
let fetch_limit = k
.saturating_mul(SPARSE_CHUNK_OVERFETCH)
.min(SPARSE_FETCH_CEILING)
.max(k);
let top_docs = searcher
.search(&combined_query, &TopDocs::with_limit(fetch_limit))
.map_err(|e| MemdError::StorageError(format!("search: {}", e)))?;
let mut results = Vec::with_capacity(k);
let mut seen: HashSet<ChunkId> = HashSet::with_capacity(k);
for (score, doc_address) in top_docs {
let doc = searcher
.doc::<tantivy::TantivyDocument>(doc_address)
.map_err(|e| MemdError::StorageError(format!("retrieve doc: {}", e)))?;
let chunk_id_str = doc
.get_first(self.chunk_field)
.and_then(|v| v.as_str())
.ok_or_else(|| MemdError::StorageError("missing chunk_id".into()))?;
let sentence_idx = doc
.get_first(self.sentence_field)
.and_then(|v| v.as_u64())
.unwrap_or(0) as usize;
let chunk_id = ChunkId::parse(chunk_id_str)?;
if !seen.insert(chunk_id.clone()) {
continue;
}
results.push(SparseSearchResult {
chunk_id,
score,
sentence_idx,
});
if results.len() >= k {
break;
}
}
Ok(results)
}
fn delete(&self, tenant_id: &TenantId, chunk_id: &ChunkId) -> Result<bool> {
self.commit_if_dirty()?;
let Some(writer) = self.writer.as_ref() else {
return Err(MemdError::ReadOnlyStore {
op: "bm25_delete".to_string(),
});
};
let writer = writer
.lock()
.map_err(|e| MemdError::StorageError(format!("lock writer: {}", e)))?;
let tenant_term = Term::from_field_text(self.tenant_field, tenant_id.as_str());
let chunk_term = Term::from_field_text(self.chunk_field, &chunk_id.to_string());
let tenant_query: Box<dyn Query> =
Box::new(TermQuery::new(tenant_term, IndexRecordOption::Basic));
let chunk_query: Box<dyn Query> =
Box::new(TermQuery::new(chunk_term, IndexRecordOption::Basic));
let delete_query = BooleanQuery::new(vec![
(Occur::Must, tenant_query),
(Occur::Must, chunk_query),
]);
self.reader
.reload()
.map_err(|e| MemdError::StorageError(format!("reload reader: {}", e)))?;
let searcher = self.reader.searcher();
let count_before = searcher
.search(&delete_query, &tantivy::collector::Count)
.unwrap_or(0);
writer
.delete_query(Box::new(delete_query))
.map_err(|e| MemdError::StorageError(format!("delete query: {}", e)))?;
self.dirty.store(true, Ordering::Release);
Ok(count_before > 0)
}
fn doc_count(&self, tenant_id: &TenantId) -> Result<u64> {
self.commit_if_dirty()?;
self.reader
.reload()
.map_err(|e| MemdError::StorageError(format!("reload reader: {}", e)))?;
let searcher = self.reader.searcher();
let tenant_term = Term::from_field_text(self.tenant_field, tenant_id.as_str());
let tenant_query = TermQuery::new(tenant_term, IndexRecordOption::Basic);
let count = searcher
.search(&tenant_query, &tantivy::collector::Count)
.map_err(|e| MemdError::StorageError(format!("count: {}", e)))?;
Ok(count as u64)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn create_test_tenant() -> TenantId {
TenantId::new("test_tenant").unwrap()
}
#[cfg(unix)]
#[test]
fn read_only_open_searches_an_index_in_an_unwritable_directory() {
use std::os::unix::fs::PermissionsExt;
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("sparse_index");
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
{
let index = Bm25Index::with_path(Some(path.clone())).unwrap();
index
.insert(&tenant, &chunk_id, &["readonly mount token".to_string()])
.unwrap();
index.commit().unwrap();
}
let _ = std::fs::remove_file(path.join(".tantivy-meta.lock"));
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o500)).unwrap();
let writable = crate::store::path_is_writable(&path);
let opened = Bm25Index::with_path_read_only(path.clone());
let found = opened
.as_ref()
.ok()
.and_then(|index| index.as_ref())
.map(|index| index.search(&tenant, "mount", 5).unwrap());
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap();
if writable {
return; }
let found = found.expect("read-only open must work without a lock file");
assert_eq!(found[0].chunk_id, chunk_id);
assert!(
!path.join(".tantivy-meta.lock").exists(),
"the read-only open must not create the lock file"
);
}
#[test]
fn test_insert_and_search() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
let sentences = vec![
"function parseJSON() { return JSON.parse(data); }".to_string(),
"Returns parsed data from JSON string".to_string(),
];
index.insert(&tenant, &chunk_id, &sentences).unwrap();
let results = index.search(&tenant, "parseJSON", 10).unwrap();
assert!(!results.is_empty(), "should find parseJSON");
assert_eq!(results[0].chunk_id, chunk_id);
assert!(results[0].score > 0.0);
let results = index.search(&tenant, "parsed data", 10).unwrap();
assert!(!results.is_empty(), "should find 'parsed data'");
}
#[test]
fn test_insert_batch_and_search() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_a = ChunkId::new();
let chunk_b = ChunkId::new();
let items = vec![
(
tenant.clone(),
chunk_a.clone(),
vec!["alpha parser".to_string(), "json utility".to_string()],
),
(
tenant.clone(),
chunk_b.clone(),
vec!["beta formatter".to_string()],
),
];
index.insert_batch(&items).unwrap();
let results = index.search(&tenant, "alpha", 10).unwrap();
assert!(!results.is_empty(), "should find batch-inserted chunk");
assert_eq!(results[0].chunk_id, chunk_a);
let results = index.search(&tenant, "formatter", 10).unwrap();
assert!(
!results.is_empty(),
"should find second batch-inserted chunk"
);
assert_eq!(results[0].chunk_id, chunk_b);
}
#[test]
fn test_keyword_exact_match() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
let sentences = vec!["The getUserById function returns a User object".to_string()];
index.insert(&tenant, &chunk_id, &sentences).unwrap();
let results = index.search(&tenant, "getUserById", 10).unwrap();
assert!(!results.is_empty(), "should find getUserById");
let results = index.search(&tenant, "user", 10).unwrap();
assert!(!results.is_empty(), "should find 'user' token");
}
#[test]
fn test_tenant_isolation() {
let index = Bm25Index::new().unwrap();
let tenant_a = TenantId::new("tenant_a").unwrap();
let tenant_b = TenantId::new("tenant_b").unwrap();
let chunk_id = ChunkId::new();
let sentences = vec!["secret data for tenant A".to_string()];
index.insert(&tenant_a, &chunk_id, &sentences).unwrap();
let results = index.search(&tenant_b, "secret", 10).unwrap();
assert!(results.is_empty(), "tenant B should not see tenant A data");
let results = index.search(&tenant_a, "secret", 10).unwrap();
assert!(!results.is_empty(), "tenant A should see their data");
}
#[test]
fn test_delete() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
let sentences = vec!["deletable content here".to_string()];
index.insert(&tenant, &chunk_id, &sentences).unwrap();
let results = index.search(&tenant, "deletable", 10).unwrap();
assert!(!results.is_empty(), "should find before delete");
let deleted = index.delete(&tenant, &chunk_id).unwrap();
assert!(deleted, "should return true for successful delete");
let results = index.search(&tenant, "deletable", 10).unwrap();
assert!(results.is_empty(), "should not find after delete");
}
#[test]
fn test_multiple_sentences() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
let sentences = vec![
"First sentence about apples".to_string(),
"Second sentence about oranges".to_string(),
"Third sentence about bananas".to_string(),
];
index.insert(&tenant, &chunk_id, &sentences).unwrap();
let results = index.search(&tenant, "oranges", 10).unwrap();
assert!(!results.is_empty());
assert_eq!(
results[0].sentence_idx, 1,
"should match sentence at index 1"
);
}
#[test]
fn test_multi_sentence_chunk_collapses_to_one() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let multi = ChunkId::new();
let single = ChunkId::new();
index
.insert(
&tenant,
&multi,
&[
"oranges are great".to_string(),
"I really love oranges".to_string(),
"more oranges please".to_string(),
],
)
.unwrap();
index
.insert(&tenant, &single, &["oranges once".to_string()])
.unwrap();
let results = index.search(&tenant, "oranges", 10).unwrap();
assert_eq!(
results.iter().filter(|r| r.chunk_id == multi).count(),
1,
"a multi-sentence chunk must collapse to a single result"
);
assert_eq!(
results.iter().filter(|r| r.chunk_id == single).count(),
1,
"a distinct single-sentence chunk must not be collapsed away"
);
assert_eq!(results.len(), 2, "exactly the two distinct chunks survive");
}
#[test]
fn test_code_identifiers() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
let sentences = vec!["snake_case_function and camelCaseMethod examples".to_string()];
index.insert(&tenant, &chunk_id, &sentences).unwrap();
let results = index.search(&tenant, "snake", 10).unwrap();
assert!(!results.is_empty(), "should find 'snake' from snake_case");
let results = index.search(&tenant, "case", 10).unwrap();
assert!(!results.is_empty(), "should find 'case' from snake_case");
let results = index.search(&tenant, "camel", 10).unwrap();
assert!(!results.is_empty(), "should find 'camel' from camelCase");
let results = index.search(&tenant, "method", 10).unwrap();
assert!(
!results.is_empty(),
"should find 'method' from camelCaseMethod"
);
}
#[test]
fn test_doc_count() {
let index = Bm25Index::new().unwrap();
let tenant_a = TenantId::new("tenant_a").unwrap();
let tenant_b = TenantId::new("tenant_b").unwrap();
index
.insert(
&tenant_a,
&ChunkId::new(),
&["sentence 1".to_string(), "sentence 2".to_string()],
)
.unwrap();
index
.insert(&tenant_b, &ChunkId::new(), &["other".to_string()])
.unwrap();
assert_eq!(index.doc_count(&tenant_a).unwrap(), 2);
assert_eq!(index.doc_count(&tenant_b).unwrap(), 1);
}
#[test]
fn test_empty_query() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let results = index.search(&tenant, "", 10);
assert!(results.is_ok() || results.is_err());
}
#[test]
fn test_special_characters() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
let sentences = vec![
"Error: SQLITE_BUSY at line 42".to_string(),
"fn main() -> Result<(), Error>".to_string(),
];
index.insert(&tenant, &chunk_id, &sentences).unwrap();
let results = index.search(&tenant, "SQLITE", 10).unwrap();
assert!(!results.is_empty(), "should find SQLITE");
let results = index.search(&tenant, "Error", 10).unwrap();
assert!(!results.is_empty(), "should find Error");
}
#[test]
fn test_natural_language_query_with_punctuation() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
let sentences = vec![
"Verify user identity with OAuth access tokens".to_string(),
"Identity verification checks token signatures".to_string(),
];
index.insert(&tenant, &chunk_id, &sentences).unwrap();
let results = index.search(&tenant, "How do I verify a user's identity?", 10);
assert!(
results.is_ok(),
"punctuated natural-language query should not error"
);
}
#[test]
fn test_unbalanced_quote_query_does_not_error() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
index
.insert(
&tenant,
&chunk_id,
&["identity verification flow".to_string()],
)
.unwrap();
let results = index.search(&tenant, "\"identity verification", 10);
assert!(results.is_ok(), "unbalanced quote query should not error");
}
#[test]
fn test_benchmark_choice_query_with_brackets_does_not_error() {
let index = Bm25Index::new().unwrap();
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
index
.insert(
&tenant,
&chunk_id,
&[
"Debbie sat with Stuart and Brent Tarleton on the porch of Tara.".to_string(),
"Debbie gazed out the window, eager to see her friend arrive.".to_string(),
],
)
.unwrap();
let query = "Search Archival Memory, complete the task below:\n\
These are the events that have already occurred:\n\
1. Debbie sat with Stuart and Brent Tarleton on the porch of Tara.\n\n\
Below is a list of possible subsequent events:\n\
['Debbie gazed out the window, eager to see her friend arrive.', \
\"Debbie leaned against the fence, excitedly watching for her neighbor's dog.\"]";
let results = index.search(&tenant, query, 10);
assert!(
results.is_ok(),
"benchmark multiple-choice query should not error"
);
assert!(
!results.unwrap().is_empty(),
"plain-token fallback should still retrieve matches"
);
}
#[test]
fn test_persistent_index_reopens_without_error() {
let tmp = tempfile::TempDir::new().unwrap();
let path = tmp.path().join("bm25");
let tenant = create_test_tenant();
let chunk_id = ChunkId::new();
{
let index = Bm25Index::with_path(Some(path.clone())).unwrap();
index
.insert(
&tenant,
&chunk_id,
&["persistent bm25 reopen sentinel".to_string()],
)
.unwrap();
index.commit().unwrap();
}
let reopened = (0..200)
.find_map(|_| match Bm25Index::with_path(Some(path.clone())) {
Ok(index) => Some(index),
Err(_) => {
std::thread::sleep(std::time::Duration::from_millis(10));
None
}
})
.expect("reopening an existing bm25 directory must succeed");
let results = reopened.search(&tenant, "sentinel", 10).unwrap();
assert!(
!results.is_empty(),
"reopened index must retain previously indexed documents"
);
}
}