use std::path::Path;
use anyhow::{Context, Result, anyhow};
use tantivy::collector::TopDocs;
use tantivy::directory::MmapDirectory;
use tantivy::query::QueryParser;
use tantivy::schema::{Field, STORED, STRING, Schema, TEXT, Value};
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term, doc};
use uuid::Uuid;
#[derive(Clone, Copy, PartialEq, Eq)]
enum LexicalMode {
ReadWrite,
Reader,
}
pub struct LexicalIndex {
index: Index,
directory: MmapDirectory,
writer: Option<IndexWriter>,
reader: IndexReader,
id_field: Field,
body_field: Field,
mode: LexicalMode,
}
impl LexicalIndex {
pub fn open(dir: &Path) -> Result<Self> {
std::fs::create_dir_all(dir)
.with_context(|| format!("creating tantivy dir {}", dir.display()))?;
let mut builder = Schema::builder();
let id_field = builder.add_text_field("memory_id", STRING | STORED);
let body_field = builder.add_text_field("body", TEXT);
let schema = builder.build();
let directory = MmapDirectory::open(dir)?;
let index =
Index::open_or_create(directory.clone(), schema).context("opening tantivy index")?;
let reader = index.reader().context("creating tantivy reader")?;
Ok(Self {
index,
directory,
writer: None,
reader,
id_field,
body_field,
mode: LexicalMode::ReadWrite,
})
}
pub fn open_reader(dir: &Path) -> Result<Self> {
let mmap_dir = MmapDirectory::open(dir)
.with_context(|| format!("opening tantivy directory {}", dir.display()))?;
let index = Index::open(mmap_dir.clone()).context("opening tantivy index")?;
let schema = index.schema();
let id_field = schema
.get_field("memory_id")
.context("tantivy schema missing memory_id field")?;
let body_field = schema
.get_field("body")
.context("tantivy schema missing body field")?;
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::Manual)
.try_into()
.context("creating tantivy reader")?;
Ok(Self {
index,
directory: mmap_dir,
writer: None,
reader,
id_field,
body_field,
mode: LexicalMode::Reader,
})
}
fn evict_dead_mmap_entries(&self) {
let _ = self.directory.get_cache_info();
}
fn writer_mut(&mut self) -> Result<&mut IndexWriter> {
if self.mode == LexicalMode::Reader {
return Err(anyhow!(
"tantivy index is a read-only reader; mutation is not permitted"
));
}
if self.writer.is_none() {
self.writer = Some(
self.index
.writer(50_000_000)
.context("creating tantivy writer")?,
);
}
Ok(self.writer.as_mut().unwrap())
}
pub fn upsert(&mut self, id: Uuid, body: &str) -> Result<()> {
let id_field = self.id_field;
let body_field = self.body_field;
let id_str = id.to_string();
let writer = self.writer_mut()?;
writer.delete_term(Term::from_field_text(id_field, &id_str));
writer.add_document(doc!(id_field => id_str, body_field => body))?;
Ok(())
}
pub fn remove(&mut self, id: Uuid) -> Result<()> {
let id_field = self.id_field;
let id_str = id.to_string();
self.writer_mut()?
.delete_term(Term::from_field_text(id_field, &id_str));
Ok(())
}
pub fn commit(&mut self) -> Result<()> {
if self.mode == LexicalMode::Reader {
return Err(anyhow!(
"tantivy index is a read-only reader; commit is not permitted"
));
}
if let Some(writer) = self.writer.as_mut() {
writer.commit().context("tantivy commit")?;
self.reader.reload().context("tantivy reader reload")?;
self.evict_dead_mmap_entries();
}
Ok(())
}
pub fn search(&self, query_text: &str, k: usize) -> Result<Vec<(Uuid, f32)>> {
let searcher = self.reader.searcher();
let parser = QueryParser::for_index(&self.index, vec![self.body_field]);
let (query, _errors) = parser.parse_query_lenient(query_text);
let top = searcher.search(&query, &TopDocs::with_limit(k.max(1)).order_by_score())?;
let mut out = Vec::with_capacity(top.len());
for (score, address) in top {
let document: TantivyDocument = searcher.doc(address)?;
let id_str = document
.get_first(self.id_field)
.and_then(|v| v.as_str())
.unwrap_or_default();
if let Ok(id) = Uuid::parse_str(id_str) {
out.push((id, score));
}
}
Ok(out)
}
pub fn num_docs(&self) -> u64 {
self.reader.searcher().num_docs()
}
pub fn reload_reader(&mut self) -> Result<()> {
self.reader.reload().context("tantivy reader reload")?;
self.evict_dead_mmap_entries();
Ok(())
}
pub fn reset(&mut self) -> Result<()> {
self.writer_mut()?
.delete_all_documents()
.context("tantivy delete_all_documents")?;
self.commit()
}
}