use std::collections::HashSet;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, anyhow};
use chrono::{DateTime, Utc};
use tantivy::collector::TopDocs;
use tantivy::directory::MmapDirectory;
use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
use tantivy::schema::{
Field, INDEXED, IndexRecordOption, STORED, STRING, Schema, TEXT, TantivyDocument, Value,
};
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, Term, doc};
use usearch::ScalarKind;
use uuid::Uuid;
use super::vector::{EMBEDDING_DIM, VectorIndex};
const SEGMENT_CAPACITY: u64 = 100_000;
const WRITER_HEAP_BYTES: usize = 15_000_000;
const OVERFETCH: usize = 4;
const TEXT_DIR: &str = "text";
pub fn dir_for_db(db_path: &Path) -> PathBuf {
db_path.with_extension("raw-index")
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RawOrigin {
Transcript { sha256: String },
Memory { id: Uuid },
}
impl RawOrigin {
pub fn source_key(&self) -> String {
match self {
Self::Transcript { sha256 } => format!("t:{sha256}"),
Self::Memory { id } => format!("m:{id}"),
}
}
fn parse(key: &str) -> Option<Self> {
match key.split_once(':')? {
("t", sha256) => Some(Self::Transcript {
sha256: sha256.to_string(),
}),
("m", id) => Uuid::parse_str(id).ok().map(|id| Self::Memory { id }),
_ => None,
}
}
}
#[derive(Debug, Clone)]
pub struct RawPassage {
pub text: String,
pub at: Option<DateTime<Utc>>,
pub project: Option<String>,
pub memory: Option<Uuid>,
}
#[derive(Debug, Clone)]
pub struct RawHit {
pub key: u64,
pub text: String,
pub at: Option<DateTime<Utc>>,
pub project: Option<String>,
pub memory: Option<Uuid>,
pub origin: RawOrigin,
}
struct Fields {
key: Field,
body: Field,
text_hash: Field,
source: Field,
project: Field,
memory: Field,
at: Field,
}
pub struct RawIndex {
dir: PathBuf,
sealed: Vec<VectorIndex>,
active: Option<VectorIndex>,
active_segment: u64,
segment_capacity: u64,
index: Index,
reader: IndexReader,
writer: Option<IndexWriter>,
fields: Fields,
writable: bool,
text_bytes: u64,
pending_texts: HashSet<String>,
pending_sources: HashSet<String>,
}
fn schema() -> (Schema, Fields) {
let mut builder = Schema::builder();
let fields = Fields {
key: builder.add_u64_field("key", INDEXED | STORED),
body: builder.add_text_field("body", TEXT | STORED),
text_hash: builder.add_text_field("text_hash", STRING),
source: builder.add_text_field("source", STRING | STORED),
project: builder.add_text_field("project", STRING | STORED),
memory: builder.add_text_field("memory", STRING | STORED),
at: builder.add_i64_field("at", STORED),
};
(builder.build(), fields)
}
fn segment_path(dir: &Path, segment: u64) -> PathBuf {
dir.join(format!("vectors-{segment:05}.usearch"))
}
fn segment_files(dir: &Path) -> Result<Vec<u64>> {
let mut segments = Vec::new();
for entry in std::fs::read_dir(dir).with_context(|| format!("listing {}", dir.display()))? {
let name = entry?.file_name();
let Some(name) = name.to_str() else { continue };
if let Some(n) = name
.strip_prefix("vectors-")
.and_then(|rest| rest.strip_suffix(".usearch"))
.and_then(|n| n.parse().ok())
{
segments.push(n);
}
}
segments.sort_unstable();
Ok(segments)
}
fn dir_bytes(dir: &Path) -> u64 {
std::fs::read_dir(dir)
.map(|entries| {
entries
.filter_map(|e| e.ok()?.metadata().ok())
.map(|m| m.len())
.sum()
})
.unwrap_or(0)
}
fn text_hash(text: &str) -> String {
use sha2::{Digest, Sha256};
let digest = Sha256::digest(text.trim().as_bytes());
digest.iter().take(16).map(|b| format!("{b:02x}")).collect()
}
impl RawIndex {
pub fn open(dir: &Path) -> Result<Self> {
std::fs::create_dir_all(dir.join(TEXT_DIR))
.with_context(|| format!("creating raw index dir {}", dir.display()))?;
let segments = segment_files(dir)?;
let mut sealed = Vec::new();
let mut active = None;
for (i, n) in segments.iter().enumerate() {
let path = segment_path(dir, *n);
if i + 1 == segments.len() {
let loaded = VectorIndex::open_with_kind(&path, ScalarKind::F16)?;
if (loaded.len() as u64) < SEGMENT_CAPACITY {
active = Some((*n, loaded));
continue;
}
}
sealed.push(VectorIndex::open_view_with_kind(&path, ScalarKind::F16)?);
}
let (segment, active) = match active {
Some(found) => found,
None => {
let next = segments.last().map_or(0, |n| n + 1);
let fresh = VectorIndex::open_with_kind(&segment_path(dir, next), ScalarKind::F16)?;
(next, fresh)
}
};
let mut index = Self::with_text(dir, sealed, Some(active), true)?;
index.active_segment = segment;
Ok(index)
}
pub fn open_readonly(dir: &Path) -> Result<Self> {
let sealed = segment_files(dir)?
.into_iter()
.map(|n| VectorIndex::open_view_with_kind(&segment_path(dir, n), ScalarKind::F16))
.collect::<Result<Vec<_>>>()?;
Self::with_text(dir, sealed, None, false)
}
fn with_text(
dir: &Path,
sealed: Vec<VectorIndex>,
active: Option<VectorIndex>,
writable: bool,
) -> Result<Self> {
let (schema, fields) = schema();
let directory = MmapDirectory::open(dir.join(TEXT_DIR))
.with_context(|| format!("opening raw text index in {}", dir.display()))?;
let index = if writable {
Index::open_or_create(directory, schema)
} else {
Index::open(directory)
}
.context("opening raw text index")?;
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::Manual)
.try_into()
.context("creating raw text reader")?;
Ok(Self {
dir: dir.to_path_buf(),
sealed,
active,
active_segment: 0,
segment_capacity: SEGMENT_CAPACITY,
index,
reader,
writer: None,
fields,
writable,
text_bytes: dir_bytes(&dir.join(TEXT_DIR)),
pending_texts: HashSet::new(),
pending_sources: HashSet::new(),
})
}
pub fn len(&self) -> usize {
self.sealed
.iter()
.chain(self.active.iter())
.map(VectorIndex::len)
.sum()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn resident_bytes(&self) -> u64 {
let vectors: usize = self
.sealed
.iter()
.chain(self.active.iter())
.map(VectorIndex::memory_usage)
.sum();
vectors as u64 + self.text_bytes
}
pub fn has_text(&self, text: &str) -> bool {
let hash = text_hash(text);
if self.pending_texts.contains(&hash) {
return true;
}
let term = Term::from_field_text(self.fields.text_hash, &hash);
self.reader.searcher().doc_freq(&term).is_ok_and(|n| n > 0)
}
pub fn has_source(&self, source: &RawOrigin) -> bool {
let key = source.source_key();
if self.pending_sources.contains(&key) {
return true;
}
let term = Term::from_field_text(self.fields.source, &key);
self.reader.searcher().doc_freq(&term).is_ok_and(|n| n > 0)
}
pub fn add_source(
&mut self,
origin: &RawOrigin,
passages: &[(RawPassage, Vec<f32>)],
) -> Result<usize> {
self.require_writable()?;
let source = origin.source_key();
let searcher = self.reader.searcher();
let mut added = 0;
for (passage, embedding) in passages {
if embedding.len() != EMBEDDING_DIM || passage.text.trim().is_empty() {
continue;
}
let hash = text_hash(&passage.text);
let term = Term::from_field_text(self.fields.text_hash, &hash);
if self.pending_texts.contains(&hash) || searcher.doc_freq(&term)? > 0 {
continue;
}
self.pending_texts.insert(hash.clone());
let key = self.next_key()?;
self.active_mut()?.upsert(key, embedding)?;
let mut document = doc!(
self.fields.key => key,
self.fields.body => passage.text.as_str(),
self.fields.text_hash => hash,
self.fields.source => source.as_str(),
);
if let Some(project) = &passage.project {
document.add_text(self.fields.project, project);
}
if let Some(memory) = passage.memory {
document.add_text(self.fields.memory, memory.to_string());
}
if let Some(at) = passage.at {
document.add_i64(self.fields.at, at.timestamp());
}
self.writer_mut()?.add_document(document)?;
added += 1;
}
self.pending_sources.insert(source.clone());
if added == 0 {
let marker = doc!(self.fields.source => source.as_str());
self.writer_mut()?.add_document(marker)?;
}
Ok(added)
}
pub fn commit(&mut self) -> Result<()> {
self.require_writable()?;
if let Some(active) = &self.active {
active.save()?;
}
if let Some(writer) = self.writer.as_mut() {
writer.commit().context("raw text commit")?;
}
self.reader.reload().context("raw text reader reload")?;
self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
self.pending_texts.clear();
self.pending_sources.clear();
Ok(())
}
pub fn reload(&mut self) -> Result<()> {
if self.writable {
return Ok(());
}
let segments = segment_files(&self.dir)?;
let mut sealed = Vec::with_capacity(segments.len());
for n in segments {
sealed.push(VectorIndex::open_view_with_kind(
&segment_path(&self.dir, n),
ScalarKind::F16,
)?);
}
self.sealed = sealed;
self.reader.reload().context("raw text reader reload")?;
self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
Ok(())
}
pub fn vector_search(
&self,
embedding: &[f32],
k: usize,
project: Option<&str>,
) -> Result<Vec<(u64, f32)>> {
let fetch = if project.is_some() { k * OVERFETCH } else { k };
let mut hits = Vec::new();
for segment in self.sealed.iter().chain(self.active.iter()) {
if segment.len() > 0 {
hits.extend(segment.search(embedding, fetch)?);
}
}
hits.sort_by(|a, b| b.1.total_cmp(&a.1));
let mut out = Vec::with_capacity(k);
for (key, score) in hits {
if out.len() == k {
break;
}
let Some(hit) = self.passage(key)? else {
continue;
};
if project.is_none() || hit.project.as_deref() == project {
out.push((key, score));
}
}
Ok(out)
}
pub fn bm25_search(
&self,
query_text: &str,
k: usize,
project: Option<&str>,
) -> Result<Vec<(u64, f32)>> {
let searcher = self.reader.searcher();
let parser = QueryParser::for_index(&self.index, vec![self.fields.body]);
let (text_query, _errors) = parser.parse_query_lenient(query_text);
let query: Box<dyn Query> = match project {
Some(project) => Box::new(BooleanQuery::new(vec![
(Occur::Must, text_query),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(self.fields.project, project),
IndexRecordOption::Basic,
)),
),
])),
None => text_query,
};
let top = searcher.search(&query, &TopDocs::with_limit(k.max(1)).order_by_score())?;
let mut out = Vec::with_capacity(top.len());
for (score, address) in top {
let document: TantivyDocument = searcher.doc(address)?;
if let Some(key) = document.get_first(self.fields.key).and_then(|v| v.as_u64()) {
out.push((key, score));
}
}
Ok(out)
}
pub fn passage(&self, key: u64) -> Result<Option<RawHit>> {
let searcher = self.reader.searcher();
let query = TermQuery::new(
Term::from_field_u64(self.fields.key, key),
IndexRecordOption::Basic,
);
let Some((_, address)) = searcher
.search(&query, &TopDocs::with_limit(1).order_by_score())?
.into_iter()
.next()
else {
return Ok(None);
};
let document: TantivyDocument = searcher.doc(address)?;
let text = |field| {
document
.get_first(field)
.and_then(|v| v.as_str())
.map(str::to_string)
};
let Some(origin) = text(self.fields.source)
.as_deref()
.and_then(RawOrigin::parse)
else {
return Ok(None);
};
Ok(Some(RawHit {
key,
text: text(self.fields.body).unwrap_or_default(),
at: document
.get_first(self.fields.at)
.and_then(|v| v.as_i64())
.and_then(|s| DateTime::from_timestamp(s, 0)),
project: text(self.fields.project),
memory: text(self.fields.memory).and_then(|m| Uuid::parse_str(&m).ok()),
origin,
}))
}
fn next_key(&mut self) -> Result<u64> {
if self.active_mut()?.len() as u64 >= self.segment_capacity {
self.seal_active()?;
}
let ordinal = self.active_mut()?.len() as u64;
Ok((self.active_segment << 32) | ordinal)
}
fn seal_active(&mut self) -> Result<()> {
let active = self
.active
.take()
.context("raw index has no active segment")?;
active.save()?;
drop(active);
let path = segment_path(&self.dir, self.active_segment);
self.sealed
.push(VectorIndex::open_view_with_kind(&path, ScalarKind::F16)?);
self.active_segment += 1;
self.active = Some(VectorIndex::open_with_kind(
&segment_path(&self.dir, self.active_segment),
ScalarKind::F16,
)?);
Ok(())
}
fn active_mut(&mut self) -> Result<&mut VectorIndex> {
self.active
.as_mut()
.context("raw index has no active segment")
}
fn writer_mut(&mut self) -> Result<&mut IndexWriter> {
if self.writer.is_none() {
self.writer = Some(
self.index
.writer_with_num_threads(1, WRITER_HEAP_BYTES)
.context("creating raw text writer")?,
);
}
Ok(self.writer.as_mut().expect("writer just created"))
}
fn require_writable(&self) -> Result<()> {
if self.writable {
Ok(())
} else {
Err(anyhow!("raw index is open read-only"))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn unit(i: usize) -> Vec<f32> {
let mut v = vec![0.0; EMBEDDING_DIM];
v[i % EMBEDDING_DIM] = 1.0;
v
}
fn passage(text: &str, project: Option<&str>) -> RawPassage {
RawPassage {
text: text.to_string(),
at: DateTime::from_timestamp(1_790_000_000, 0),
project: project.map(str::to_string),
memory: None,
}
}
fn transcript(sha: &str) -> RawOrigin {
RawOrigin::Transcript {
sha256: sha.to_string(),
}
}
#[test]
fn a_passage_keeps_the_raw_memory_it_maps_to_and_its_text_is_known_before_commit() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let memory = Uuid::new_v4();
let mut mapped = passage("the mirror job runs at four", None);
mapped.memory = Some(memory);
assert!(!index.has_text("the mirror job runs at four"));
index
.add_source(&transcript("aa"), &[(mapped, unit(1))])
.unwrap();
assert!(index.has_text("the mirror job runs at four"));
index.commit().unwrap();
assert!(index.has_text(" the mirror job runs at four \n"));
assert!(!index.has_text("the mirror job runs at five"));
let key = index.vector_search(&unit(1), 1, None).unwrap()[0].0;
assert_eq!(index.passage(key).unwrap().unwrap().memory, Some(memory));
assert!(index.resident_bytes() > 0);
}
#[test]
fn a_passage_is_found_by_vector_and_by_words_with_its_metadata() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let source = transcript("aa");
index
.add_source(
&source,
&[
(
passage(
"the garden daemon listens on port 7200",
Some("-srv-garden"),
),
unit(1),
),
(passage("lunch was soup", Some("-srv-garden")), unit(2)),
],
)
.unwrap();
index.commit().unwrap();
let by_vector = index.vector_search(&unit(1), 1, None).unwrap();
let hit = index.passage(by_vector[0].0).unwrap().unwrap();
assert_eq!(hit.text, "the garden daemon listens on port 7200");
assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
assert_eq!(hit.origin, source);
assert_eq!(hit.at.unwrap().timestamp(), 1_790_000_000);
let by_words = index.bm25_search("daemon port", 5, None).unwrap();
assert_eq!(by_words.len(), 1);
assert_eq!(by_words[0].0, by_vector[0].0);
}
#[test]
fn a_project_filter_keeps_only_that_projects_passages() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
index
.add_source(
&transcript("aa"),
&[
(
passage("deploy uses argo here", Some("-srv-garden")),
unit(1),
),
(
passage("deploy uses argo there", Some("-srv-orchard")),
unit(1),
),
],
)
.unwrap();
index.commit().unwrap();
for hits in [
index
.vector_search(&unit(1), 5, Some("-srv-orchard"))
.unwrap(),
index
.bm25_search("deploy argo", 5, Some("-srv-orchard"))
.unwrap(),
] {
assert_eq!(hits.len(), 1);
let hit = index.passage(hits[0].0).unwrap().unwrap();
assert_eq!(hit.project.as_deref(), Some("-srv-orchard"));
}
}
#[test]
fn the_same_text_from_two_sources_is_stored_once_and_both_sources_are_recorded() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let memory = RawOrigin::Memory { id: Uuid::new_v4() };
let text = "the port moved to 7200";
assert_eq!(
index
.add_source(&transcript("aa"), &[(passage(text, None), unit(1))])
.unwrap(),
1
);
index.commit().unwrap();
assert_eq!(
index
.add_source(&memory, &[(passage(text, None), unit(1))])
.unwrap(),
0
);
index.commit().unwrap();
assert_eq!(index.len(), 1);
assert!(index.has_source(&transcript("aa")));
assert!(index.has_source(&memory));
assert!(!index.has_source(&transcript("bb")));
}
#[test]
fn a_passage_without_a_full_embedding_is_skipped() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let added = index
.add_source(
&transcript("aa"),
&[(passage("short vector", None), vec![1.0; 3])],
)
.unwrap();
index.commit().unwrap();
assert_eq!(added, 0);
assert!(index.is_empty());
assert!(index.has_source(&transcript("aa")));
}
#[test]
fn full_segments_are_sealed_and_everything_survives_a_reopen() {
let dir = tempfile::tempdir().unwrap();
{
let mut index = RawIndex::open(dir.path()).unwrap();
index.segment_capacity = 2;
let passages: Vec<_> = (0..5)
.map(|i| (passage(&format!("passage number {i}"), None), unit(i)))
.collect();
index.add_source(&transcript("aa"), &passages).unwrap();
index.commit().unwrap();
assert_eq!(index.sealed.len(), 2);
}
assert_eq!(segment_files(dir.path()).unwrap(), vec![0, 1, 2]);
let mut reopened = RawIndex::open(dir.path()).unwrap();
assert_eq!(reopened.len(), 5);
let hit = reopened.vector_search(&unit(3), 1, None).unwrap()[0].0;
assert_eq!(
reopened.passage(hit).unwrap().unwrap().text,
"passage number 3"
);
reopened
.add_source(
&transcript("bb"),
&[(passage("one more passage", None), unit(9))],
)
.unwrap();
reopened.commit().unwrap();
let readonly = RawIndex::open_readonly(dir.path()).unwrap();
assert_eq!(readonly.len(), 6);
let hit = readonly.bm25_search("one more", 1, None).unwrap()[0].0;
assert_eq!(
readonly.passage(hit).unwrap().unwrap().text,
"one more passage"
);
}
#[test]
fn a_read_only_index_refuses_writes() {
let dir = tempfile::tempdir().unwrap();
RawIndex::open(dir.path()).unwrap().commit().unwrap();
let mut readonly = RawIndex::open_readonly(dir.path()).unwrap();
assert!(readonly.add_source(&transcript("aa"), &[]).is_err());
}
#[test]
fn origins_round_trip_through_their_source_key() {
for origin in [transcript("ab12"), RawOrigin::Memory { id: Uuid::new_v4() }] {
assert_eq!(RawOrigin::parse(&origin.source_key()), Some(origin));
}
assert_eq!(RawOrigin::parse("x:1"), None);
}
}