use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use anyhow::{Context, Result, anyhow, bail, ensure};
use chrono::{DateTime, Utc};
use tantivy::collector::{DocSetCollector, TopDocs};
use tantivy::directory::{Directory, MmapDirectory};
use tantivy::merge_policy::{DefaultMergePolicy, NoMergePolicy};
use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
use tantivy::schema::{
Field, INDEXED, IndexRecordOption, STORED, STRING, Schema, TEXT, TantivyDocument, Value,
};
use tantivy::{
DocAddress, DocSet, FutureResult, Index, IndexReader, IndexSettings, IndexWriter, ReloadPolicy,
Searcher, SegmentMeta, TERMINATED, Term, doc,
};
use usearch::{IndexOptions, MetricKind, ScalarKind};
use uuid::Uuid;
use super::vector::{EMBEDDING_DIM, VectorIndex};
use crate::sealed::{self, SealKeys, SealedDirectory};
const SEGMENT_CAPACITY: u64 = 100_000;
const WRITER_HEAP_BYTES: usize = 15_000_000;
const OVERFETCH: usize = 4;
const TEXT_DIR: &str = "text";
const MIGRATED_TEXT_DIR: &str = "text.migrated";
const REPLACED_TEXT_DIR: &str = "text.replaced";
const MIGRATION_JOURNAL: &str = "text-migration.pending";
const REWRITE_JOURNAL: &str = "rewrite.pending";
const META_FILE: &str = "meta.json";
pub fn dir_for_db(db_path: &Path) -> PathBuf {
db_path.with_extension("raw-index")
}
pub fn text_hash(text: &str) -> String {
use sha2::{Digest, Sha256};
let digest = Sha256::digest(text.trim().as_bytes());
digest.iter().take(16).map(|b| format!("{b:02x}")).collect()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RawOrigin {
Transcript { sha256: String },
Memory { id: Uuid },
}
impl RawOrigin {
pub fn source_key(&self) -> String {
match self {
Self::Transcript { sha256 } => format!("t:{sha256}"),
Self::Memory { id } => format!("m:{id}"),
}
}
fn parse(key: &str) -> Option<Self> {
match key.split_once(':')? {
("t", sha256) => Some(Self::Transcript {
sha256: sha256.to_string(),
}),
("m", id) => Uuid::parse_str(id).ok().map(|id| Self::Memory { id }),
_ => None,
}
}
}
#[derive(Debug, Clone)]
pub struct RawPassage {
pub text: String,
pub at: Option<DateTime<Utc>>,
pub project: Option<String>,
pub memory: Option<Uuid>,
}
#[derive(Debug, Clone)]
pub struct RawHit {
pub key: u64,
pub text: String,
pub text_hash: String,
pub at: Option<DateTime<Utc>>,
pub project: Option<String>,
pub memory: Option<Uuid>,
pub origin: RawOrigin,
}
struct Fields {
key: Field,
body: Field,
text: Field,
text_hash: Field,
source: Field,
project: Field,
memory: Field,
at: Field,
}
type AtRestKeys = Arc<Mutex<Option<SealKeys>>>;
fn lock_keys(keys: &AtRestKeys) -> MutexGuard<'_, Option<SealKeys>> {
keys.lock().unwrap_or_else(PoisonError::into_inner)
}
pub struct RawIndex {
dir: PathBuf,
full_segments: Vec<(u64, VectorIndex)>,
active: Option<VectorIndex>,
active_segment: u64,
next_ordinal: u64,
segment_capacity: u64,
index: Index,
reader: IndexReader,
writer: Option<IndexWriter>,
fields: Fields,
writable: bool,
text_bytes: u64,
pending_texts: HashSet<String>,
pending_sources: HashSet<String>,
purge_generation: u64,
at_rest: AtRestKeys,
text_at_rest: Option<SealedDirectory>,
}
pub struct SealedRewrite {
dir: PathBuf,
keys: BTreeSet<u64>,
at_rest: AtRestKeys,
}
pub struct Compaction {
merge: Option<FutureResult<Option<SegmentMeta>>>,
}
fn schema() -> (Schema, Fields) {
let mut builder = Schema::builder();
let fields = Fields {
key: builder.add_u64_field("key", INDEXED | STORED),
body: builder.add_text_field("body", TEXT),
text: builder.add_text_field("text", STORED),
text_hash: builder.add_text_field("text_hash", STRING | STORED),
source: builder.add_text_field("source", STRING | STORED),
project: builder.add_text_field("project", STRING | STORED),
memory: builder.add_text_field("memory", STRING | STORED),
at: builder.add_i64_field("at", STORED),
};
(builder.build(), fields)
}
fn legacy_schema() -> Schema {
let mut builder = Schema::builder();
builder.add_u64_field("key", INDEXED | STORED);
builder.add_text_field("body", TEXT | STORED);
builder.add_text_field("text_hash", STRING);
builder.add_text_field("source", STRING | STORED);
builder.add_text_field("project", STRING | STORED);
builder.add_text_field("memory", STRING | STORED);
builder.add_i64_field("at", STORED);
builder.build()
}
fn stores_text(source: &str) -> bool {
!matches!(RawOrigin::parse(source), Some(RawOrigin::Memory { .. }))
}
fn passage_document(
fields: &Fields,
key: u64,
source: &str,
passage: &RawPassage,
) -> TantivyDocument {
let mut document = doc!(
fields.key => key,
fields.body => passage.text.as_str(),
fields.text_hash => text_hash(&passage.text),
fields.source => source,
);
if stores_text(source) {
document.add_text(fields.text, &passage.text);
}
if let Some(project) = &passage.project {
document.add_text(fields.project, project);
}
if let Some(memory) = passage.memory {
document.add_text(fields.memory, memory.to_string());
}
if let Some(at) = passage.at {
document.add_i64(fields.at, at.timestamp());
}
document
}
fn segment_name(segment: u64) -> String {
format!("vectors-{segment:05}.usearch")
}
fn segment_path(dir: &Path, segment: u64) -> PathBuf {
dir.join(segment_name(segment))
}
fn segment_files(dir: &Path) -> Result<Vec<u64>> {
let mut segments = Vec::new();
for entry in std::fs::read_dir(dir).with_context(|| format!("listing {}", dir.display()))? {
let name = entry?.file_name();
let Some(name) = name.to_str() else { continue };
if let Some(n) = name
.strip_prefix("vectors-")
.and_then(|rest| rest.strip_suffix(".usearch"))
.and_then(|n| n.parse().ok())
{
segments.push(n);
}
}
segments.sort_unstable();
Ok(segments)
}
fn open_writable_segment(path: &Path, at_rest: Option<&SealKeys>) -> Result<VectorIndex> {
match at_rest {
Some(keys) => VectorIndex::open_with_kind_keyed(path, ScalarKind::F16, keys),
None => VectorIndex::open_with_kind(path, ScalarKind::F16),
}
}
fn open_full_segment(path: &Path, at_rest: Option<&SealKeys>) -> Result<VectorIndex> {
match at_rest {
Some(keys) => VectorIndex::open_loaded_with_kind(path, ScalarKind::F16, keys),
None => VectorIndex::open_view_with_kind(path, ScalarKind::F16),
}
}
fn is_scratch_file(name: &str) -> bool {
sealed::is_tmp_name(name) || name.ends_with(".tmp") || name.ends_with(".rebuilt.usearch")
}
fn top_level_files(dir: &Path) -> Result<Vec<String>> {
let mut names = Vec::new();
for entry in std::fs::read_dir(dir).with_context(|| format!("listing {}", dir.display()))? {
let entry = entry?;
if entry.file_type()?.is_file()
&& let Some(name) = entry.file_name().to_str()
{
names.push(name.to_string());
}
}
Ok(names)
}
fn remove_scratch_files(dir: &Path) -> Result<()> {
for name in top_level_files(dir)? {
if is_scratch_file(&name) {
let path = dir.join(&name);
std::fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?;
}
}
Ok(())
}
fn dir_bytes(dir: &Path) -> u64 {
std::fs::read_dir(dir)
.map(|entries| {
entries
.filter_map(|e| e.ok()?.metadata().ok())
.map(|m| m.len())
.sum()
})
.unwrap_or(0)
}
fn sync_dir(dir: &Path) -> Result<()> {
std::fs::File::open(dir)
.and_then(|d| d.sync_all())
.with_context(|| format!("syncing {}", dir.display()))
}
fn holds_live(searcher: &Searcher, term: Term) -> Result<bool> {
for segment in searcher.segment_readers() {
let Some(mut postings) = segment
.inverted_index(term.field())?
.read_postings(&term, IndexRecordOption::Basic)?
else {
continue;
};
let mut doc = postings.doc();
while doc != TERMINATED {
if !segment.is_deleted(doc) {
return Ok(true);
}
doc = postings.advance();
}
}
Ok(false)
}
fn read_journal(dir: &Path, at_rest: Option<&SealKeys>) -> Result<BTreeSet<u64>> {
let path = dir.join(REWRITE_JOURNAL);
if !path.exists() {
return Ok(BTreeSet::new());
}
let bytes = sealed::read_file(&path, at_rest, REWRITE_JOURNAL)?;
let text = String::from_utf8(bytes).with_context(|| format!("reading {}", path.display()))?;
text.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
.map(|line| {
line.parse::<u64>()
.with_context(|| format!("reading {}", path.display()))
})
.collect()
}
fn write_journal(dir: &Path, keys: &BTreeSet<u64>, at_rest: Option<&SealKeys>) -> Result<()> {
let body: String = keys.iter().map(|k| format!("{k}\n")).collect();
sealed::write_file(
&dir.join(REWRITE_JOURNAL),
at_rest,
REWRITE_JOURNAL,
body.as_bytes(),
)
}
fn segment_options() -> IndexOptions {
IndexOptions {
dimensions: EMBEDDING_DIM,
metric: MetricKind::Cos,
quantization: ScalarKind::F16,
..IndexOptions::default()
}
}
fn segment_vectors(path: &Path, name: &str, at_rest: Option<&SealKeys>) -> Result<usearch::Index> {
let index = usearch::Index::new(&segment_options())
.map_err(|e| anyhow!("creating usearch index: {e}"))?;
let kind = match at_rest {
Some(keys) => {
let bytes = sealed::read_file(path, Some(keys), name)?;
let kind = usearch::Index::metadata_from_buffer(&bytes)
.map_err(|e| anyhow!("reading usearch header at {}: {e}", path.display()))?
.quantization;
if kind == ScalarKind::F16 {
index
.load_from_buffer(&bytes)
.map_err(|e| anyhow!("loading usearch index from {}: {e}", path.display()))?;
}
Some(kind)
}
None => {
let kind = VectorIndex::on_disk_kind(path)?;
if kind == Some(ScalarKind::F16) {
let path_str = path.to_str().context("vector index path is not utf-8")?;
index
.view(path_str)
.map_err(|e| anyhow!("viewing usearch index at {path_str}: {e}"))?;
}
kind
}
};
if kind != Some(ScalarKind::F16) {
bail!("raw vector segment {} is not F16", path.display());
}
Ok(index)
}
fn rewrite_segment(
dir: &Path,
segment: u64,
keys: &HashSet<u64>,
at_rest: Option<&SealKeys>,
) -> Result<()> {
let path = segment_path(dir, segment);
if !path.exists() {
return Ok(());
}
let name = segment_name(segment);
let source = segment_vectors(&path, &name, at_rest)?;
let live = source.size();
let rebuilt = usearch::Index::new(&segment_options())
.map_err(|e| anyhow!("creating usearch index: {e}"))?;
rebuilt
.reserve(live.max(1))
.map_err(|e| anyhow!("reserving usearch capacity: {e}"))?;
let mut seen = 0;
let mut vector = vec![0.0f32; EMBEDDING_DIM];
for ordinal in 0..SEGMENT_CAPACITY {
if seen == live {
break;
}
let key = (segment << 32) | ordinal;
if !source.contains(key) {
continue;
}
seen += 1;
if keys.contains(&key) {
continue;
}
source
.get(key, vector.as_mut_slice())
.map_err(|e| anyhow!("reading key {key} from {}: {e}", path.display()))?;
rebuilt
.add(key, vector.as_slice())
.map_err(|e| anyhow!("adding key {key}: {e}"))?;
}
if seen < live {
bail!(
"raw vector segment {} holds keys outside its range",
path.display()
);
}
let mut bytes = vec![0u8; rebuilt.serialized_length()];
rebuilt
.save_to_buffer(&mut bytes)
.map_err(|e| anyhow!("serializing usearch index: {e}"))?;
drop(rebuilt);
drop(source);
sealed::write_file(&path, at_rest, &name, &bytes)
}
fn open_text_dir(
text_dir: &Path,
at_rest: Option<&SealKeys>,
create: bool,
) -> Result<(Index, Option<SealedDirectory>)> {
let meta = text_dir.join(META_FILE);
if meta.exists() {
sealed::read_file(&meta, at_rest, META_FILE).context("opening raw text index")?;
}
let (directory, text_at_rest): (Box<dyn Directory>, _) = match at_rest {
Some(keys) => {
let sealed = SealedDirectory::open(text_dir, keys.clone())?;
if create {
sealed.remove_leftover_tmp_files()?;
}
(Box::new(sealed.clone()), Some(sealed))
}
None => (
Box::new(
MmapDirectory::open(text_dir)
.with_context(|| format!("opening raw text index in {}", text_dir.display()))?,
),
None,
),
};
let index = if create && !Index::exists(directory.as_ref())? {
Index::create(directory, schema().0, IndexSettings::default())
} else {
Index::open(directory)
}
.context("opening raw text index")?;
Ok((index, text_at_rest))
}
fn open_text(
dir: &Path,
at_rest: Option<&SealKeys>,
writable: bool,
) -> Result<(Index, Option<SealedDirectory>)> {
let text_dir = dir.join(TEXT_DIR);
let opened = open_text_dir(&text_dir, at_rest, writable)?;
let found = opened.0.schema();
if found == schema().0 {
return Ok(opened);
}
if found != legacy_schema() {
bail!("raw text index in {} has an unknown schema", dir.display());
}
if !writable {
bail!(
"raw text index in {} predates the current schema; open it for writing to migrate it",
dir.display()
);
}
drop(opened);
migrate_text(dir, at_rest)?;
let opened = open_text_dir(&text_dir, at_rest, true)?;
ensure!(
opened.0.schema() == schema().0,
"raw text index in {} did not migrate",
dir.display()
);
Ok(opened)
}
fn migrate_text(dir: &Path, at_rest: Option<&SealKeys>) -> Result<()> {
build_migrated_text(dir, at_rest)?;
sealed::write_file(
&dir.join(MIGRATION_JOURNAL),
at_rest,
MIGRATION_JOURNAL,
b"",
)?;
finish_text_swap(dir)
}
fn build_migrated_text(dir: &Path, at_rest: Option<&SealKeys>) -> Result<()> {
let migrated_dir = dir.join(MIGRATED_TEXT_DIR);
if migrated_dir.exists() {
std::fs::remove_dir_all(&migrated_dir)
.with_context(|| format!("removing {}", migrated_dir.display()))?;
}
std::fs::create_dir_all(&migrated_dir)
.with_context(|| format!("creating {}", migrated_dir.display()))?;
let (legacy, _) = open_text_dir(&dir.join(TEXT_DIR), at_rest, false)?;
let (migrated, _) = open_text_dir(&migrated_dir, at_rest, true)?;
let legacy_schema = legacy.schema();
let legacy_field = |name: &str| {
legacy_schema
.get_field(name)
.with_context(|| format!("the legacy raw text index has no {name} field"))
};
let (key, body, source, project, memory, at) = (
legacy_field("key")?,
legacy_field("body")?,
legacy_field("source")?,
legacy_field("project")?,
legacy_field("memory")?,
legacy_field("at")?,
);
let (_, fields) = schema();
let reader: IndexReader = legacy
.reader_builder()
.reload_policy(ReloadPolicy::Manual)
.try_into()
.context("creating legacy raw text reader")?;
let searcher = reader.searcher();
let mut writer: IndexWriter = migrated
.writer_with_num_threads(1, WRITER_HEAP_BYTES)
.context("creating migrated raw text writer")?;
let mut copied = 0u64;
for (ord, segment) in searcher.segment_readers().iter().enumerate() {
for doc_id in segment.doc_ids_alive() {
let old: TantivyDocument = searcher.doc(DocAddress::new(ord as u32, doc_id))?;
let text = |field| {
old.get_first(field)
.and_then(|v| v.as_str())
.map(str::to_string)
};
let source_key = text(source).context("a raw text document has no source")?;
let document = match old.get_first(key).and_then(|v| v.as_u64()) {
Some(key) => passage_document(
&fields,
key,
&source_key,
&RawPassage {
text: text(body).unwrap_or_default(),
at: old
.get_first(at)
.and_then(|v| v.as_i64())
.and_then(|s| DateTime::from_timestamp(s, 0)),
project: text(project),
memory: text(memory).and_then(|m| Uuid::parse_str(&m).ok()),
},
),
None => doc!(fields.source => source_key.as_str()),
};
writer.add_document(document)?;
copied += 1;
}
}
writer.commit().context("migrated raw text commit")?;
writer
.wait_merging_threads()
.context("waiting for migrated raw text merges")?;
let reader: IndexReader = migrated
.reader_builder()
.reload_policy(ReloadPolicy::Manual)
.try_into()
.context("creating migrated raw text reader")?;
let landed = reader.searcher().num_docs();
ensure!(
landed == copied && copied == searcher.num_docs(),
"raw text migration copied {copied} of {} documents and {landed} landed",
searcher.num_docs()
);
Ok(())
}
fn finish_text_swap(dir: &Path) -> Result<()> {
let text = dir.join(TEXT_DIR);
let migrated = dir.join(MIGRATED_TEXT_DIR);
let replaced = dir.join(REPLACED_TEXT_DIR);
let journal = dir.join(MIGRATION_JOURNAL);
let rename = |from: &Path, to: &Path| {
std::fs::rename(from, to)
.with_context(|| format!("renaming {} to {}", from.display(), to.display()))
};
if journal.exists() {
if migrated.exists() {
if text.exists() {
if replaced.exists() {
std::fs::remove_dir_all(&replaced)
.with_context(|| format!("removing {}", replaced.display()))?;
}
rename(&text, &replaced)?;
}
rename(&migrated, &text)?;
sync_dir(dir)?;
}
} else if migrated.exists() {
std::fs::remove_dir_all(&migrated)
.with_context(|| format!("removing {}", migrated.display()))?;
}
if replaced.exists() {
std::fs::remove_dir_all(&replaced)
.with_context(|| format!("removing {}", replaced.display()))?;
}
if journal.exists() {
std::fs::remove_file(&journal)
.with_context(|| format!("removing {}", journal.display()))?;
sync_dir(dir)?;
}
Ok(())
}
impl SealedRewrite {
pub fn is_empty(&self) -> bool {
self.keys.is_empty()
}
pub fn run(&self) -> Result<()> {
if self.keys.is_empty() {
return Ok(());
}
let at_rest = lock_keys(&self.at_rest);
let mut by_segment: BTreeMap<u64, HashSet<u64>> = BTreeMap::new();
for key in &self.keys {
by_segment.entry(key >> 32).or_default().insert(*key);
}
for (segment, keys) in &by_segment {
rewrite_segment(&self.dir, *segment, keys, at_rest.as_ref())?;
}
let journal = self.dir.join(REWRITE_JOURNAL);
if journal.exists() {
std::fs::remove_file(&journal)
.with_context(|| format!("removing {}", journal.display()))?;
}
Ok(())
}
}
impl Compaction {
pub fn wait(self) -> Result<()> {
match self.merge {
Some(merge) => merge.wait().map(|_| ()).context("raw text merge"),
None => Ok(()),
}
}
}
impl RawIndex {
pub fn open(dir: &Path) -> Result<Self> {
Self::open_writable(dir, None)
}
pub fn open_keyed(dir: &Path, keys: &SealKeys) -> Result<Self> {
Self::open_writable(dir, Some(keys))
}
fn open_writable(dir: &Path, at_rest: Option<&SealKeys>) -> Result<Self> {
finish_text_swap(dir)?;
std::fs::create_dir_all(dir.join(TEXT_DIR))
.with_context(|| format!("creating raw index dir {}", dir.display()))?;
remove_scratch_files(dir)?;
let keys: AtRestKeys = Arc::new(Mutex::new(at_rest.cloned()));
SealedRewrite {
dir: dir.to_path_buf(),
keys: read_journal(dir, at_rest)?,
at_rest: keys.clone(),
}
.run()?;
let segments = segment_files(dir)?;
let mut full_segments = Vec::new();
let mut active = None;
for (i, n) in segments.iter().enumerate() {
let path = segment_path(dir, *n);
if i + 1 == segments.len() {
let loaded = open_writable_segment(&path, at_rest)?;
if (loaded.len() as u64) < SEGMENT_CAPACITY {
active = Some((*n, loaded));
continue;
}
}
full_segments.push((*n, open_full_segment(&path, at_rest)?));
}
let (segment, active) = match active {
Some(found) => found,
None => {
let next = segments.last().map_or(0, |n| n + 1);
(
next,
open_writable_segment(&segment_path(dir, next), at_rest)?,
)
}
};
let mut index = Self::with_text(dir, full_segments, Some(active), keys, true)?;
index.active_segment = segment;
index.next_ordinal = index.ordinal_after(segment)?;
Ok(index)
}
pub fn open_readonly(dir: &Path) -> Result<Self> {
Self::open_search_only(dir, None)
}
pub fn open_readonly_keyed(dir: &Path, keys: &SealKeys) -> Result<Self> {
Self::open_search_only(dir, Some(keys))
}
fn open_search_only(dir: &Path, at_rest: Option<&SealKeys>) -> Result<Self> {
let full_segments = segment_files(dir)?
.into_iter()
.map(|n| open_full_segment(&segment_path(dir, n), at_rest).map(|s| (n, s)))
.collect::<Result<Vec<_>>>()?;
let keys = Arc::new(Mutex::new(at_rest.cloned()));
Self::with_text(dir, full_segments, None, keys, false)
}
fn with_text(
dir: &Path,
full_segments: Vec<(u64, VectorIndex)>,
active: Option<VectorIndex>,
at_rest: AtRestKeys,
writable: bool,
) -> Result<Self> {
let keys = lock_keys(&at_rest).clone();
let (index, text_at_rest) = open_text(dir, keys.as_ref(), writable)?;
let (_, fields) = schema();
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::Manual)
.try_into()
.context("creating raw text reader")?;
Ok(Self {
dir: dir.to_path_buf(),
full_segments,
active,
active_segment: 0,
next_ordinal: 0,
segment_capacity: SEGMENT_CAPACITY,
index,
reader,
writer: None,
fields,
writable,
text_bytes: dir_bytes(&dir.join(TEXT_DIR)),
pending_texts: HashSet::new(),
pending_sources: HashSet::new(),
purge_generation: 0,
at_rest,
text_at_rest,
})
}
pub fn seal_keys(&self) -> Option<SealKeys> {
lock_keys(&self.at_rest).clone()
}
pub fn set_seal_keys(&mut self, keys: SealKeys) -> Result<()> {
let Some(text) = &self.text_at_rest else {
return self.reopen_keyed(&keys);
};
text.set_keys(keys.clone());
if let Some(active) = self.active.as_mut() {
active.set_seal_keys(keys.clone());
}
for (_, segment) in &mut self.full_segments {
segment.set_seal_keys(keys.clone());
}
*lock_keys(&self.at_rest) = Some(keys);
Ok(())
}
fn reopen_keyed(&mut self, keys: &SealKeys) -> Result<()> {
if !self.writable {
bail!(
"raw index at {} was opened read-only without keys; reopen it keyed to seal it",
self.dir.display()
);
}
let shared = self.at_rest.clone();
let mut held = lock_keys(&shared);
self.commit()?;
if let Some(writer) = self.writer.take() {
writer
.wait_merging_threads()
.context("waiting for raw text merges")?;
}
let generation = self.purge_generation + 1;
let mut reopened = Self::open_keyed(&self.dir.clone(), keys)?;
*held = Some(keys.clone());
reopened.at_rest = shared.clone();
reopened.purge_generation = generation;
drop(held);
*self = reopened;
Ok(())
}
pub fn reseal(&mut self) -> Result<usize> {
self.require_writable()?;
let Some(text) = self.text_at_rest.clone() else {
bail!(
"raw index at {} was opened without keys; reopen it keyed to seal it",
self.dir.display()
);
};
self.commit()?;
if let Some(writer) = self.writer.take() {
writer
.wait_merging_threads()
.context("waiting for raw text merges")?;
}
let mut rewritten = text.reseal_all()?;
let at_rest = self.at_rest.clone();
let keys = lock_keys(&at_rest);
let keys = keys
.as_ref()
.context("a raw index with a sealed text directory holds its keys")?;
for name in top_level_files(&self.dir)? {
let path = self.dir.join(&name);
if is_scratch_file(&name) {
std::fs::remove_file(&path)
.with_context(|| format!("removing {}", path.display()))?;
} else if sealed::reseal_file(&path, keys, &name)? {
rewritten += 1;
}
}
Ok(rewritten)
}
fn ordinal_after(&self, segment: u64) -> Result<u64> {
let mut next = self.active.as_ref().map_or(0, |a| a.len() as u64);
let low = (segment << 32).to_be_bytes();
let high = ((segment + 1) << 32).to_be_bytes();
for reader in self.reader.searcher().segment_readers() {
let inverted = reader.inverted_index(self.fields.key)?;
let mut terms = inverted.terms().range().ge(low).lt(high).into_stream()?;
while terms.advance() {
if let Ok(bytes) = <[u8; 8]>::try_from(terms.key()) {
next = next.max((u64::from_be_bytes(bytes) & u64::from(u32::MAX)) + 1);
}
}
}
Ok(next)
}
fn segments(&self) -> impl Iterator<Item = &VectorIndex> {
self.full_segments
.iter()
.map(|(_, segment)| segment)
.chain(self.active.iter())
}
pub fn len(&self) -> usize {
self.segments().map(VectorIndex::len).sum()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn resident_bytes(&self) -> u64 {
let vectors: usize = self.segments().map(VectorIndex::memory_usage).sum();
vectors as u64 + self.text_bytes
}
pub fn has_text(&self, text: &str) -> bool {
let hash = text_hash(text);
if self.pending_texts.contains(&hash) {
return true;
}
let term = Term::from_field_text(self.fields.text_hash, &hash);
holds_live(&self.reader.searcher(), term).unwrap_or(false)
}
pub fn has_source(&self, source: &RawOrigin) -> bool {
let key = source.source_key();
if self.pending_sources.contains(&key) {
return true;
}
let term = Term::from_field_text(self.fields.source, &key);
holds_live(&self.reader.searcher(), term).unwrap_or(false)
}
pub fn purge_generation(&self) -> u64 {
self.purge_generation
}
pub fn add_source(
&mut self,
origin: &RawOrigin,
passages: &[(RawPassage, Vec<f32>)],
) -> Result<usize> {
self.require_writable()?;
let source = origin.source_key();
let searcher = self.reader.searcher();
let mut added = 0;
for (passage, embedding) in passages {
if embedding.len() != EMBEDDING_DIM || passage.text.trim().is_empty() {
continue;
}
let hash = text_hash(&passage.text);
let term = Term::from_field_text(self.fields.text_hash, &hash);
if self.pending_texts.contains(&hash) || holds_live(&searcher, term)? {
continue;
}
self.pending_texts.insert(hash);
let key = self.next_key()?;
self.active_mut()?.upsert(key, embedding)?;
let document = passage_document(&self.fields, key, &source, passage);
self.writer_mut()?.add_document(document)?;
added += 1;
}
self.pending_sources.insert(source.clone());
if added == 0 {
let marker = doc!(self.fields.source => source.as_str());
self.writer_mut()?.add_document(marker)?;
}
Ok(added)
}
pub fn commit(&mut self) -> Result<()> {
self.require_writable()?;
if let Some(active) = &self.active {
active.save()?;
}
if let Some(writer) = self.writer.as_mut() {
writer.commit().context("raw text commit")?;
}
self.reader.reload().context("raw text reader reload")?;
self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
self.pending_texts.clear();
self.pending_sources.clear();
Ok(())
}
pub fn commit_without_merging(&mut self) -> Result<()> {
self.require_writable()?;
self.writer_mut()?.set_merge_policy(Box::new(NoMergePolicy));
let committed = self.commit();
if let Some(writer) = &self.writer {
writer.set_merge_policy(Box::new(DefaultMergePolicy::default()));
}
committed
}
pub fn reload(&mut self) -> Result<()> {
if self.writable {
return Ok(());
}
let at_rest = self.seal_keys();
let segments = segment_files(&self.dir)?;
let mut full_segments = Vec::with_capacity(segments.len());
for n in segments {
full_segments.push((
n,
open_full_segment(&segment_path(&self.dir, n), at_rest.as_ref())?,
));
}
self.full_segments = full_segments;
self.reader.reload().context("raw text reader reload")?;
self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
Ok(())
}
pub fn vector_search(
&self,
embedding: &[f32],
k: usize,
project: Option<&str>,
) -> Result<Vec<(u64, f32)>> {
let fetch = if project.is_some() { k * OVERFETCH } else { k };
let mut hits = Vec::new();
for segment in self.segments() {
if segment.len() > 0 {
hits.extend(segment.search(embedding, fetch)?);
}
}
hits.sort_by(|a, b| b.1.total_cmp(&a.1));
let mut out = Vec::with_capacity(k);
for (key, score) in hits {
if out.len() == k {
break;
}
let Some(hit) = self.passage(key)? else {
continue;
};
if project.is_none() || hit.project.as_deref() == project {
out.push((key, score));
}
}
Ok(out)
}
pub fn bm25_search(
&self,
query_text: &str,
k: usize,
project: Option<&str>,
) -> Result<Vec<(u64, f32)>> {
let searcher = self.reader.searcher();
let parser = QueryParser::for_index(&self.index, vec![self.fields.body]);
let (text_query, _errors) = parser.parse_query_lenient(query_text);
let query: Box<dyn Query> = match project {
Some(project) => Box::new(BooleanQuery::new(vec![
(Occur::Must, text_query),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(self.fields.project, project),
IndexRecordOption::Basic,
)),
),
])),
None => text_query,
};
let top = searcher.search(&query, &TopDocs::with_limit(k.max(1)).order_by_score())?;
let mut out = Vec::with_capacity(top.len());
for (score, address) in top {
let document: TantivyDocument = searcher.doc(address)?;
if let Some(key) = document.get_first(self.fields.key).and_then(|v| v.as_u64()) {
out.push((key, score));
}
}
Ok(out)
}
pub fn passage(&self, key: u64) -> Result<Option<RawHit>> {
let searcher = self.reader.searcher();
let query = TermQuery::new(
Term::from_field_u64(self.fields.key, key),
IndexRecordOption::Basic,
);
let Some((_, address)) = searcher
.search(&query, &TopDocs::with_limit(1).order_by_score())?
.into_iter()
.next()
else {
return Ok(None);
};
let document: TantivyDocument = searcher.doc(address)?;
Ok(self.hit(&document))
}
fn hit(&self, document: &TantivyDocument) -> Option<RawHit> {
let text = |field| {
document
.get_first(field)
.and_then(|v| v.as_str())
.map(str::to_string)
};
let key = document
.get_first(self.fields.key)
.and_then(|v| v.as_u64())?;
let origin = text(self.fields.source)
.as_deref()
.and_then(RawOrigin::parse)?;
Some(RawHit {
key,
text: text(self.fields.text).unwrap_or_default(),
text_hash: text(self.fields.text_hash).unwrap_or_default(),
at: document
.get_first(self.fields.at)
.and_then(|v| v.as_i64())
.and_then(|s| DateTime::from_timestamp(s, 0)),
project: text(self.fields.project),
memory: text(self.fields.memory).and_then(|m| Uuid::parse_str(&m).ok()),
origin,
})
}
pub fn sources(&self) -> Result<Vec<RawOrigin>> {
Ok(self
.terms(self.fields.source)?
.iter()
.filter_map(|key| RawOrigin::parse(key))
.collect())
}
pub fn passage_memories(&self) -> Result<Vec<Uuid>> {
Ok(self
.terms(self.fields.memory)?
.iter()
.filter_map(|id| Uuid::parse_str(id).ok())
.collect())
}
pub fn matching(
&self,
sources: &[RawOrigin],
memories: &[Uuid],
texts: &[&str],
) -> Result<Vec<RawHit>> {
let hashes: BTreeSet<String> = texts.iter().map(|t| text_hash(t)).collect();
let terms = sources
.iter()
.map(|s| Term::from_field_text(self.fields.source, &s.source_key()))
.chain(
memories
.iter()
.map(|m| Term::from_field_text(self.fields.memory, &m.to_string())),
)
.chain(
hashes
.iter()
.map(|h| Term::from_field_text(self.fields.text_hash, h)),
);
let searcher = self.reader.searcher();
let mut seen = HashSet::new();
let mut hits = Vec::new();
for term in terms {
let query = TermQuery::new(term, IndexRecordOption::Basic);
for address in searcher.search(&query, &DocSetCollector)? {
let document: TantivyDocument = searcher.doc(address)?;
if let Some(hit) = self.hit(&document)
&& seen.insert(hit.key)
{
hits.push(hit);
}
}
}
Ok(hits)
}
pub fn withdraw(&mut self, keys: &[u64]) -> Result<()> {
self.require_writable()?;
if keys.is_empty() {
return Ok(());
}
self.purge_generation += 1;
let active_segment = self.active_segment;
if let Some(active) = self.active.as_mut() {
for key in keys.iter().filter(|k| *k >> 32 == active_segment) {
active.remove(*key)?;
}
}
let field = self.fields.key;
let writer = self.writer_mut()?;
for key in keys {
writer.delete_term(Term::from_field_u64(field, *key));
}
Ok(())
}
pub fn rekey(&mut self, key: u64, origin: &RawOrigin, passage: &RawPassage) -> Result<()> {
self.require_writable()?;
self.purge_generation += 1;
let document = passage_document(&self.fields, key, &origin.source_key(), passage);
let term = Term::from_field_u64(self.fields.key, key);
let writer = self.writer_mut()?;
writer.delete_term(term);
writer.add_document(document)?;
Ok(())
}
pub fn forget_sources(&mut self, sources: &[RawOrigin]) -> Result<()> {
self.require_writable()?;
let field = self.fields.source;
let writer = self.writer_mut()?;
for source in sources {
writer.delete_term(Term::from_field_text(field, &source.source_key()));
}
Ok(())
}
pub fn seal_for_rewrite(&mut self, keys: &[u64]) -> Result<SealedRewrite> {
self.require_writable()?;
let at_rest = self.seal_keys();
let mut pending = read_journal(&self.dir, at_rest.as_ref())?;
pending.extend(keys.iter().copied());
if pending.iter().any(|k| k >> 32 == self.active_segment) {
self.close_active()?;
}
if !pending.is_empty() {
write_journal(&self.dir, &pending, at_rest.as_ref())?;
}
Ok(SealedRewrite {
dir: self.dir.clone(),
keys: pending,
at_rest: self.at_rest.clone(),
})
}
pub fn reload_sealed(&mut self, rewrite: &SealedRewrite) -> Result<()> {
let segments: HashSet<u64> = rewrite.keys.iter().map(|k| k >> 32).collect();
for (n, segment) in &mut self.full_segments {
if segments.contains(n) {
segment.reload_view()?;
}
}
Ok(())
}
pub fn start_compaction(&mut self) -> Result<Compaction> {
self.require_writable()?;
let segments = self
.index
.searchable_segment_ids()
.context("listing raw text segments")?;
if segments.is_empty() {
return Ok(Compaction { merge: None });
}
let merge = self.writer_mut()?.merge(&segments);
Ok(Compaction { merge: Some(merge) })
}
pub fn finish_compaction(&mut self, merged: Result<()>) -> Result<()> {
self.require_writable()?;
if let Err(refused) = merged {
return self
.compact()
.with_context(|| format!("compacting after the merge failed: {refused:#}"));
}
if let Some(writer) = &self.writer {
writer
.garbage_collect_files()
.wait()
.context("raw text garbage collection")?;
}
self.reader.reload().context("raw text reader reload")?;
self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
Ok(())
}
pub fn compact(&mut self) -> Result<()> {
self.commit()?;
if let Some(writer) = self.writer.take() {
writer
.wait_merging_threads()
.context("waiting for raw text merges")?;
}
let segments = self
.index
.searchable_segment_ids()
.context("listing raw text segments")?;
let writer = self.writer_mut()?;
if !segments.is_empty() {
writer.merge(&segments).wait().context("raw text merge")?;
}
writer
.garbage_collect_files()
.wait()
.context("raw text garbage collection")?;
self.reader.reload().context("raw text reader reload")?;
self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
Ok(())
}
fn terms(&self, field: Field) -> Result<Vec<String>> {
let searcher = self.reader.searcher();
let mut found = std::collections::BTreeSet::new();
for segment in searcher.segment_readers() {
let inverted = segment.inverted_index(field)?;
let mut stream = inverted.terms().stream()?;
while stream.advance() {
found.insert(String::from_utf8_lossy(stream.key()).into_owned());
}
}
Ok(found.into_iter().collect())
}
fn next_key(&mut self) -> Result<u64> {
if self.next_ordinal >= self.segment_capacity {
self.close_active()?;
}
let ordinal = self.next_ordinal;
self.next_ordinal += 1;
Ok((self.active_segment << 32) | ordinal)
}
fn close_active(&mut self) -> Result<()> {
let active = self
.active
.take()
.context("raw index has no active segment")?;
active.save()?;
drop(active);
let at_rest = self.seal_keys();
let path = segment_path(&self.dir, self.active_segment);
self.full_segments.push((
self.active_segment,
open_full_segment(&path, at_rest.as_ref())?,
));
self.active_segment += 1;
self.next_ordinal = 0;
self.active = Some(open_writable_segment(
&segment_path(&self.dir, self.active_segment),
at_rest.as_ref(),
)?);
Ok(())
}
fn active_mut(&mut self) -> Result<&mut VectorIndex> {
self.active
.as_mut()
.context("raw index has no active segment")
}
fn writer_mut(&mut self) -> Result<&mut IndexWriter> {
if self.writer.is_none() {
self.writer = Some(
self.index
.writer_with_num_threads(1, WRITER_HEAP_BYTES)
.context("creating raw text writer")?,
);
}
Ok(self.writer.as_mut().expect("writer just created"))
}
fn require_writable(&self) -> Result<()> {
if self.writable {
Ok(())
} else {
Err(anyhow!("raw index is open read-only"))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::keys::KeyGuard;
use crate::sealed::{SealError, SealKey, SealKeyId, seal_error, sealed_key_id};
use tantivy::{DocAddress, Document, IndexSettings};
fn unit(i: usize) -> Vec<f32> {
let mut v = vec![0.0; EMBEDDING_DIM];
v[i % EMBEDDING_DIM] = 1.0;
v
}
fn passage(text: &str, project: Option<&str>) -> RawPassage {
RawPassage {
text: text.to_string(),
at: DateTime::from_timestamp(1_790_000_000, 0),
project: project.map(str::to_string),
memory: None,
}
}
fn transcript(sha: &str) -> RawOrigin {
RawOrigin::Transcript {
sha256: sha.to_string(),
}
}
fn purge(
index: &mut RawIndex,
sources: &[RawOrigin],
memories: &[Uuid],
texts: &[&str],
) -> usize {
let keys: Vec<u64> = index
.matching(sources, memories, texts)
.unwrap()
.iter()
.map(|hit| hit.key)
.collect();
index.withdraw(&keys).unwrap();
index.forget_sources(sources).unwrap();
let rewrite = index.seal_for_rewrite(&keys).unwrap();
index.commit().unwrap();
rewrite.run().unwrap();
index.reload_sealed(&rewrite).unwrap();
keys.len()
}
fn holds_key(index: &RawIndex, key: u64, near: &[f32]) -> bool {
index
.segments()
.filter(|segment| segment.len() > 0)
.any(|segment| {
segment
.search(near, 10)
.unwrap()
.iter()
.any(|(k, _)| *k == key)
})
}
#[test]
fn a_passage_keeps_the_raw_memory_it_maps_to_and_its_text_is_known_before_commit() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let memory = Uuid::new_v4();
let mut mapped = passage("the mirror job runs at four", None);
mapped.memory = Some(memory);
assert!(!index.has_text("the mirror job runs at four"));
index
.add_source(&transcript("aa"), &[(mapped, unit(1))])
.unwrap();
assert!(index.has_text("the mirror job runs at four"));
index.commit().unwrap();
assert!(index.has_text(" the mirror job runs at four \n"));
assert!(!index.has_text("the mirror job runs at five"));
let key = index.vector_search(&unit(1), 1, None).unwrap()[0].0;
assert_eq!(index.passage(key).unwrap().unwrap().memory, Some(memory));
assert!(index.resident_bytes() > 0);
}
#[test]
fn a_passage_is_found_by_vector_and_by_words_with_its_metadata() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let source = transcript("aa");
index
.add_source(
&source,
&[
(
passage(
"the garden daemon listens on port 7200",
Some("-srv-garden"),
),
unit(1),
),
(passage("lunch was soup", Some("-srv-garden")), unit(2)),
],
)
.unwrap();
index.commit().unwrap();
let by_vector = index.vector_search(&unit(1), 1, None).unwrap();
let hit = index.passage(by_vector[0].0).unwrap().unwrap();
assert_eq!(hit.text, "the garden daemon listens on port 7200");
assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
assert_eq!(hit.origin, source);
assert_eq!(hit.at.unwrap().timestamp(), 1_790_000_000);
let by_words = index.bm25_search("daemon port", 5, None).unwrap();
assert_eq!(by_words.len(), 1);
assert_eq!(by_words[0].0, by_vector[0].0);
}
#[test]
fn a_project_filter_keeps_only_that_projects_passages() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
index
.add_source(
&transcript("aa"),
&[
(
passage("deploy uses argo here", Some("-srv-garden")),
unit(1),
),
(
passage("deploy uses argo there", Some("-srv-orchard")),
unit(1),
),
],
)
.unwrap();
index.commit().unwrap();
for hits in [
index
.vector_search(&unit(1), 5, Some("-srv-orchard"))
.unwrap(),
index
.bm25_search("deploy argo", 5, Some("-srv-orchard"))
.unwrap(),
] {
assert_eq!(hits.len(), 1);
let hit = index.passage(hits[0].0).unwrap().unwrap();
assert_eq!(hit.project.as_deref(), Some("-srv-orchard"));
}
}
#[test]
fn the_same_text_from_two_sources_is_stored_once_and_both_sources_are_recorded() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let memory = RawOrigin::Memory { id: Uuid::new_v4() };
let text = "the port moved to 7200";
assert_eq!(
index
.add_source(&transcript("aa"), &[(passage(text, None), unit(1))])
.unwrap(),
1
);
index.commit().unwrap();
assert_eq!(
index
.add_source(&memory, &[(passage(text, None), unit(1))])
.unwrap(),
0
);
index.commit().unwrap();
assert_eq!(index.len(), 1);
assert!(index.has_source(&transcript("aa")));
assert!(index.has_source(&memory));
assert!(!index.has_source(&transcript("bb")));
}
#[test]
fn a_passage_without_a_full_embedding_is_skipped() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let added = index
.add_source(
&transcript("aa"),
&[(passage("short vector", None), vec![1.0; 3])],
)
.unwrap();
index.commit().unwrap();
assert_eq!(added, 0);
assert!(index.is_empty());
assert!(index.has_source(&transcript("aa")));
}
#[test]
fn full_segments_are_sealed_and_everything_survives_a_reopen() {
let dir = tempfile::tempdir().unwrap();
{
let mut index = RawIndex::open(dir.path()).unwrap();
index.segment_capacity = 2;
let passages: Vec<_> = (0..5)
.map(|i| (passage(&format!("passage number {i}"), None), unit(i)))
.collect();
index.add_source(&transcript("aa"), &passages).unwrap();
index.commit().unwrap();
assert_eq!(index.full_segments.len(), 2);
}
assert_eq!(segment_files(dir.path()).unwrap(), vec![0, 1, 2]);
let mut reopened = RawIndex::open(dir.path()).unwrap();
assert_eq!(reopened.len(), 5);
let hit = reopened.vector_search(&unit(3), 1, None).unwrap()[0].0;
assert_eq!(
reopened.passage(hit).unwrap().unwrap().text,
"passage number 3"
);
reopened
.add_source(
&transcript("bb"),
&[(passage("one more passage", None), unit(9))],
)
.unwrap();
reopened.commit().unwrap();
let readonly = RawIndex::open_readonly(dir.path()).unwrap();
assert_eq!(readonly.len(), 6);
let hit = readonly.bm25_search("one more", 1, None).unwrap()[0].0;
assert_eq!(
readonly.passage(hit).unwrap().unwrap().text,
"one more passage"
);
}
#[test]
fn a_read_only_index_refuses_writes() {
let dir = tempfile::tempdir().unwrap();
RawIndex::open(dir.path()).unwrap().commit().unwrap();
let mut readonly = RawIndex::open_readonly(dir.path()).unwrap();
assert!(readonly.add_source(&transcript("aa"), &[]).is_err());
}
#[test]
fn origins_round_trip_through_their_source_key() {
for origin in [transcript("ab12"), RawOrigin::Memory { id: Uuid::new_v4() }] {
assert_eq!(RawOrigin::parse(&origin.source_key()), Some(origin));
}
assert_eq!(RawOrigin::parse("x:1"), None);
}
#[test]
fn removing_a_source_withdraws_its_passages_and_their_text_from_the_files() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
index
.add_source(
&transcript("aa"),
&[
(passage("the gate code sits under a stone", None), unit(1)),
(
passage("a second line of the same transcript", None),
unit(2),
),
],
)
.unwrap();
index
.add_source(
&transcript("bb"),
&[(passage("an unrelated line", None), unit(3))],
)
.unwrap();
index.commit().unwrap();
assert_eq!(index.sources().unwrap().len(), 2);
assert_eq!(purge(&mut index, &[transcript("aa")], &[], &[]), 2);
index.commit().unwrap();
index.compact().unwrap();
assert!(
index
.bm25_search("gate code stone", 5, None)
.unwrap()
.is_empty()
);
assert!(
index
.vector_search(&unit(1), 5, None)
.unwrap()
.iter()
.all(|(k, _)| { index.passage(*k).unwrap().unwrap().text == "an unrelated line" })
);
assert!(!index.has_source(&transcript("aa")));
assert!(!index.has_text("the gate code sits under a stone"));
assert_eq!(index.sources().unwrap(), vec![transcript("bb")]);
assert_eq!(index.len(), 1);
let leftovers: Vec<String> = std::fs::read_dir(dir.path().join(TEXT_DIR))
.unwrap()
.filter_map(|e| std::fs::read(e.ok()?.path()).ok())
.map(|b| String::from_utf8_lossy(&b).into_owned())
.filter(|t| t.contains("stone"))
.collect();
assert!(
leftovers.is_empty(),
"the removed text left the index files"
);
}
#[test]
fn removing_a_memory_withdraws_the_transcript_passages_that_stand_for_it() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let gone = Uuid::new_v4();
let mut mapped = passage("passage the memory stands for", None);
mapped.memory = Some(gone);
index
.add_source(
&transcript("aa"),
&[
(mapped, unit(1)),
(passage("a passage of nobody", None), unit(2)),
],
)
.unwrap();
index
.add_source(
&RawOrigin::Memory { id: gone },
&[(passage("the archived memory itself", None), unit(3))],
)
.unwrap();
index.commit().unwrap();
assert_eq!(index.passage_memories().unwrap(), vec![gone]);
let removed = purge(&mut index, &[RawOrigin::Memory { id: gone }], &[gone], &[]);
index.commit().unwrap();
assert_eq!(removed, 2);
assert!(index.bm25_search("memory", 5, None).unwrap().is_empty());
assert_eq!(index.bm25_search("nobody", 5, None).unwrap().len(), 1);
assert_eq!(index.len(), 1);
}
#[test]
fn removing_by_text_withdraws_the_passage_whichever_source_holds_it() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
index
.add_source(
&transcript("old"),
&[(passage("a line the user typed", None), unit(1))],
)
.unwrap();
index
.add_source(
&transcript("new"),
&[(passage("a later line", None), unit(2))],
)
.unwrap();
index.commit().unwrap();
let removed = purge(
&mut index,
&[transcript("new")],
&[],
&["a line the user typed", "a later line"],
);
index.commit().unwrap();
assert_eq!(removed, 2);
assert!(index.bm25_search("line", 5, None).unwrap().is_empty());
}
#[test]
fn a_passage_in_a_sealed_segment_stops_resolving_once_removed() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
index.segment_capacity = 1;
index
.add_source(
&transcript("aa"),
&[(passage("sealed away early", None), unit(1))],
)
.unwrap();
index
.add_source(
&transcript("bb"),
&[(passage("kept passage", None), unit(2))],
)
.unwrap();
index.commit().unwrap();
assert!(!index.full_segments.is_empty());
purge(&mut index, &[transcript("aa")], &[], &[]);
index.commit().unwrap();
let hits = index.vector_search(&unit(1), 5, None).unwrap();
assert!(
hits.iter()
.all(|(k, _)| { index.passage(*k).unwrap().unwrap().text == "kept passage" })
);
}
#[test]
fn a_withdrawn_key_leaves_the_sealed_segment_file_that_held_it() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
index.segment_capacity = 1;
index
.add_source(
&transcript("aa"),
&[(passage("sealed away early", None), unit(1))],
)
.unwrap();
index
.add_source(
&transcript("bb"),
&[(passage("kept passage", None), unit(2))],
)
.unwrap();
index.commit().unwrap();
let key = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
assert_eq!(key >> 32, 0);
assert!(holds_key(&index, key, &unit(1)));
let before = std::fs::metadata(segment_path(dir.path(), 0))
.unwrap()
.len();
purge(&mut index, &[transcript("aa")], &[], &[]);
assert!(!holds_key(&index, key, &unit(1)));
drop(index);
let after = std::fs::metadata(segment_path(dir.path(), 0))
.unwrap()
.len();
assert!(after < before, "the embedding stayed in the file");
assert!(!dir.path().join(REWRITE_JOURNAL).exists());
let reopened = RawIndex::open(dir.path()).unwrap();
assert!(!holds_key(&reopened, key, &unit(1)));
assert_eq!(reopened.len(), 1);
let hit = reopened.vector_search(&unit(2), 1, None).unwrap()[0].0;
assert_eq!(reopened.passage(hit).unwrap().unwrap().text, "kept passage");
}
#[test]
fn a_rewrite_a_crash_interrupted_is_finished_at_open() {
let dir = tempfile::tempdir().unwrap();
let key = {
let mut index = RawIndex::open(dir.path()).unwrap();
index
.add_source(
&transcript("aa"),
&[(passage("withdrawn before the crash", None), unit(1))],
)
.unwrap();
index
.add_source(
&transcript("bb"),
&[(passage("kept passage", None), unit(2))],
)
.unwrap();
index.commit().unwrap();
let key = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
index.withdraw(&[key]).unwrap();
index.forget_sources(&[transcript("aa")]).unwrap();
index.seal_for_rewrite(&[key]).unwrap();
index.commit().unwrap();
key
};
assert!(dir.path().join(REWRITE_JOURNAL).exists());
let reopened = RawIndex::open(dir.path()).unwrap();
assert!(!dir.path().join(REWRITE_JOURNAL).exists());
assert!(!holds_key(&reopened, key, &unit(1)));
assert_eq!(reopened.len(), 1);
}
#[test]
fn a_key_withdrawn_from_the_loaded_segment_is_never_handed_out_again() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let passages: Vec<_> = (0..3)
.map(|i| (passage(&format!("passage number {i}"), None), unit(i)))
.collect();
index.add_source(&transcript("aa"), &passages).unwrap();
index.commit().unwrap();
let first = index.matching(&[], &[], &["passage number 0"]).unwrap()[0].key;
index.withdraw(&[first]).unwrap();
index.commit().unwrap();
index
.add_source(&transcript("bb"), &[(passage("one more", None), unit(7))])
.unwrap();
index.commit().unwrap();
drop(index);
let mut reopened = RawIndex::open(dir.path()).unwrap();
reopened
.add_source(
&transcript("cc"),
&[(passage("and another", None), unit(8))],
)
.unwrap();
reopened.commit().unwrap();
let all = reopened
.matching(
&[transcript("aa"), transcript("bb"), transcript("cc")],
&[],
&[],
)
.unwrap();
assert_eq!(all.len(), 4);
for (text, near) in [
("passage number 1", unit(1)),
("passage number 2", unit(2)),
("one more", unit(7)),
("and another", unit(8)),
] {
let key = reopened.vector_search(&near, 1, None).unwrap()[0].0;
assert_eq!(reopened.passage(key).unwrap().unwrap().text, text);
}
}
#[test]
fn a_withdrawn_text_reads_as_not_indexed_before_any_merge() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
index
.add_source(
&transcript("aa"),
&[
(
passage("the lantern hangs by the north door", None),
unit(1),
),
(passage("the gate is painted red", None), unit(2)),
],
)
.unwrap();
index.commit().unwrap();
let generation = index.purge_generation();
let key = index
.matching(&[], &[], &["the lantern hangs by the north door"])
.unwrap()[0]
.key;
index.withdraw(&[key]).unwrap();
index.commit_without_merging().unwrap();
let term = Term::from_field_text(
index.fields.text_hash,
&text_hash("the lantern hangs by the north door"),
);
assert_eq!(index.reader.searcher().doc_freq(&term).unwrap(), 1);
assert_ne!(index.purge_generation(), generation);
assert!(!index.has_text("the lantern hangs by the north door"));
assert!(index.has_text("the gate is painted red"));
assert!(index.has_source(&transcript("aa")));
assert_eq!(
index
.add_source(
&transcript("bb"),
&[(
passage("the lantern hangs by the north door", None),
unit(1)
)],
)
.unwrap(),
1
);
let rest = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
index.withdraw(&[rest]).unwrap();
index.forget_sources(&[transcript("aa")]).unwrap();
index.commit_without_merging().unwrap();
assert!(!index.has_source(&transcript("aa")));
assert!(index.has_source(&transcript("bb")));
}
#[test]
fn a_rekeyed_passage_keeps_its_vector_under_its_new_source() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
index
.add_source(&transcript("gone"), &[(passage("yes", None), unit(1))])
.unwrap();
index.commit().unwrap();
let key = index.matching(&[transcript("gone")], &[], &[]).unwrap()[0].key;
index
.rekey(
key,
&transcript("kept"),
&passage("yes", Some("-srv-garden")),
)
.unwrap();
index.forget_sources(&[transcript("gone")]).unwrap();
index.commit().unwrap();
index.compact().unwrap();
let hit = index.vector_search(&unit(1), 1, None).unwrap()[0].0;
assert_eq!(hit, key);
let hit = index.passage(hit).unwrap().unwrap();
assert_eq!(hit.origin, transcript("kept"));
assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
assert_eq!(index.sources().unwrap(), vec![transcript("kept")]);
assert!(index.has_text("yes"));
}
const MARKER: &str = "zanzibarquokka";
fn seal_key(byte: u8) -> SealKey {
SealKey::from_bytes(KeyGuard::new([byte; 32])).unwrap()
}
fn seal_keys(byte: u8) -> SealKeys {
SealKeys::single(seal_key(byte))
}
fn files_under(dir: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
for entry in std::fs::read_dir(dir).unwrap() {
let path = entry.unwrap().path();
let name = path.file_name().unwrap().to_string_lossy().into_owned();
if path.is_dir() {
out.extend(files_under(&path));
} else if !(name.starts_with(".tantivy-") && name.ends_with(".lock")) {
out.push(path);
}
}
out
}
fn assert_all_sealed_under(dir: &Path, id: SealKeyId) {
let files = files_under(dir);
assert!(files.iter().any(|p| p.starts_with(dir.join(TEXT_DIR))));
assert!(
files
.iter()
.any(|p| p.extension().is_some_and(|e| e == "usearch"))
);
for path in files {
assert_eq!(
sealed_key_id(&path).unwrap(),
Some(id),
"{} is not sealed under the current key",
path.display()
);
}
}
fn holds_bytes(dir: &Path, needle: &str) -> bool {
files_under(dir).iter().any(|p| {
std::fs::read(p)
.unwrap()
.windows(needle.len())
.any(|w| w == needle.as_bytes())
})
}
fn spread(i: usize) -> Vec<f32> {
let mut v = unit(i);
v[(i + 1) % EMBEDDING_DIM] = 0.5;
v
}
fn ramp(i: usize) -> Vec<f32> {
let mut v: Vec<f32> = (0..EMBEDDING_DIM)
.map(|j| (j + 1) as f32 / EMBEDDING_DIM as f32)
.collect();
v[i] += 5.0;
v
}
fn populate(index: &mut RawIndex) {
index.segment_capacity = 2;
let passages: Vec<_> = (0..5)
.map(|i| {
(
passage(&format!("{MARKER} passage number {i}"), Some("-srv-garden")),
spread(i),
)
})
.collect();
index.add_source(&transcript("aa"), &passages).unwrap();
let id = Uuid::new_v4();
let mut memory = passage(&format!("{MARKER} archived memory"), None);
memory.memory = Some(id);
index
.add_source(&RawOrigin::Memory { id }, &[(memory, spread(7))])
.unwrap();
index.commit().unwrap();
}
type TopK = Vec<Vec<(u64, f32)>>;
fn top_k(index: &RawIndex) -> TopK {
let mut out: TopK = (0..8)
.map(|i| index.vector_search(&ramp(i), 4, None).unwrap())
.collect();
out.push(index.bm25_search(MARKER, 10, None).unwrap());
out.push(
index
.bm25_search("number 3", 10, Some("-srv-garden"))
.unwrap(),
);
assert!(out.iter().all(|hits| !hits.is_empty()));
out
}
fn stored_json(index: &RawIndex) -> String {
let searcher = index.reader.searcher();
let schema = index.index.schema();
let mut out = String::new();
for (ord, segment) in searcher.segment_readers().iter().enumerate() {
for doc in 0..segment.max_doc() {
let document: TantivyDocument =
searcher.doc(DocAddress::new(ord as u32, doc)).unwrap();
out.push_str(&document.to_json(&schema));
out.push('\n');
}
}
out
}
#[test]
fn a_keyed_index_does_not_open_without_its_key_or_under_another() {
let dir = tempfile::tempdir().unwrap();
populate(&mut RawIndex::open_keyed(dir.path(), &seal_keys(1)).unwrap());
for opened in [
RawIndex::open(dir.path()),
RawIndex::open_readonly(dir.path()),
] {
let err = opened.err().expect("a sealed index opened without keys");
assert_eq!(seal_error(&err), Some(&SealError::KeyRequired), "{err:#}");
}
for opened in [
RawIndex::open_keyed(dir.path(), &seal_keys(2)),
RawIndex::open_readonly_keyed(dir.path(), &seal_keys(2)),
] {
let err = opened
.err()
.expect("a sealed index opened under another key");
assert!(
matches!(seal_error(&err), Some(SealError::UnknownKey(_))),
"{err:#}"
);
}
let reopened = RawIndex::open_readonly_keyed(dir.path(), &seal_keys(1)).unwrap();
assert_eq!(reopened.len(), 6);
assert_eq!(
reopened.seal_keys().unwrap().current_id(),
seal_keys(1).current_id()
);
}
#[test]
fn a_plaintext_index_opened_keyed_searches_the_same_and_reseal_seals_every_file() {
let dir = tempfile::tempdir().unwrap();
populate(&mut RawIndex::open(dir.path()).unwrap());
let baseline = top_k(&RawIndex::open_readonly(dir.path()).unwrap());
let keys = seal_keys(3);
let mut index = RawIndex::open_keyed(dir.path(), &keys).unwrap();
assert_eq!(top_k(&index), baseline);
assert!(index.reseal().unwrap() > 0);
assert_all_sealed_under(dir.path(), keys.current_id());
assert_eq!(top_k(&index), baseline);
assert_eq!(index.reseal().unwrap(), 0);
drop(index);
assert_eq!(
top_k(&RawIndex::open_readonly_keyed(dir.path(), &keys).unwrap()),
baseline
);
let reopened = RawIndex::open_keyed(dir.path(), &keys).unwrap();
assert_eq!(top_k(&reopened), baseline);
let hit = reopened.bm25_search("number 3", 1, None).unwrap()[0].0;
assert_eq!(
reopened.passage(hit).unwrap().unwrap().text,
format!("{MARKER} passage number 3")
);
}
#[test]
fn no_passage_text_reaches_a_file_of_a_keyed_index() {
let dir = tempfile::tempdir().unwrap();
let keys = seal_keys(4);
let mut index = RawIndex::open_keyed(dir.path(), &keys).unwrap();
populate(&mut index);
index.compact().unwrap();
assert!(!holds_bytes(dir.path(), MARKER));
assert_all_sealed_under(dir.path(), keys.current_id());
let hit = index.bm25_search("number 3", 1, None).unwrap()[0].0;
assert_eq!(
index.passage(hit).unwrap().unwrap().text,
format!("{MARKER} passage number 3")
);
}
#[test]
fn a_rotation_reseals_every_file_under_the_new_key_only() {
let dir = tempfile::tempdir().unwrap();
populate(&mut RawIndex::open_keyed(dir.path(), &SealKeys::single(seal_key(5))).unwrap());
let mut index = RawIndex::open_keyed(dir.path(), &SealKeys::single(seal_key(5))).unwrap();
let baseline = top_k(&index);
index
.set_seal_keys(SealKeys::new(seal_key(6), vec![seal_key(5)]))
.unwrap();
assert!(index.reseal().unwrap() > 0);
assert_all_sealed_under(dir.path(), seal_key(6).id());
assert_eq!(top_k(&index), baseline);
index
.add_source(
&transcript("bb"),
&[(passage("one more passage", None), unit(9))],
)
.unwrap();
index.commit().unwrap();
drop(index);
let reopened = RawIndex::open_keyed(dir.path(), &SealKeys::single(seal_key(6))).unwrap();
assert_eq!(reopened.len(), 7);
assert_all_sealed_under(dir.path(), seal_key(6).id());
}
#[test]
fn a_rewrite_drops_withdrawn_vectors_from_sealed_segments_and_keeps_its_journal_sealed() {
let dir = tempfile::tempdir().unwrap();
let keys = seal_keys(7);
let (gone, crashed) = {
let mut index = RawIndex::open_keyed(dir.path(), &keys).unwrap();
index.segment_capacity = 1;
for (sha, text, i) in [
("aa", "sealed away early", 1),
("bb", "withdrawn before the crash", 2),
("cc", "kept passage", 3),
] {
index
.add_source(&transcript(sha), &[(passage(text, None), unit(i))])
.unwrap();
}
index.commit().unwrap();
let gone = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
assert!(holds_key(&index, gone, &unit(1)));
purge(&mut index, &[transcript("aa")], &[], &[]);
assert!(!holds_key(&index, gone, &unit(1)));
let crashed = index.matching(&[transcript("bb")], &[], &[]).unwrap()[0].key;
index.withdraw(&[crashed]).unwrap();
index.forget_sources(&[transcript("bb")]).unwrap();
index.seal_for_rewrite(&[crashed]).unwrap();
index.commit().unwrap();
(gone, crashed)
};
let journal = dir.path().join(REWRITE_JOURNAL);
assert_eq!(sealed_key_id(&journal).unwrap(), Some(keys.current_id()));
let err = RawIndex::open(dir.path())
.err()
.expect("a sealed journal read without keys");
assert_eq!(seal_error(&err), Some(&SealError::KeyRequired), "{err:#}");
assert!(journal.exists());
let reopened = RawIndex::open_keyed(dir.path(), &keys).unwrap();
assert!(!journal.exists());
assert!(!holds_key(&reopened, gone, &unit(1)));
assert!(!holds_key(&reopened, crashed, &unit(2)));
assert_eq!(reopened.len(), 1);
for n in segment_files(dir.path()).unwrap() {
assert_eq!(
sealed_key_id(&segment_path(dir.path(), n)).unwrap(),
Some(keys.current_id())
);
}
let hit = reopened.vector_search(&unit(3), 1, None).unwrap()[0].0;
assert_eq!(reopened.passage(hit).unwrap().unwrap().text, "kept passage");
}
#[test]
fn a_rewrite_pending_when_the_index_is_reopened_keyed_writes_sealed() {
let dir = tempfile::tempdir().unwrap();
let keys = seal_keys(8);
let mut index = RawIndex::open(dir.path()).unwrap();
index.segment_capacity = 1;
for (sha, text, i) in [
("aa", "withdrawn while unkeyed", 1),
("bb", "kept passage", 2),
] {
index
.add_source(&transcript(sha), &[(passage(text, None), unit(i))])
.unwrap();
}
index.commit().unwrap();
let gone = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
index.withdraw(&[gone]).unwrap();
index.forget_sources(&[transcript("aa")]).unwrap();
let rewrite = index.seal_for_rewrite(&[gone]).unwrap();
index.commit().unwrap();
index.set_seal_keys(keys.clone()).unwrap();
index.reseal().unwrap();
rewrite.run().unwrap();
for n in segment_files(dir.path()).unwrap() {
assert_eq!(
sealed_key_id(&segment_path(dir.path(), n)).unwrap(),
Some(keys.current_id()),
"segment {n}"
);
}
let reopened = RawIndex::open_keyed(dir.path(), &keys).unwrap();
assert!(!holds_key(&reopened, gone, &unit(1)));
assert_eq!(reopened.len(), 1);
}
#[test]
fn an_archived_memorys_passage_is_searchable_but_keeps_no_stored_text() {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
let id = Uuid::new_v4();
let text = "the quokka ferry leaves at dawn";
let mut memory = passage(text, None);
memory.memory = Some(id);
index
.add_source(&RawOrigin::Memory { id }, &[(memory.clone(), unit(1))])
.unwrap();
index
.add_source(
&transcript("aa"),
&[(
passage("the lantern hangs by the north door", None),
unit(2),
)],
)
.unwrap();
index.commit().unwrap();
let stored = stored_json(&index);
assert!(!stored.contains("quokka"), "{stored}");
assert!(stored.contains("the lantern hangs by the north door"));
let key = index.bm25_search("quokka ferry", 5, None).unwrap()[0].0;
assert_eq!(index.vector_search(&unit(1), 1, None).unwrap()[0].0, key);
let hit = index.passage(key).unwrap().unwrap();
assert_eq!(hit.origin, RawOrigin::Memory { id });
assert_eq!(hit.memory, Some(id));
assert!(hit.text.is_empty());
assert_eq!(hit.text_hash, text_hash(text));
assert!(index.has_text(text));
assert_eq!(index.matching(&[], &[], &[text]).unwrap()[0].key, key);
index
.rekey(key, &transcript("bb"), &passage(text, None))
.unwrap();
index.commit().unwrap();
assert_eq!(index.passage(key).unwrap().unwrap().text, text);
index
.rekey(key, &RawOrigin::Memory { id }, &memory)
.unwrap();
index.compact().unwrap();
assert!(!stored_json(&index).contains("quokka"));
assert_eq!(index.bm25_search("quokka", 5, None).unwrap()[0].0, key);
}
type LegacyDoc = (RawOrigin, &'static str, Option<Uuid>);
fn write_legacy_index(dir: &Path, docs: &[LegacyDoc]) -> Vec<u64> {
let text_dir = dir.join(TEXT_DIR);
std::fs::create_dir_all(&text_dir).unwrap();
let schema = legacy_schema();
let field = |name: &str| schema.get_field(name).unwrap();
let index = Index::create(
MmapDirectory::open(&text_dir).unwrap(),
schema.clone(),
IndexSettings::default(),
)
.unwrap();
let mut writer: IndexWriter = index.writer_with_num_threads(1, WRITER_HEAP_BYTES).unwrap();
let mut vectors =
VectorIndex::open_with_kind(&segment_path(dir, 0), ScalarKind::F16).unwrap();
let mut keys = Vec::new();
for (i, (origin, text, memory)) in docs.iter().enumerate() {
let key = i as u64;
let mut document = doc!(
field("key") => key,
field("body") => *text,
field("text_hash") => text_hash(text),
field("source") => origin.source_key(),
field("project") => "-srv-garden",
field("at") => 1_790_000_000i64,
);
if let Some(memory) = memory {
document.add_text(field("memory"), memory.to_string());
}
writer.add_document(document).unwrap();
vectors.upsert(key, &unit(i)).unwrap();
keys.push(key);
}
let withdrawn = docs.len() as u64;
writer
.add_document(doc!(
field("key") => withdrawn,
field("body") => "a withdrawn line",
field("text_hash") => text_hash("a withdrawn line"),
field("source") => transcript("aa").source_key(),
))
.unwrap();
writer.delete_term(Term::from_field_u64(field("key"), withdrawn));
writer
.add_document(doc!(field("source") => transcript("empty").source_key()))
.unwrap();
writer.commit().unwrap();
writer.wait_merging_threads().unwrap();
vectors.save().unwrap();
keys
}
fn legacy_docs(memory: Uuid) -> Vec<LegacyDoc> {
vec![
(
transcript("aa"),
"the lantern hangs by the north door",
None,
),
(
RawOrigin::Memory { id: memory },
"the quokka ferry leaves at dawn",
Some(memory),
),
(
transcript("aa"),
"a third line of the transcript",
Some(Uuid::new_v4()),
),
]
}
fn assert_migrated(dir: &Path, index: &RawIndex, docs: &[LegacyDoc], keys: &[u64]) {
assert_eq!(index.index.schema(), schema().0);
assert_eq!(index.len(), docs.len());
for (i, ((origin, text, memory), key)) in docs.iter().zip(keys).enumerate() {
assert_eq!(index.vector_search(&unit(i), 1, None).unwrap()[0].0, *key);
let hit = index.passage(*key).unwrap().unwrap();
assert_eq!(&hit.origin, origin);
assert_eq!(hit.memory, *memory);
assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
assert_eq!(hit.at.unwrap().timestamp(), 1_790_000_000);
assert_eq!(hit.text_hash, text_hash(text));
let stored = match origin {
RawOrigin::Memory { .. } => "",
RawOrigin::Transcript { .. } => text,
};
assert_eq!(hit.text, stored);
assert!(index.has_text(text));
}
assert_eq!(index.bm25_search("quokka", 5, None).unwrap()[0].0, keys[1]);
assert!(!index.has_text("a withdrawn line"));
assert!(index.has_source(&transcript("empty")));
assert!(!stored_json(index).contains("quokka"));
for leftover in [MIGRATED_TEXT_DIR, REPLACED_TEXT_DIR, MIGRATION_JOURNAL] {
assert!(!dir.join(leftover).exists(), "{leftover} was left behind");
}
}
#[test]
fn an_index_written_with_the_old_schema_migrates_keeping_every_passage_and_vector() {
let dir = tempfile::tempdir().unwrap();
let docs = legacy_docs(Uuid::new_v4());
let keys = write_legacy_index(dir.path(), &docs);
let mut index = RawIndex::open(dir.path()).unwrap();
assert_migrated(dir.path(), &index, &docs, &keys);
assert_migrated(
dir.path(),
&RawIndex::open_readonly(dir.path()).unwrap(),
&docs,
&keys,
);
index
.add_source(
&transcript("bb"),
&[(passage("one more passage", None), unit(9))],
)
.unwrap();
index.commit().unwrap();
assert_eq!(index.len(), docs.len() + 1);
let key = index.vector_search(&unit(9), 1, None).unwrap()[0].0;
assert_eq!(
index.passage(key).unwrap().unwrap().text,
"one more passage"
);
assert!(!keys.contains(&key));
}
#[test]
fn an_old_schema_index_opened_keyed_migrates_into_sealed_text_files() {
let dir = tempfile::tempdir().unwrap();
let docs = legacy_docs(Uuid::new_v4());
let keys = write_legacy_index(dir.path(), &docs);
let sealing = seal_keys(8);
let index = RawIndex::open_keyed(dir.path(), &sealing).unwrap();
assert_migrated(dir.path(), &index, &docs, &keys);
for path in files_under(&dir.path().join(TEXT_DIR)) {
assert_eq!(
sealed_key_id(&path).unwrap(),
Some(sealing.current_id()),
"{}",
path.display()
);
}
assert!(!holds_bytes(dir.path(), "lantern"));
}
#[test]
fn a_migration_a_crash_cut_short_is_redone_or_finished_at_open() {
let before_the_journal = tempfile::tempdir().unwrap();
let docs = legacy_docs(Uuid::new_v4());
let keys = write_legacy_index(before_the_journal.path(), &docs);
let partial = before_the_journal.path().join(MIGRATED_TEXT_DIR);
std::fs::create_dir_all(&partial).unwrap();
std::fs::write(partial.join(META_FILE), "a copy cut short").unwrap();
let index = RawIndex::open(before_the_journal.path()).unwrap();
assert_migrated(before_the_journal.path(), &index, &docs, &keys);
let mid_swap = tempfile::tempdir().unwrap();
let keys = write_legacy_index(mid_swap.path(), &docs);
build_migrated_text(mid_swap.path(), None).unwrap();
sealed::write_file(
&mid_swap.path().join(MIGRATION_JOURNAL),
None,
MIGRATION_JOURNAL,
b"",
)
.unwrap();
std::fs::rename(
mid_swap.path().join(TEXT_DIR),
mid_swap.path().join(REPLACED_TEXT_DIR),
)
.unwrap();
let index = RawIndex::open(mid_swap.path()).unwrap();
assert_migrated(mid_swap.path(), &index, &docs, &keys);
}
}