1use std::collections::{BTreeMap, BTreeSet, HashSet};
2use std::path::{Path, PathBuf};
3use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
4
5use anyhow::{Context, Result, anyhow, bail, ensure};
6use chrono::{DateTime, Utc};
7use tantivy::collector::{DocSetCollector, TopDocs};
8use tantivy::directory::{Directory, MmapDirectory};
9use tantivy::merge_policy::{DefaultMergePolicy, NoMergePolicy};
10use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
11use tantivy::schema::{
12 Field, INDEXED, IndexRecordOption, STORED, STRING, Schema, TEXT, TantivyDocument, Value,
13};
14use tantivy::{
15 DocAddress, DocSet, FutureResult, Index, IndexReader, IndexSettings, IndexWriter, ReloadPolicy,
16 Searcher, SegmentMeta, TERMINATED, Term, doc,
17};
18use usearch::{IndexOptions, MetricKind, ScalarKind};
19use uuid::Uuid;
20
21use super::vector::{EMBEDDING_DIM, VectorIndex};
22use crate::sealed::{self, SealKeys, SealedDirectory};
23
24const SEGMENT_CAPACITY: u64 = 100_000;
25const WRITER_HEAP_BYTES: usize = 15_000_000;
26const OVERFETCH: usize = 4;
27const TEXT_DIR: &str = "text";
28const MIGRATED_TEXT_DIR: &str = "text.migrated";
29const REPLACED_TEXT_DIR: &str = "text.replaced";
30const MIGRATION_JOURNAL: &str = "text-migration.pending";
31const REWRITE_JOURNAL: &str = "rewrite.pending";
32const META_FILE: &str = "meta.json";
33
34pub fn dir_for_db(db_path: &Path) -> PathBuf {
36 db_path.with_extension("raw-index")
37}
38
39pub fn text_hash(text: &str) -> String {
42 use sha2::{Digest, Sha256};
43 let digest = Sha256::digest(text.trim().as_bytes());
44 digest.iter().take(16).map(|b| format!("{b:02x}")).collect()
45}
46
47#[derive(Debug, Clone, PartialEq, Eq)]
49pub enum RawOrigin {
50 Transcript { sha256: String },
52 Memory { id: Uuid },
54}
55
56impl RawOrigin {
57 pub fn source_key(&self) -> String {
59 match self {
60 Self::Transcript { sha256 } => format!("t:{sha256}"),
61 Self::Memory { id } => format!("m:{id}"),
62 }
63 }
64
65 fn parse(key: &str) -> Option<Self> {
66 match key.split_once(':')? {
67 ("t", sha256) => Some(Self::Transcript {
68 sha256: sha256.to_string(),
69 }),
70 ("m", id) => Uuid::parse_str(id).ok().map(|id| Self::Memory { id }),
71 _ => None,
72 }
73 }
74}
75
76#[derive(Debug, Clone)]
77pub struct RawPassage {
78 pub text: String,
79 pub at: Option<DateTime<Utc>>,
80 pub project: Option<String>,
81 pub memory: Option<Uuid>,
84}
85
86#[derive(Debug, Clone)]
87pub struct RawHit {
88 pub key: u64,
89 pub text: String,
92 pub text_hash: String,
94 pub at: Option<DateTime<Utc>>,
95 pub project: Option<String>,
96 pub memory: Option<Uuid>,
97 pub origin: RawOrigin,
98}
99
100struct Fields {
101 key: Field,
102 body: Field,
103 text: Field,
104 text_hash: Field,
105 source: Field,
106 project: Field,
107 memory: Field,
108 at: Field,
109}
110
111type AtRestKeys = Arc<Mutex<Option<SealKeys>>>;
115
116fn lock_keys(keys: &AtRestKeys) -> MutexGuard<'_, Option<SealKeys>> {
117 keys.lock().unwrap_or_else(PoisonError::into_inner)
118}
119
120pub struct RawIndex {
127 dir: PathBuf,
128 full_segments: Vec<(u64, VectorIndex)>,
129 active: Option<VectorIndex>,
130 active_segment: u64,
131 next_ordinal: u64,
132 segment_capacity: u64,
133 index: Index,
134 reader: IndexReader,
135 writer: Option<IndexWriter>,
136 fields: Fields,
137 writable: bool,
138 text_bytes: u64,
139 pending_texts: HashSet<String>,
140 pending_sources: HashSet<String>,
141 purge_generation: u64,
142 at_rest: AtRestKeys,
143 text_at_rest: Option<SealedDirectory>,
144}
145
146pub struct SealedRewrite {
152 dir: PathBuf,
153 keys: BTreeSet<u64>,
154 at_rest: AtRestKeys,
155}
156
157pub struct Compaction {
160 merge: Option<FutureResult<Option<SegmentMeta>>>,
161}
162
163fn schema() -> (Schema, Fields) {
168 let mut builder = Schema::builder();
169 let fields = Fields {
170 key: builder.add_u64_field("key", INDEXED | STORED),
171 body: builder.add_text_field("body", TEXT),
172 text: builder.add_text_field("text", STORED),
173 text_hash: builder.add_text_field("text_hash", STRING | STORED),
174 source: builder.add_text_field("source", STRING | STORED),
175 project: builder.add_text_field("project", STRING | STORED),
176 memory: builder.add_text_field("memory", STRING | STORED),
177 at: builder.add_i64_field("at", STORED),
178 };
179 (builder.build(), fields)
180}
181
182fn legacy_schema() -> Schema {
185 let mut builder = Schema::builder();
186 builder.add_u64_field("key", INDEXED | STORED);
187 builder.add_text_field("body", TEXT | STORED);
188 builder.add_text_field("text_hash", STRING);
189 builder.add_text_field("source", STRING | STORED);
190 builder.add_text_field("project", STRING | STORED);
191 builder.add_text_field("memory", STRING | STORED);
192 builder.add_i64_field("at", STORED);
193 builder.build()
194}
195
196fn stores_text(source: &str) -> bool {
197 !matches!(RawOrigin::parse(source), Some(RawOrigin::Memory { .. }))
198}
199
200fn passage_document(
201 fields: &Fields,
202 key: u64,
203 source: &str,
204 passage: &RawPassage,
205) -> TantivyDocument {
206 let mut document = doc!(
207 fields.key => key,
208 fields.body => passage.text.as_str(),
209 fields.text_hash => text_hash(&passage.text),
210 fields.source => source,
211 );
212 if stores_text(source) {
213 document.add_text(fields.text, &passage.text);
214 }
215 if let Some(project) = &passage.project {
216 document.add_text(fields.project, project);
217 }
218 if let Some(memory) = passage.memory {
219 document.add_text(fields.memory, memory.to_string());
220 }
221 if let Some(at) = passage.at {
222 document.add_i64(fields.at, at.timestamp());
223 }
224 document
225}
226
227fn segment_name(segment: u64) -> String {
228 format!("vectors-{segment:05}.usearch")
229}
230
231fn segment_path(dir: &Path, segment: u64) -> PathBuf {
232 dir.join(segment_name(segment))
233}
234
235fn segment_files(dir: &Path) -> Result<Vec<u64>> {
236 let mut segments = Vec::new();
237 for entry in std::fs::read_dir(dir).with_context(|| format!("listing {}", dir.display()))? {
238 let name = entry?.file_name();
239 let Some(name) = name.to_str() else { continue };
240 if let Some(n) = name
241 .strip_prefix("vectors-")
242 .and_then(|rest| rest.strip_suffix(".usearch"))
243 .and_then(|n| n.parse().ok())
244 {
245 segments.push(n);
246 }
247 }
248 segments.sort_unstable();
249 Ok(segments)
250}
251
252fn open_writable_segment(path: &Path, at_rest: Option<&SealKeys>) -> Result<VectorIndex> {
253 match at_rest {
254 Some(keys) => VectorIndex::open_with_kind_keyed(path, ScalarKind::F16, keys),
255 None => VectorIndex::open_with_kind(path, ScalarKind::F16),
256 }
257}
258
259fn open_full_segment(path: &Path, at_rest: Option<&SealKeys>) -> Result<VectorIndex> {
260 match at_rest {
261 Some(keys) => VectorIndex::open_loaded_with_kind(path, ScalarKind::F16, keys),
262 None => VectorIndex::open_view_with_kind(path, ScalarKind::F16),
263 }
264}
265
266fn is_scratch_file(name: &str) -> bool {
269 sealed::is_tmp_name(name) || name.ends_with(".tmp") || name.ends_with(".rebuilt.usearch")
270}
271
272fn top_level_files(dir: &Path) -> Result<Vec<String>> {
274 let mut names = Vec::new();
275 for entry in std::fs::read_dir(dir).with_context(|| format!("listing {}", dir.display()))? {
276 let entry = entry?;
277 if entry.file_type()?.is_file()
278 && let Some(name) = entry.file_name().to_str()
279 {
280 names.push(name.to_string());
281 }
282 }
283 Ok(names)
284}
285
286fn remove_scratch_files(dir: &Path) -> Result<()> {
287 for name in top_level_files(dir)? {
288 if is_scratch_file(&name) {
289 let path = dir.join(&name);
290 std::fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?;
291 }
292 }
293 Ok(())
294}
295
296fn dir_bytes(dir: &Path) -> u64 {
297 std::fs::read_dir(dir)
298 .map(|entries| {
299 entries
300 .filter_map(|e| e.ok()?.metadata().ok())
301 .map(|m| m.len())
302 .sum()
303 })
304 .unwrap_or(0)
305}
306
307fn sync_dir(dir: &Path) -> Result<()> {
308 std::fs::File::open(dir)
309 .and_then(|d| d.sync_all())
310 .with_context(|| format!("syncing {}", dir.display()))
311}
312
313fn holds_live(searcher: &Searcher, term: Term) -> Result<bool> {
316 for segment in searcher.segment_readers() {
317 let Some(mut postings) = segment
318 .inverted_index(term.field())?
319 .read_postings(&term, IndexRecordOption::Basic)?
320 else {
321 continue;
322 };
323 let mut doc = postings.doc();
324 while doc != TERMINATED {
325 if !segment.is_deleted(doc) {
326 return Ok(true);
327 }
328 doc = postings.advance();
329 }
330 }
331 Ok(false)
332}
333
334fn read_journal(dir: &Path, at_rest: Option<&SealKeys>) -> Result<BTreeSet<u64>> {
335 let path = dir.join(REWRITE_JOURNAL);
336 if !path.exists() {
337 return Ok(BTreeSet::new());
338 }
339 let bytes = sealed::read_file(&path, at_rest, REWRITE_JOURNAL)?;
340 let text = String::from_utf8(bytes).with_context(|| format!("reading {}", path.display()))?;
341 text.lines()
342 .map(str::trim)
343 .filter(|line| !line.is_empty())
344 .map(|line| {
345 line.parse::<u64>()
346 .with_context(|| format!("reading {}", path.display()))
347 })
348 .collect()
349}
350
351fn write_journal(dir: &Path, keys: &BTreeSet<u64>, at_rest: Option<&SealKeys>) -> Result<()> {
352 let body: String = keys.iter().map(|k| format!("{k}\n")).collect();
353 sealed::write_file(
354 &dir.join(REWRITE_JOURNAL),
355 at_rest,
356 REWRITE_JOURNAL,
357 body.as_bytes(),
358 )
359}
360
361fn segment_options() -> IndexOptions {
362 IndexOptions {
363 dimensions: EMBEDDING_DIM,
364 metric: MetricKind::Cos,
365 quantization: ScalarKind::F16,
366 ..IndexOptions::default()
367 }
368}
369
370fn segment_vectors(path: &Path, name: &str, at_rest: Option<&SealKeys>) -> Result<usearch::Index> {
373 let index = usearch::Index::new(&segment_options())
374 .map_err(|e| anyhow!("creating usearch index: {e}"))?;
375 let kind = match at_rest {
376 Some(keys) => {
377 let bytes = sealed::read_file(path, Some(keys), name)?;
378 let kind = usearch::Index::metadata_from_buffer(&bytes)
379 .map_err(|e| anyhow!("reading usearch header at {}: {e}", path.display()))?
380 .quantization;
381 if kind == ScalarKind::F16 {
382 index
383 .load_from_buffer(&bytes)
384 .map_err(|e| anyhow!("loading usearch index from {}: {e}", path.display()))?;
385 }
386 Some(kind)
387 }
388 None => {
389 let kind = VectorIndex::on_disk_kind(path)?;
390 if kind == Some(ScalarKind::F16) {
391 let path_str = path.to_str().context("vector index path is not utf-8")?;
392 index
393 .view(path_str)
394 .map_err(|e| anyhow!("viewing usearch index at {path_str}: {e}"))?;
395 }
396 kind
397 }
398 };
399 if kind != Some(ScalarKind::F16) {
400 bail!("raw vector segment {} is not F16", path.display());
401 }
402 Ok(index)
403}
404
405fn rewrite_segment(
410 dir: &Path,
411 segment: u64,
412 keys: &HashSet<u64>,
413 at_rest: Option<&SealKeys>,
414) -> Result<()> {
415 let path = segment_path(dir, segment);
416 if !path.exists() {
417 return Ok(());
418 }
419 let name = segment_name(segment);
420 let source = segment_vectors(&path, &name, at_rest)?;
421 let live = source.size();
422 let rebuilt = usearch::Index::new(&segment_options())
423 .map_err(|e| anyhow!("creating usearch index: {e}"))?;
424 rebuilt
425 .reserve(live.max(1))
426 .map_err(|e| anyhow!("reserving usearch capacity: {e}"))?;
427 let mut seen = 0;
428 let mut vector = vec![0.0f32; EMBEDDING_DIM];
429 for ordinal in 0..SEGMENT_CAPACITY {
430 if seen == live {
431 break;
432 }
433 let key = (segment << 32) | ordinal;
434 if !source.contains(key) {
435 continue;
436 }
437 seen += 1;
438 if keys.contains(&key) {
439 continue;
440 }
441 source
442 .get(key, vector.as_mut_slice())
443 .map_err(|e| anyhow!("reading key {key} from {}: {e}", path.display()))?;
444 rebuilt
445 .add(key, vector.as_slice())
446 .map_err(|e| anyhow!("adding key {key}: {e}"))?;
447 }
448 if seen < live {
449 bail!(
450 "raw vector segment {} holds keys outside its range",
451 path.display()
452 );
453 }
454 let mut bytes = vec![0u8; rebuilt.serialized_length()];
455 rebuilt
456 .save_to_buffer(&mut bytes)
457 .map_err(|e| anyhow!("serializing usearch index: {e}"))?;
458 drop(rebuilt);
459 drop(source);
460 sealed::write_file(&path, at_rest, &name, &bytes)
461}
462
463fn open_text_dir(
467 text_dir: &Path,
468 at_rest: Option<&SealKeys>,
469 create: bool,
470) -> Result<(Index, Option<SealedDirectory>)> {
471 let meta = text_dir.join(META_FILE);
472 if meta.exists() {
473 sealed::read_file(&meta, at_rest, META_FILE).context("opening raw text index")?;
474 }
475 let (directory, text_at_rest): (Box<dyn Directory>, _) = match at_rest {
476 Some(keys) => {
477 let sealed = SealedDirectory::open(text_dir, keys.clone())?;
478 if create {
479 sealed.remove_leftover_tmp_files()?;
480 }
481 (Box::new(sealed.clone()), Some(sealed))
482 }
483 None => (
484 Box::new(
485 MmapDirectory::open(text_dir)
486 .with_context(|| format!("opening raw text index in {}", text_dir.display()))?,
487 ),
488 None,
489 ),
490 };
491 let index = if create && !Index::exists(directory.as_ref())? {
492 Index::create(directory, schema().0, IndexSettings::default())
493 } else {
494 Index::open(directory)
495 }
496 .context("opening raw text index")?;
497 Ok((index, text_at_rest))
498}
499
500fn open_text(
503 dir: &Path,
504 at_rest: Option<&SealKeys>,
505 writable: bool,
506) -> Result<(Index, Option<SealedDirectory>)> {
507 let text_dir = dir.join(TEXT_DIR);
508 let opened = open_text_dir(&text_dir, at_rest, writable)?;
509 let found = opened.0.schema();
510 if found == schema().0 {
511 return Ok(opened);
512 }
513 if found != legacy_schema() {
514 bail!("raw text index in {} has an unknown schema", dir.display());
515 }
516 if !writable {
517 bail!(
518 "raw text index in {} predates the current schema; open it for writing to migrate it",
519 dir.display()
520 );
521 }
522 drop(opened);
523 migrate_text(dir, at_rest)?;
524 let opened = open_text_dir(&text_dir, at_rest, true)?;
525 ensure!(
526 opened.0.schema() == schema().0,
527 "raw text index in {} did not migrate",
528 dir.display()
529 );
530 Ok(opened)
531}
532
533fn migrate_text(dir: &Path, at_rest: Option<&SealKeys>) -> Result<()> {
534 build_migrated_text(dir, at_rest)?;
535 sealed::write_file(
536 &dir.join(MIGRATION_JOURNAL),
537 at_rest,
538 MIGRATION_JOURNAL,
539 b"",
540 )?;
541 finish_text_swap(dir)
542}
543
544fn build_migrated_text(dir: &Path, at_rest: Option<&SealKeys>) -> Result<()> {
548 let migrated_dir = dir.join(MIGRATED_TEXT_DIR);
549 if migrated_dir.exists() {
550 std::fs::remove_dir_all(&migrated_dir)
551 .with_context(|| format!("removing {}", migrated_dir.display()))?;
552 }
553 std::fs::create_dir_all(&migrated_dir)
554 .with_context(|| format!("creating {}", migrated_dir.display()))?;
555 let (legacy, _) = open_text_dir(&dir.join(TEXT_DIR), at_rest, false)?;
556 let (migrated, _) = open_text_dir(&migrated_dir, at_rest, true)?;
557 let legacy_schema = legacy.schema();
558 let legacy_field = |name: &str| {
559 legacy_schema
560 .get_field(name)
561 .with_context(|| format!("the legacy raw text index has no {name} field"))
562 };
563 let (key, body, source, project, memory, at) = (
564 legacy_field("key")?,
565 legacy_field("body")?,
566 legacy_field("source")?,
567 legacy_field("project")?,
568 legacy_field("memory")?,
569 legacy_field("at")?,
570 );
571 let (_, fields) = schema();
572 let reader: IndexReader = legacy
573 .reader_builder()
574 .reload_policy(ReloadPolicy::Manual)
575 .try_into()
576 .context("creating legacy raw text reader")?;
577 let searcher = reader.searcher();
578 let mut writer: IndexWriter = migrated
579 .writer_with_num_threads(1, WRITER_HEAP_BYTES)
580 .context("creating migrated raw text writer")?;
581 let mut copied = 0u64;
582 for (ord, segment) in searcher.segment_readers().iter().enumerate() {
583 for doc_id in segment.doc_ids_alive() {
584 let old: TantivyDocument = searcher.doc(DocAddress::new(ord as u32, doc_id))?;
585 let text = |field| {
586 old.get_first(field)
587 .and_then(|v| v.as_str())
588 .map(str::to_string)
589 };
590 let source_key = text(source).context("a raw text document has no source")?;
591 let document = match old.get_first(key).and_then(|v| v.as_u64()) {
592 Some(key) => passage_document(
593 &fields,
594 key,
595 &source_key,
596 &RawPassage {
597 text: text(body).unwrap_or_default(),
598 at: old
599 .get_first(at)
600 .and_then(|v| v.as_i64())
601 .and_then(|s| DateTime::from_timestamp(s, 0)),
602 project: text(project),
603 memory: text(memory).and_then(|m| Uuid::parse_str(&m).ok()),
604 },
605 ),
606 None => doc!(fields.source => source_key.as_str()),
607 };
608 writer.add_document(document)?;
609 copied += 1;
610 }
611 }
612 writer.commit().context("migrated raw text commit")?;
613 writer
614 .wait_merging_threads()
615 .context("waiting for migrated raw text merges")?;
616 let reader: IndexReader = migrated
617 .reader_builder()
618 .reload_policy(ReloadPolicy::Manual)
619 .try_into()
620 .context("creating migrated raw text reader")?;
621 let landed = reader.searcher().num_docs();
622 ensure!(
623 landed == copied && copied == searcher.num_docs(),
624 "raw text migration copied {copied} of {} documents and {landed} landed",
625 searcher.num_docs()
626 );
627 Ok(())
628}
629
630fn finish_text_swap(dir: &Path) -> Result<()> {
634 let text = dir.join(TEXT_DIR);
635 let migrated = dir.join(MIGRATED_TEXT_DIR);
636 let replaced = dir.join(REPLACED_TEXT_DIR);
637 let journal = dir.join(MIGRATION_JOURNAL);
638 let rename = |from: &Path, to: &Path| {
639 std::fs::rename(from, to)
640 .with_context(|| format!("renaming {} to {}", from.display(), to.display()))
641 };
642 if journal.exists() {
643 if migrated.exists() {
644 if text.exists() {
645 if replaced.exists() {
646 std::fs::remove_dir_all(&replaced)
647 .with_context(|| format!("removing {}", replaced.display()))?;
648 }
649 rename(&text, &replaced)?;
650 }
651 rename(&migrated, &text)?;
652 sync_dir(dir)?;
653 }
654 } else if migrated.exists() {
655 std::fs::remove_dir_all(&migrated)
656 .with_context(|| format!("removing {}", migrated.display()))?;
657 }
658 if replaced.exists() {
659 std::fs::remove_dir_all(&replaced)
660 .with_context(|| format!("removing {}", replaced.display()))?;
661 }
662 if journal.exists() {
663 std::fs::remove_file(&journal)
664 .with_context(|| format!("removing {}", journal.display()))?;
665 sync_dir(dir)?;
666 }
667 Ok(())
668}
669
670impl SealedRewrite {
671 pub fn is_empty(&self) -> bool {
672 self.keys.is_empty()
673 }
674
675 pub fn run(&self) -> Result<()> {
680 if self.keys.is_empty() {
681 return Ok(());
682 }
683 let at_rest = lock_keys(&self.at_rest);
684 let mut by_segment: BTreeMap<u64, HashSet<u64>> = BTreeMap::new();
685 for key in &self.keys {
686 by_segment.entry(key >> 32).or_default().insert(*key);
687 }
688 for (segment, keys) in &by_segment {
689 rewrite_segment(&self.dir, *segment, keys, at_rest.as_ref())?;
690 }
691 let journal = self.dir.join(REWRITE_JOURNAL);
692 if journal.exists() {
693 std::fs::remove_file(&journal)
694 .with_context(|| format!("removing {}", journal.display()))?;
695 }
696 Ok(())
697 }
698}
699
700impl Compaction {
701 pub fn wait(self) -> Result<()> {
702 match self.merge {
703 Some(merge) => merge.wait().map(|_| ()).context("raw text merge"),
704 None => Ok(()),
705 }
706 }
707}
708
709impl RawIndex {
710 pub fn open(dir: &Path) -> Result<Self> {
714 Self::open_writable(dir, None)
715 }
716
717 pub fn open_keyed(dir: &Path, keys: &SealKeys) -> Result<Self> {
721 Self::open_writable(dir, Some(keys))
722 }
723
724 fn open_writable(dir: &Path, at_rest: Option<&SealKeys>) -> Result<Self> {
725 finish_text_swap(dir)?;
726 std::fs::create_dir_all(dir.join(TEXT_DIR))
727 .with_context(|| format!("creating raw index dir {}", dir.display()))?;
728 remove_scratch_files(dir)?;
729 let keys: AtRestKeys = Arc::new(Mutex::new(at_rest.cloned()));
730 SealedRewrite {
731 dir: dir.to_path_buf(),
732 keys: read_journal(dir, at_rest)?,
733 at_rest: keys.clone(),
734 }
735 .run()?;
736 let segments = segment_files(dir)?;
737 let mut full_segments = Vec::new();
738 let mut active = None;
739 for (i, n) in segments.iter().enumerate() {
740 let path = segment_path(dir, *n);
741 if i + 1 == segments.len() {
742 let loaded = open_writable_segment(&path, at_rest)?;
743 if (loaded.len() as u64) < SEGMENT_CAPACITY {
744 active = Some((*n, loaded));
745 continue;
746 }
747 }
748 full_segments.push((*n, open_full_segment(&path, at_rest)?));
749 }
750 let (segment, active) = match active {
751 Some(found) => found,
752 None => {
753 let next = segments.last().map_or(0, |n| n + 1);
754 (
755 next,
756 open_writable_segment(&segment_path(dir, next), at_rest)?,
757 )
758 }
759 };
760 let mut index = Self::with_text(dir, full_segments, Some(active), keys, true)?;
761 index.active_segment = segment;
762 index.next_ordinal = index.ordinal_after(segment)?;
763 Ok(index)
764 }
765
766 pub fn open_readonly(dir: &Path) -> Result<Self> {
769 Self::open_search_only(dir, None)
770 }
771
772 pub fn open_readonly_keyed(dir: &Path, keys: &SealKeys) -> Result<Self> {
775 Self::open_search_only(dir, Some(keys))
776 }
777
778 fn open_search_only(dir: &Path, at_rest: Option<&SealKeys>) -> Result<Self> {
779 let full_segments = segment_files(dir)?
780 .into_iter()
781 .map(|n| open_full_segment(&segment_path(dir, n), at_rest).map(|s| (n, s)))
782 .collect::<Result<Vec<_>>>()?;
783 let keys = Arc::new(Mutex::new(at_rest.cloned()));
784 Self::with_text(dir, full_segments, None, keys, false)
785 }
786
787 fn with_text(
788 dir: &Path,
789 full_segments: Vec<(u64, VectorIndex)>,
790 active: Option<VectorIndex>,
791 at_rest: AtRestKeys,
792 writable: bool,
793 ) -> Result<Self> {
794 let keys = lock_keys(&at_rest).clone();
795 let (index, text_at_rest) = open_text(dir, keys.as_ref(), writable)?;
796 let (_, fields) = schema();
797 let reader = index
798 .reader_builder()
799 .reload_policy(ReloadPolicy::Manual)
800 .try_into()
801 .context("creating raw text reader")?;
802 Ok(Self {
803 dir: dir.to_path_buf(),
804 full_segments,
805 active,
806 active_segment: 0,
807 next_ordinal: 0,
808 segment_capacity: SEGMENT_CAPACITY,
809 index,
810 reader,
811 writer: None,
812 fields,
813 writable,
814 text_bytes: dir_bytes(&dir.join(TEXT_DIR)),
815 pending_texts: HashSet::new(),
816 pending_sources: HashSet::new(),
817 purge_generation: 0,
818 at_rest,
819 text_at_rest,
820 })
821 }
822
823 pub fn seal_keys(&self) -> Option<SealKeys> {
826 lock_keys(&self.at_rest).clone()
827 }
828
829 pub fn set_seal_keys(&mut self, keys: SealKeys) -> Result<()> {
834 let Some(text) = &self.text_at_rest else {
835 return self.reopen_keyed(&keys);
836 };
837 text.set_keys(keys.clone());
838 if let Some(active) = self.active.as_mut() {
839 active.set_seal_keys(keys.clone());
840 }
841 for (_, segment) in &mut self.full_segments {
842 segment.set_seal_keys(keys.clone());
843 }
844 *lock_keys(&self.at_rest) = Some(keys);
845 Ok(())
846 }
847
848 fn reopen_keyed(&mut self, keys: &SealKeys) -> Result<()> {
849 if !self.writable {
850 bail!(
851 "raw index at {} was opened read-only without keys; reopen it keyed to seal it",
852 self.dir.display()
853 );
854 }
855 let shared = self.at_rest.clone();
856 let mut held = lock_keys(&shared);
857 self.commit()?;
858 if let Some(writer) = self.writer.take() {
859 writer
860 .wait_merging_threads()
861 .context("waiting for raw text merges")?;
862 }
863 let generation = self.purge_generation + 1;
864 let mut reopened = Self::open_keyed(&self.dir.clone(), keys)?;
865 *held = Some(keys.clone());
866 reopened.at_rest = shared.clone();
867 reopened.purge_generation = generation;
868 drop(held);
869 *self = reopened;
870 Ok(())
871 }
872
873 pub fn reseal(&mut self) -> Result<usize> {
878 self.require_writable()?;
879 let Some(text) = self.text_at_rest.clone() else {
880 bail!(
881 "raw index at {} was opened without keys; reopen it keyed to seal it",
882 self.dir.display()
883 );
884 };
885 self.commit()?;
886 if let Some(writer) = self.writer.take() {
887 writer
888 .wait_merging_threads()
889 .context("waiting for raw text merges")?;
890 }
891 let mut rewritten = text.reseal_all()?;
892 let at_rest = self.at_rest.clone();
893 let keys = lock_keys(&at_rest);
894 let keys = keys
895 .as_ref()
896 .context("a raw index with a sealed text directory holds its keys")?;
897 for name in top_level_files(&self.dir)? {
898 let path = self.dir.join(&name);
899 if is_scratch_file(&name) {
900 std::fs::remove_file(&path)
901 .with_context(|| format!("removing {}", path.display()))?;
902 } else if sealed::reseal_file(&path, keys, &name)? {
903 rewritten += 1;
904 }
905 }
906 Ok(rewritten)
907 }
908
909 fn ordinal_after(&self, segment: u64) -> Result<u64> {
913 let mut next = self.active.as_ref().map_or(0, |a| a.len() as u64);
914 let low = (segment << 32).to_be_bytes();
915 let high = ((segment + 1) << 32).to_be_bytes();
916 for reader in self.reader.searcher().segment_readers() {
917 let inverted = reader.inverted_index(self.fields.key)?;
918 let mut terms = inverted.terms().range().ge(low).lt(high).into_stream()?;
919 while terms.advance() {
920 if let Ok(bytes) = <[u8; 8]>::try_from(terms.key()) {
921 next = next.max((u64::from_be_bytes(bytes) & u64::from(u32::MAX)) + 1);
922 }
923 }
924 }
925 Ok(next)
926 }
927
928 fn segments(&self) -> impl Iterator<Item = &VectorIndex> {
929 self.full_segments
930 .iter()
931 .map(|(_, segment)| segment)
932 .chain(self.active.iter())
933 }
934
935 pub fn len(&self) -> usize {
936 self.segments().map(VectorIndex::len).sum()
937 }
938
939 pub fn is_empty(&self) -> bool {
940 self.len() == 0
941 }
942
943 pub fn resident_bytes(&self) -> u64 {
947 let vectors: usize = self.segments().map(VectorIndex::memory_usage).sum();
948 vectors as u64 + self.text_bytes
949 }
950
951 pub fn has_text(&self, text: &str) -> bool {
953 let hash = text_hash(text);
954 if self.pending_texts.contains(&hash) {
955 return true;
956 }
957 let term = Term::from_field_text(self.fields.text_hash, &hash);
958 holds_live(&self.reader.searcher(), term).unwrap_or(false)
959 }
960
961 pub fn has_source(&self, source: &RawOrigin) -> bool {
963 let key = source.source_key();
964 if self.pending_sources.contains(&key) {
965 return true;
966 }
967 let term = Term::from_field_text(self.fields.source, &key);
968 holds_live(&self.reader.searcher(), term).unwrap_or(false)
969 }
970
971 pub fn purge_generation(&self) -> u64 {
975 self.purge_generation
976 }
977
978 pub fn add_source(
983 &mut self,
984 origin: &RawOrigin,
985 passages: &[(RawPassage, Vec<f32>)],
986 ) -> Result<usize> {
987 self.require_writable()?;
988 let source = origin.source_key();
989 let searcher = self.reader.searcher();
990 let mut added = 0;
991 for (passage, embedding) in passages {
992 if embedding.len() != EMBEDDING_DIM || passage.text.trim().is_empty() {
993 continue;
994 }
995 let hash = text_hash(&passage.text);
996 let term = Term::from_field_text(self.fields.text_hash, &hash);
997 if self.pending_texts.contains(&hash) || holds_live(&searcher, term)? {
998 continue;
999 }
1000 self.pending_texts.insert(hash);
1001 let key = self.next_key()?;
1002 self.active_mut()?.upsert(key, embedding)?;
1003 let document = passage_document(&self.fields, key, &source, passage);
1004 self.writer_mut()?.add_document(document)?;
1005 added += 1;
1006 }
1007 self.pending_sources.insert(source.clone());
1008 if added == 0 {
1009 let marker = doc!(self.fields.source => source.as_str());
1010 self.writer_mut()?.add_document(marker)?;
1011 }
1012 Ok(added)
1013 }
1014
1015 pub fn commit(&mut self) -> Result<()> {
1019 self.require_writable()?;
1020 if let Some(active) = &self.active {
1021 active.save()?;
1022 }
1023 if let Some(writer) = self.writer.as_mut() {
1024 writer.commit().context("raw text commit")?;
1025 }
1026 self.reader.reload().context("raw text reader reload")?;
1027 self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
1028 self.pending_texts.clear();
1029 self.pending_sources.clear();
1030 Ok(())
1031 }
1032
1033 pub fn commit_without_merging(&mut self) -> Result<()> {
1036 self.require_writable()?;
1037 self.writer_mut()?.set_merge_policy(Box::new(NoMergePolicy));
1038 let committed = self.commit();
1039 if let Some(writer) = &self.writer {
1040 writer.set_merge_policy(Box::new(DefaultMergePolicy::default()));
1041 }
1042 committed
1043 }
1044
1045 pub fn reload(&mut self) -> Result<()> {
1047 if self.writable {
1048 return Ok(());
1049 }
1050 let at_rest = self.seal_keys();
1051 let segments = segment_files(&self.dir)?;
1052 let mut full_segments = Vec::with_capacity(segments.len());
1053 for n in segments {
1054 full_segments.push((
1055 n,
1056 open_full_segment(&segment_path(&self.dir, n), at_rest.as_ref())?,
1057 ));
1058 }
1059 self.full_segments = full_segments;
1060 self.reader.reload().context("raw text reader reload")?;
1061 self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
1062 Ok(())
1063 }
1064
1065 pub fn vector_search(
1068 &self,
1069 embedding: &[f32],
1070 k: usize,
1071 project: Option<&str>,
1072 ) -> Result<Vec<(u64, f32)>> {
1073 let fetch = if project.is_some() { k * OVERFETCH } else { k };
1074 let mut hits = Vec::new();
1075 for segment in self.segments() {
1076 if segment.len() > 0 {
1077 hits.extend(segment.search(embedding, fetch)?);
1078 }
1079 }
1080 hits.sort_by(|a, b| b.1.total_cmp(&a.1));
1081 let mut out = Vec::with_capacity(k);
1082 for (key, score) in hits {
1083 if out.len() == k {
1084 break;
1085 }
1086 let Some(hit) = self.passage(key)? else {
1087 continue;
1088 };
1089 if project.is_none() || hit.project.as_deref() == project {
1090 out.push((key, score));
1091 }
1092 }
1093 Ok(out)
1094 }
1095
1096 pub fn bm25_search(
1099 &self,
1100 query_text: &str,
1101 k: usize,
1102 project: Option<&str>,
1103 ) -> Result<Vec<(u64, f32)>> {
1104 let searcher = self.reader.searcher();
1105 let parser = QueryParser::for_index(&self.index, vec![self.fields.body]);
1106 let (text_query, _errors) = parser.parse_query_lenient(query_text);
1107 let query: Box<dyn Query> = match project {
1108 Some(project) => Box::new(BooleanQuery::new(vec![
1109 (Occur::Must, text_query),
1110 (
1111 Occur::Must,
1112 Box::new(TermQuery::new(
1113 Term::from_field_text(self.fields.project, project),
1114 IndexRecordOption::Basic,
1115 )),
1116 ),
1117 ])),
1118 None => text_query,
1119 };
1120 let top = searcher.search(&query, &TopDocs::with_limit(k.max(1)).order_by_score())?;
1121 let mut out = Vec::with_capacity(top.len());
1122 for (score, address) in top {
1123 let document: TantivyDocument = searcher.doc(address)?;
1124 if let Some(key) = document.get_first(self.fields.key).and_then(|v| v.as_u64()) {
1125 out.push((key, score));
1126 }
1127 }
1128 Ok(out)
1129 }
1130
1131 pub fn passage(&self, key: u64) -> Result<Option<RawHit>> {
1132 let searcher = self.reader.searcher();
1133 let query = TermQuery::new(
1134 Term::from_field_u64(self.fields.key, key),
1135 IndexRecordOption::Basic,
1136 );
1137 let Some((_, address)) = searcher
1138 .search(&query, &TopDocs::with_limit(1).order_by_score())?
1139 .into_iter()
1140 .next()
1141 else {
1142 return Ok(None);
1143 };
1144 let document: TantivyDocument = searcher.doc(address)?;
1145 Ok(self.hit(&document))
1146 }
1147
1148 fn hit(&self, document: &TantivyDocument) -> Option<RawHit> {
1149 let text = |field| {
1150 document
1151 .get_first(field)
1152 .and_then(|v| v.as_str())
1153 .map(str::to_string)
1154 };
1155 let key = document
1156 .get_first(self.fields.key)
1157 .and_then(|v| v.as_u64())?;
1158 let origin = text(self.fields.source)
1159 .as_deref()
1160 .and_then(RawOrigin::parse)?;
1161 Some(RawHit {
1162 key,
1163 text: text(self.fields.text).unwrap_or_default(),
1164 text_hash: text(self.fields.text_hash).unwrap_or_default(),
1165 at: document
1166 .get_first(self.fields.at)
1167 .and_then(|v| v.as_i64())
1168 .and_then(|s| DateTime::from_timestamp(s, 0)),
1169 project: text(self.fields.project),
1170 memory: text(self.fields.memory).and_then(|m| Uuid::parse_str(&m).ok()),
1171 origin,
1172 })
1173 }
1174
1175 pub fn sources(&self) -> Result<Vec<RawOrigin>> {
1178 Ok(self
1179 .terms(self.fields.source)?
1180 .iter()
1181 .filter_map(|key| RawOrigin::parse(key))
1182 .collect())
1183 }
1184
1185 pub fn passage_memories(&self) -> Result<Vec<Uuid>> {
1187 Ok(self
1188 .terms(self.fields.memory)?
1189 .iter()
1190 .filter_map(|id| Uuid::parse_str(id).ok())
1191 .collect())
1192 }
1193
1194 pub fn matching(
1197 &self,
1198 sources: &[RawOrigin],
1199 memories: &[Uuid],
1200 texts: &[&str],
1201 ) -> Result<Vec<RawHit>> {
1202 let hashes: BTreeSet<String> = texts.iter().map(|t| text_hash(t)).collect();
1203 let terms = sources
1204 .iter()
1205 .map(|s| Term::from_field_text(self.fields.source, &s.source_key()))
1206 .chain(
1207 memories
1208 .iter()
1209 .map(|m| Term::from_field_text(self.fields.memory, &m.to_string())),
1210 )
1211 .chain(
1212 hashes
1213 .iter()
1214 .map(|h| Term::from_field_text(self.fields.text_hash, h)),
1215 );
1216 let searcher = self.reader.searcher();
1217 let mut seen = HashSet::new();
1218 let mut hits = Vec::new();
1219 for term in terms {
1220 let query = TermQuery::new(term, IndexRecordOption::Basic);
1221 for address in searcher.search(&query, &DocSetCollector)? {
1222 let document: TantivyDocument = searcher.doc(address)?;
1223 if let Some(hit) = self.hit(&document)
1224 && seen.insert(hit.key)
1225 {
1226 hits.push(hit);
1227 }
1228 }
1229 }
1230 Ok(hits)
1231 }
1232
1233 pub fn withdraw(&mut self, keys: &[u64]) -> Result<()> {
1237 self.require_writable()?;
1238 if keys.is_empty() {
1239 return Ok(());
1240 }
1241 self.purge_generation += 1;
1242 let active_segment = self.active_segment;
1243 if let Some(active) = self.active.as_mut() {
1244 for key in keys.iter().filter(|k| *k >> 32 == active_segment) {
1245 active.remove(*key)?;
1246 }
1247 }
1248 let field = self.fields.key;
1249 let writer = self.writer_mut()?;
1250 for key in keys {
1251 writer.delete_term(Term::from_field_u64(field, *key));
1252 }
1253 Ok(())
1254 }
1255
1256 pub fn rekey(&mut self, key: u64, origin: &RawOrigin, passage: &RawPassage) -> Result<()> {
1260 self.require_writable()?;
1261 self.purge_generation += 1;
1262 let document = passage_document(&self.fields, key, &origin.source_key(), passage);
1263 let term = Term::from_field_u64(self.fields.key, key);
1264 let writer = self.writer_mut()?;
1265 writer.delete_term(term);
1266 writer.add_document(document)?;
1267 Ok(())
1268 }
1269
1270 pub fn forget_sources(&mut self, sources: &[RawOrigin]) -> Result<()> {
1273 self.require_writable()?;
1274 let field = self.fields.source;
1275 let writer = self.writer_mut()?;
1276 for source in sources {
1277 writer.delete_term(Term::from_field_text(field, &source.source_key()));
1278 }
1279 Ok(())
1280 }
1281
1282 pub fn seal_for_rewrite(&mut self, keys: &[u64]) -> Result<SealedRewrite> {
1287 self.require_writable()?;
1288 let at_rest = self.seal_keys();
1289 let mut pending = read_journal(&self.dir, at_rest.as_ref())?;
1290 pending.extend(keys.iter().copied());
1291 if pending.iter().any(|k| k >> 32 == self.active_segment) {
1292 self.close_active()?;
1293 }
1294 if !pending.is_empty() {
1295 write_journal(&self.dir, &pending, at_rest.as_ref())?;
1296 }
1297 Ok(SealedRewrite {
1298 dir: self.dir.clone(),
1299 keys: pending,
1300 at_rest: self.at_rest.clone(),
1301 })
1302 }
1303
1304 pub fn reload_sealed(&mut self, rewrite: &SealedRewrite) -> Result<()> {
1306 let segments: HashSet<u64> = rewrite.keys.iter().map(|k| k >> 32).collect();
1307 for (n, segment) in &mut self.full_segments {
1308 if segments.contains(n) {
1309 segment.reload_view()?;
1310 }
1311 }
1312 Ok(())
1313 }
1314
1315 pub fn start_compaction(&mut self) -> Result<Compaction> {
1319 self.require_writable()?;
1320 let segments = self
1321 .index
1322 .searchable_segment_ids()
1323 .context("listing raw text segments")?;
1324 if segments.is_empty() {
1325 return Ok(Compaction { merge: None });
1326 }
1327 let merge = self.writer_mut()?.merge(&segments);
1328 Ok(Compaction { merge: Some(merge) })
1329 }
1330
1331 pub fn finish_compaction(&mut self, merged: Result<()>) -> Result<()> {
1335 self.require_writable()?;
1336 if let Err(refused) = merged {
1337 return self
1338 .compact()
1339 .with_context(|| format!("compacting after the merge failed: {refused:#}"));
1340 }
1341 if let Some(writer) = &self.writer {
1342 writer
1343 .garbage_collect_files()
1344 .wait()
1345 .context("raw text garbage collection")?;
1346 }
1347 self.reader.reload().context("raw text reader reload")?;
1348 self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
1349 Ok(())
1350 }
1351
1352 pub fn compact(&mut self) -> Result<()> {
1355 self.commit()?;
1356 if let Some(writer) = self.writer.take() {
1357 writer
1358 .wait_merging_threads()
1359 .context("waiting for raw text merges")?;
1360 }
1361 let segments = self
1362 .index
1363 .searchable_segment_ids()
1364 .context("listing raw text segments")?;
1365 let writer = self.writer_mut()?;
1366 if !segments.is_empty() {
1367 writer.merge(&segments).wait().context("raw text merge")?;
1368 }
1369 writer
1370 .garbage_collect_files()
1371 .wait()
1372 .context("raw text garbage collection")?;
1373 self.reader.reload().context("raw text reader reload")?;
1374 self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
1375 Ok(())
1376 }
1377
1378 fn terms(&self, field: Field) -> Result<Vec<String>> {
1379 let searcher = self.reader.searcher();
1380 let mut found = std::collections::BTreeSet::new();
1381 for segment in searcher.segment_readers() {
1382 let inverted = segment.inverted_index(field)?;
1383 let mut stream = inverted.terms().stream()?;
1384 while stream.advance() {
1385 found.insert(String::from_utf8_lossy(stream.key()).into_owned());
1386 }
1387 }
1388 Ok(found.into_iter().collect())
1389 }
1390
1391 fn next_key(&mut self) -> Result<u64> {
1392 if self.next_ordinal >= self.segment_capacity {
1393 self.close_active()?;
1394 }
1395 let ordinal = self.next_ordinal;
1396 self.next_ordinal += 1;
1397 Ok((self.active_segment << 32) | ordinal)
1398 }
1399
1400 fn close_active(&mut self) -> Result<()> {
1403 let active = self
1404 .active
1405 .take()
1406 .context("raw index has no active segment")?;
1407 active.save()?;
1408 drop(active);
1409 let at_rest = self.seal_keys();
1410 let path = segment_path(&self.dir, self.active_segment);
1411 self.full_segments.push((
1412 self.active_segment,
1413 open_full_segment(&path, at_rest.as_ref())?,
1414 ));
1415 self.active_segment += 1;
1416 self.next_ordinal = 0;
1417 self.active = Some(open_writable_segment(
1418 &segment_path(&self.dir, self.active_segment),
1419 at_rest.as_ref(),
1420 )?);
1421 Ok(())
1422 }
1423
1424 fn active_mut(&mut self) -> Result<&mut VectorIndex> {
1425 self.active
1426 .as_mut()
1427 .context("raw index has no active segment")
1428 }
1429
1430 fn writer_mut(&mut self) -> Result<&mut IndexWriter> {
1431 if self.writer.is_none() {
1432 self.writer = Some(
1433 self.index
1434 .writer_with_num_threads(1, WRITER_HEAP_BYTES)
1435 .context("creating raw text writer")?,
1436 );
1437 }
1438 Ok(self.writer.as_mut().expect("writer just created"))
1439 }
1440
1441 fn require_writable(&self) -> Result<()> {
1442 if self.writable {
1443 Ok(())
1444 } else {
1445 Err(anyhow!("raw index is open read-only"))
1446 }
1447 }
1448}
1449
1450#[cfg(test)]
1451mod tests {
1452 use super::*;
1453 use crate::keys::KeyGuard;
1454 use crate::sealed::{SealError, SealKey, SealKeyId, seal_error, sealed_key_id};
1455 use tantivy::{DocAddress, Document, IndexSettings};
1456
1457 fn unit(i: usize) -> Vec<f32> {
1458 let mut v = vec![0.0; EMBEDDING_DIM];
1459 v[i % EMBEDDING_DIM] = 1.0;
1460 v
1461 }
1462
1463 fn passage(text: &str, project: Option<&str>) -> RawPassage {
1464 RawPassage {
1465 text: text.to_string(),
1466 at: DateTime::from_timestamp(1_790_000_000, 0),
1467 project: project.map(str::to_string),
1468 memory: None,
1469 }
1470 }
1471
1472 fn transcript(sha: &str) -> RawOrigin {
1473 RawOrigin::Transcript {
1474 sha256: sha.to_string(),
1475 }
1476 }
1477
1478 fn purge(
1479 index: &mut RawIndex,
1480 sources: &[RawOrigin],
1481 memories: &[Uuid],
1482 texts: &[&str],
1483 ) -> usize {
1484 let keys: Vec<u64> = index
1485 .matching(sources, memories, texts)
1486 .unwrap()
1487 .iter()
1488 .map(|hit| hit.key)
1489 .collect();
1490 index.withdraw(&keys).unwrap();
1491 index.forget_sources(sources).unwrap();
1492 let rewrite = index.seal_for_rewrite(&keys).unwrap();
1493 index.commit().unwrap();
1494 rewrite.run().unwrap();
1495 index.reload_sealed(&rewrite).unwrap();
1496 keys.len()
1497 }
1498
1499 fn holds_key(index: &RawIndex, key: u64, near: &[f32]) -> bool {
1500 index
1501 .segments()
1502 .filter(|segment| segment.len() > 0)
1503 .any(|segment| {
1504 segment
1505 .search(near, 10)
1506 .unwrap()
1507 .iter()
1508 .any(|(k, _)| *k == key)
1509 })
1510 }
1511
1512 #[test]
1513 fn a_passage_keeps_the_raw_memory_it_maps_to_and_its_text_is_known_before_commit() {
1514 let dir = tempfile::tempdir().unwrap();
1515 let mut index = RawIndex::open(dir.path()).unwrap();
1516 let memory = Uuid::new_v4();
1517 let mut mapped = passage("the mirror job runs at four", None);
1518 mapped.memory = Some(memory);
1519 assert!(!index.has_text("the mirror job runs at four"));
1520
1521 index
1522 .add_source(&transcript("aa"), &[(mapped, unit(1))])
1523 .unwrap();
1524 assert!(index.has_text("the mirror job runs at four"));
1525 index.commit().unwrap();
1526
1527 assert!(index.has_text(" the mirror job runs at four \n"));
1528 assert!(!index.has_text("the mirror job runs at five"));
1529 let key = index.vector_search(&unit(1), 1, None).unwrap()[0].0;
1530 assert_eq!(index.passage(key).unwrap().unwrap().memory, Some(memory));
1531 assert!(index.resident_bytes() > 0);
1532 }
1533
1534 #[test]
1535 fn a_passage_is_found_by_vector_and_by_words_with_its_metadata() {
1536 let dir = tempfile::tempdir().unwrap();
1537 let mut index = RawIndex::open(dir.path()).unwrap();
1538 let source = transcript("aa");
1539 index
1540 .add_source(
1541 &source,
1542 &[
1543 (
1544 passage(
1545 "the garden daemon listens on port 7200",
1546 Some("-srv-garden"),
1547 ),
1548 unit(1),
1549 ),
1550 (passage("lunch was soup", Some("-srv-garden")), unit(2)),
1551 ],
1552 )
1553 .unwrap();
1554 index.commit().unwrap();
1555
1556 let by_vector = index.vector_search(&unit(1), 1, None).unwrap();
1557 let hit = index.passage(by_vector[0].0).unwrap().unwrap();
1558 assert_eq!(hit.text, "the garden daemon listens on port 7200");
1559 assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
1560 assert_eq!(hit.origin, source);
1561 assert_eq!(hit.at.unwrap().timestamp(), 1_790_000_000);
1562
1563 let by_words = index.bm25_search("daemon port", 5, None).unwrap();
1564 assert_eq!(by_words.len(), 1);
1565 assert_eq!(by_words[0].0, by_vector[0].0);
1566 }
1567
1568 #[test]
1569 fn a_project_filter_keeps_only_that_projects_passages() {
1570 let dir = tempfile::tempdir().unwrap();
1571 let mut index = RawIndex::open(dir.path()).unwrap();
1572 index
1573 .add_source(
1574 &transcript("aa"),
1575 &[
1576 (
1577 passage("deploy uses argo here", Some("-srv-garden")),
1578 unit(1),
1579 ),
1580 (
1581 passage("deploy uses argo there", Some("-srv-orchard")),
1582 unit(1),
1583 ),
1584 ],
1585 )
1586 .unwrap();
1587 index.commit().unwrap();
1588
1589 for hits in [
1590 index
1591 .vector_search(&unit(1), 5, Some("-srv-orchard"))
1592 .unwrap(),
1593 index
1594 .bm25_search("deploy argo", 5, Some("-srv-orchard"))
1595 .unwrap(),
1596 ] {
1597 assert_eq!(hits.len(), 1);
1598 let hit = index.passage(hits[0].0).unwrap().unwrap();
1599 assert_eq!(hit.project.as_deref(), Some("-srv-orchard"));
1600 }
1601 }
1602
1603 #[test]
1604 fn the_same_text_from_two_sources_is_stored_once_and_both_sources_are_recorded() {
1605 let dir = tempfile::tempdir().unwrap();
1606 let mut index = RawIndex::open(dir.path()).unwrap();
1607 let memory = RawOrigin::Memory { id: Uuid::new_v4() };
1608 let text = "the port moved to 7200";
1609 assert_eq!(
1610 index
1611 .add_source(&transcript("aa"), &[(passage(text, None), unit(1))])
1612 .unwrap(),
1613 1
1614 );
1615 index.commit().unwrap();
1616 assert_eq!(
1617 index
1618 .add_source(&memory, &[(passage(text, None), unit(1))])
1619 .unwrap(),
1620 0
1621 );
1622 index.commit().unwrap();
1623
1624 assert_eq!(index.len(), 1);
1625 assert!(index.has_source(&transcript("aa")));
1626 assert!(index.has_source(&memory));
1627 assert!(!index.has_source(&transcript("bb")));
1628 }
1629
1630 #[test]
1631 fn a_passage_without_a_full_embedding_is_skipped() {
1632 let dir = tempfile::tempdir().unwrap();
1633 let mut index = RawIndex::open(dir.path()).unwrap();
1634 let added = index
1635 .add_source(
1636 &transcript("aa"),
1637 &[(passage("short vector", None), vec![1.0; 3])],
1638 )
1639 .unwrap();
1640 index.commit().unwrap();
1641 assert_eq!(added, 0);
1642 assert!(index.is_empty());
1643 assert!(index.has_source(&transcript("aa")));
1644 }
1645
1646 #[test]
1647 fn full_segments_are_sealed_and_everything_survives_a_reopen() {
1648 let dir = tempfile::tempdir().unwrap();
1649 {
1650 let mut index = RawIndex::open(dir.path()).unwrap();
1651 index.segment_capacity = 2;
1652 let passages: Vec<_> = (0..5)
1653 .map(|i| (passage(&format!("passage number {i}"), None), unit(i)))
1654 .collect();
1655 index.add_source(&transcript("aa"), &passages).unwrap();
1656 index.commit().unwrap();
1657 assert_eq!(index.full_segments.len(), 2);
1658 }
1659 assert_eq!(segment_files(dir.path()).unwrap(), vec![0, 1, 2]);
1660
1661 let mut reopened = RawIndex::open(dir.path()).unwrap();
1662 assert_eq!(reopened.len(), 5);
1663 let hit = reopened.vector_search(&unit(3), 1, None).unwrap()[0].0;
1664 assert_eq!(
1665 reopened.passage(hit).unwrap().unwrap().text,
1666 "passage number 3"
1667 );
1668 reopened
1669 .add_source(
1670 &transcript("bb"),
1671 &[(passage("one more passage", None), unit(9))],
1672 )
1673 .unwrap();
1674 reopened.commit().unwrap();
1675
1676 let readonly = RawIndex::open_readonly(dir.path()).unwrap();
1677 assert_eq!(readonly.len(), 6);
1678 let hit = readonly.bm25_search("one more", 1, None).unwrap()[0].0;
1679 assert_eq!(
1680 readonly.passage(hit).unwrap().unwrap().text,
1681 "one more passage"
1682 );
1683 }
1684
1685 #[test]
1686 fn a_read_only_index_refuses_writes() {
1687 let dir = tempfile::tempdir().unwrap();
1688 RawIndex::open(dir.path()).unwrap().commit().unwrap();
1689 let mut readonly = RawIndex::open_readonly(dir.path()).unwrap();
1690 assert!(readonly.add_source(&transcript("aa"), &[]).is_err());
1691 }
1692
1693 #[test]
1694 fn origins_round_trip_through_their_source_key() {
1695 for origin in [transcript("ab12"), RawOrigin::Memory { id: Uuid::new_v4() }] {
1696 assert_eq!(RawOrigin::parse(&origin.source_key()), Some(origin));
1697 }
1698 assert_eq!(RawOrigin::parse("x:1"), None);
1699 }
1700
1701 #[test]
1702 fn removing_a_source_withdraws_its_passages_and_their_text_from_the_files() {
1703 let dir = tempfile::tempdir().unwrap();
1704 let mut index = RawIndex::open(dir.path()).unwrap();
1705 index
1706 .add_source(
1707 &transcript("aa"),
1708 &[
1709 (passage("the gate code sits under a stone", None), unit(1)),
1710 (
1711 passage("a second line of the same transcript", None),
1712 unit(2),
1713 ),
1714 ],
1715 )
1716 .unwrap();
1717 index
1718 .add_source(
1719 &transcript("bb"),
1720 &[(passage("an unrelated line", None), unit(3))],
1721 )
1722 .unwrap();
1723 index.commit().unwrap();
1724 assert_eq!(index.sources().unwrap().len(), 2);
1725
1726 assert_eq!(purge(&mut index, &[transcript("aa")], &[], &[]), 2);
1727 index.commit().unwrap();
1728 index.compact().unwrap();
1729
1730 assert!(
1731 index
1732 .bm25_search("gate code stone", 5, None)
1733 .unwrap()
1734 .is_empty()
1735 );
1736 assert!(
1737 index
1738 .vector_search(&unit(1), 5, None)
1739 .unwrap()
1740 .iter()
1741 .all(|(k, _)| { index.passage(*k).unwrap().unwrap().text == "an unrelated line" })
1742 );
1743 assert!(!index.has_source(&transcript("aa")));
1744 assert!(!index.has_text("the gate code sits under a stone"));
1745 assert_eq!(index.sources().unwrap(), vec![transcript("bb")]);
1746 assert_eq!(index.len(), 1);
1747 let leftovers: Vec<String> = std::fs::read_dir(dir.path().join(TEXT_DIR))
1748 .unwrap()
1749 .filter_map(|e| std::fs::read(e.ok()?.path()).ok())
1750 .map(|b| String::from_utf8_lossy(&b).into_owned())
1751 .filter(|t| t.contains("stone"))
1752 .collect();
1753 assert!(
1754 leftovers.is_empty(),
1755 "the removed text left the index files"
1756 );
1757 }
1758
1759 #[test]
1760 fn removing_a_memory_withdraws_the_transcript_passages_that_stand_for_it() {
1761 let dir = tempfile::tempdir().unwrap();
1762 let mut index = RawIndex::open(dir.path()).unwrap();
1763 let gone = Uuid::new_v4();
1764 let mut mapped = passage("passage the memory stands for", None);
1765 mapped.memory = Some(gone);
1766 index
1767 .add_source(
1768 &transcript("aa"),
1769 &[
1770 (mapped, unit(1)),
1771 (passage("a passage of nobody", None), unit(2)),
1772 ],
1773 )
1774 .unwrap();
1775 index
1776 .add_source(
1777 &RawOrigin::Memory { id: gone },
1778 &[(passage("the archived memory itself", None), unit(3))],
1779 )
1780 .unwrap();
1781 index.commit().unwrap();
1782 assert_eq!(index.passage_memories().unwrap(), vec![gone]);
1783
1784 let removed = purge(&mut index, &[RawOrigin::Memory { id: gone }], &[gone], &[]);
1785 index.commit().unwrap();
1786
1787 assert_eq!(removed, 2);
1788 assert!(index.bm25_search("memory", 5, None).unwrap().is_empty());
1789 assert_eq!(index.bm25_search("nobody", 5, None).unwrap().len(), 1);
1790 assert_eq!(index.len(), 1);
1791 }
1792
1793 #[test]
1794 fn removing_by_text_withdraws_the_passage_whichever_source_holds_it() {
1795 let dir = tempfile::tempdir().unwrap();
1796 let mut index = RawIndex::open(dir.path()).unwrap();
1797 index
1798 .add_source(
1799 &transcript("old"),
1800 &[(passage("a line the user typed", None), unit(1))],
1801 )
1802 .unwrap();
1803 index
1804 .add_source(
1805 &transcript("new"),
1806 &[(passage("a later line", None), unit(2))],
1807 )
1808 .unwrap();
1809 index.commit().unwrap();
1810
1811 let removed = purge(
1812 &mut index,
1813 &[transcript("new")],
1814 &[],
1815 &["a line the user typed", "a later line"],
1816 );
1817 index.commit().unwrap();
1818
1819 assert_eq!(removed, 2);
1820 assert!(index.bm25_search("line", 5, None).unwrap().is_empty());
1821 }
1822
1823 #[test]
1824 fn a_passage_in_a_sealed_segment_stops_resolving_once_removed() {
1825 let dir = tempfile::tempdir().unwrap();
1826 let mut index = RawIndex::open(dir.path()).unwrap();
1827 index.segment_capacity = 1;
1828 index
1829 .add_source(
1830 &transcript("aa"),
1831 &[(passage("sealed away early", None), unit(1))],
1832 )
1833 .unwrap();
1834 index
1835 .add_source(
1836 &transcript("bb"),
1837 &[(passage("kept passage", None), unit(2))],
1838 )
1839 .unwrap();
1840 index.commit().unwrap();
1841 assert!(!index.full_segments.is_empty());
1842
1843 purge(&mut index, &[transcript("aa")], &[], &[]);
1844 index.commit().unwrap();
1845
1846 let hits = index.vector_search(&unit(1), 5, None).unwrap();
1847 assert!(
1848 hits.iter()
1849 .all(|(k, _)| { index.passage(*k).unwrap().unwrap().text == "kept passage" })
1850 );
1851 }
1852
1853 #[test]
1854 fn a_withdrawn_key_leaves_the_sealed_segment_file_that_held_it() {
1855 let dir = tempfile::tempdir().unwrap();
1856 let mut index = RawIndex::open(dir.path()).unwrap();
1857 index.segment_capacity = 1;
1858 index
1859 .add_source(
1860 &transcript("aa"),
1861 &[(passage("sealed away early", None), unit(1))],
1862 )
1863 .unwrap();
1864 index
1865 .add_source(
1866 &transcript("bb"),
1867 &[(passage("kept passage", None), unit(2))],
1868 )
1869 .unwrap();
1870 index.commit().unwrap();
1871 let key = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
1872 assert_eq!(key >> 32, 0);
1873 assert!(holds_key(&index, key, &unit(1)));
1874 let before = std::fs::metadata(segment_path(dir.path(), 0))
1875 .unwrap()
1876 .len();
1877
1878 purge(&mut index, &[transcript("aa")], &[], &[]);
1879
1880 assert!(!holds_key(&index, key, &unit(1)));
1881 drop(index);
1882 let after = std::fs::metadata(segment_path(dir.path(), 0))
1883 .unwrap()
1884 .len();
1885 assert!(after < before, "the embedding stayed in the file");
1886 assert!(!dir.path().join(REWRITE_JOURNAL).exists());
1887 let reopened = RawIndex::open(dir.path()).unwrap();
1888 assert!(!holds_key(&reopened, key, &unit(1)));
1889 assert_eq!(reopened.len(), 1);
1890 let hit = reopened.vector_search(&unit(2), 1, None).unwrap()[0].0;
1891 assert_eq!(reopened.passage(hit).unwrap().unwrap().text, "kept passage");
1892 }
1893
1894 #[test]
1895 fn a_rewrite_a_crash_interrupted_is_finished_at_open() {
1896 let dir = tempfile::tempdir().unwrap();
1897 let key = {
1898 let mut index = RawIndex::open(dir.path()).unwrap();
1899 index
1900 .add_source(
1901 &transcript("aa"),
1902 &[(passage("withdrawn before the crash", None), unit(1))],
1903 )
1904 .unwrap();
1905 index
1906 .add_source(
1907 &transcript("bb"),
1908 &[(passage("kept passage", None), unit(2))],
1909 )
1910 .unwrap();
1911 index.commit().unwrap();
1912 let key = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
1913 index.withdraw(&[key]).unwrap();
1914 index.forget_sources(&[transcript("aa")]).unwrap();
1915 index.seal_for_rewrite(&[key]).unwrap();
1916 index.commit().unwrap();
1917 key
1918 };
1919 assert!(dir.path().join(REWRITE_JOURNAL).exists());
1920
1921 let reopened = RawIndex::open(dir.path()).unwrap();
1922
1923 assert!(!dir.path().join(REWRITE_JOURNAL).exists());
1924 assert!(!holds_key(&reopened, key, &unit(1)));
1925 assert_eq!(reopened.len(), 1);
1926 }
1927
1928 #[test]
1929 fn a_key_withdrawn_from_the_loaded_segment_is_never_handed_out_again() {
1930 let dir = tempfile::tempdir().unwrap();
1931 let mut index = RawIndex::open(dir.path()).unwrap();
1932 let passages: Vec<_> = (0..3)
1933 .map(|i| (passage(&format!("passage number {i}"), None), unit(i)))
1934 .collect();
1935 index.add_source(&transcript("aa"), &passages).unwrap();
1936 index.commit().unwrap();
1937 let first = index.matching(&[], &[], &["passage number 0"]).unwrap()[0].key;
1938 index.withdraw(&[first]).unwrap();
1939 index.commit().unwrap();
1940
1941 index
1942 .add_source(&transcript("bb"), &[(passage("one more", None), unit(7))])
1943 .unwrap();
1944 index.commit().unwrap();
1945 drop(index);
1946 let mut reopened = RawIndex::open(dir.path()).unwrap();
1947 reopened
1948 .add_source(
1949 &transcript("cc"),
1950 &[(passage("and another", None), unit(8))],
1951 )
1952 .unwrap();
1953 reopened.commit().unwrap();
1954
1955 let all = reopened
1956 .matching(
1957 &[transcript("aa"), transcript("bb"), transcript("cc")],
1958 &[],
1959 &[],
1960 )
1961 .unwrap();
1962 assert_eq!(all.len(), 4);
1963 for (text, near) in [
1964 ("passage number 1", unit(1)),
1965 ("passage number 2", unit(2)),
1966 ("one more", unit(7)),
1967 ("and another", unit(8)),
1968 ] {
1969 let key = reopened.vector_search(&near, 1, None).unwrap()[0].0;
1970 assert_eq!(reopened.passage(key).unwrap().unwrap().text, text);
1971 }
1972 }
1973
1974 #[test]
1975 fn a_withdrawn_text_reads_as_not_indexed_before_any_merge() {
1976 let dir = tempfile::tempdir().unwrap();
1977 let mut index = RawIndex::open(dir.path()).unwrap();
1978 index
1979 .add_source(
1980 &transcript("aa"),
1981 &[
1982 (
1983 passage("the lantern hangs by the north door", None),
1984 unit(1),
1985 ),
1986 (passage("the gate is painted red", None), unit(2)),
1987 ],
1988 )
1989 .unwrap();
1990 index.commit().unwrap();
1991 let generation = index.purge_generation();
1992 let key = index
1993 .matching(&[], &[], &["the lantern hangs by the north door"])
1994 .unwrap()[0]
1995 .key;
1996
1997 index.withdraw(&[key]).unwrap();
1998 index.commit_without_merging().unwrap();
1999
2000 let term = Term::from_field_text(
2001 index.fields.text_hash,
2002 &text_hash("the lantern hangs by the north door"),
2003 );
2004 assert_eq!(index.reader.searcher().doc_freq(&term).unwrap(), 1);
2005 assert_ne!(index.purge_generation(), generation);
2006 assert!(!index.has_text("the lantern hangs by the north door"));
2007 assert!(index.has_text("the gate is painted red"));
2008 assert!(index.has_source(&transcript("aa")));
2009 assert_eq!(
2010 index
2011 .add_source(
2012 &transcript("bb"),
2013 &[(
2014 passage("the lantern hangs by the north door", None),
2015 unit(1)
2016 )],
2017 )
2018 .unwrap(),
2019 1
2020 );
2021
2022 let rest = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
2023 index.withdraw(&[rest]).unwrap();
2024 index.forget_sources(&[transcript("aa")]).unwrap();
2025 index.commit_without_merging().unwrap();
2026 assert!(!index.has_source(&transcript("aa")));
2027 assert!(index.has_source(&transcript("bb")));
2028 }
2029
2030 #[test]
2031 fn a_rekeyed_passage_keeps_its_vector_under_its_new_source() {
2032 let dir = tempfile::tempdir().unwrap();
2033 let mut index = RawIndex::open(dir.path()).unwrap();
2034 index
2035 .add_source(&transcript("gone"), &[(passage("yes", None), unit(1))])
2036 .unwrap();
2037 index.commit().unwrap();
2038 let key = index.matching(&[transcript("gone")], &[], &[]).unwrap()[0].key;
2039
2040 index
2041 .rekey(
2042 key,
2043 &transcript("kept"),
2044 &passage("yes", Some("-srv-garden")),
2045 )
2046 .unwrap();
2047 index.forget_sources(&[transcript("gone")]).unwrap();
2048 index.commit().unwrap();
2049 index.compact().unwrap();
2050
2051 let hit = index.vector_search(&unit(1), 1, None).unwrap()[0].0;
2052 assert_eq!(hit, key);
2053 let hit = index.passage(hit).unwrap().unwrap();
2054 assert_eq!(hit.origin, transcript("kept"));
2055 assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
2056 assert_eq!(index.sources().unwrap(), vec![transcript("kept")]);
2057 assert!(index.has_text("yes"));
2058 }
2059
2060 const MARKER: &str = "zanzibarquokka";
2061
2062 fn seal_key(byte: u8) -> SealKey {
2063 SealKey::from_bytes(KeyGuard::new([byte; 32])).unwrap()
2064 }
2065
2066 fn seal_keys(byte: u8) -> SealKeys {
2067 SealKeys::single(seal_key(byte))
2068 }
2069
2070 fn files_under(dir: &Path) -> Vec<PathBuf> {
2071 let mut out = Vec::new();
2072 for entry in std::fs::read_dir(dir).unwrap() {
2073 let path = entry.unwrap().path();
2074 let name = path.file_name().unwrap().to_string_lossy().into_owned();
2075 if path.is_dir() {
2076 out.extend(files_under(&path));
2077 } else if !(name.starts_with(".tantivy-") && name.ends_with(".lock")) {
2078 out.push(path);
2079 }
2080 }
2081 out
2082 }
2083
2084 fn assert_all_sealed_under(dir: &Path, id: SealKeyId) {
2085 let files = files_under(dir);
2086 assert!(files.iter().any(|p| p.starts_with(dir.join(TEXT_DIR))));
2087 assert!(
2088 files
2089 .iter()
2090 .any(|p| p.extension().is_some_and(|e| e == "usearch"))
2091 );
2092 for path in files {
2093 assert_eq!(
2094 sealed_key_id(&path).unwrap(),
2095 Some(id),
2096 "{} is not sealed under the current key",
2097 path.display()
2098 );
2099 }
2100 }
2101
2102 fn holds_bytes(dir: &Path, needle: &str) -> bool {
2103 files_under(dir).iter().any(|p| {
2104 std::fs::read(p)
2105 .unwrap()
2106 .windows(needle.len())
2107 .any(|w| w == needle.as_bytes())
2108 })
2109 }
2110
2111 fn spread(i: usize) -> Vec<f32> {
2112 let mut v = unit(i);
2113 v[(i + 1) % EMBEDDING_DIM] = 0.5;
2114 v
2115 }
2116
2117 fn ramp(i: usize) -> Vec<f32> {
2118 let mut v: Vec<f32> = (0..EMBEDDING_DIM)
2119 .map(|j| (j + 1) as f32 / EMBEDDING_DIM as f32)
2120 .collect();
2121 v[i] += 5.0;
2122 v
2123 }
2124
2125 fn populate(index: &mut RawIndex) {
2126 index.segment_capacity = 2;
2127 let passages: Vec<_> = (0..5)
2128 .map(|i| {
2129 (
2130 passage(&format!("{MARKER} passage number {i}"), Some("-srv-garden")),
2131 spread(i),
2132 )
2133 })
2134 .collect();
2135 index.add_source(&transcript("aa"), &passages).unwrap();
2136 let id = Uuid::new_v4();
2137 let mut memory = passage(&format!("{MARKER} archived memory"), None);
2138 memory.memory = Some(id);
2139 index
2140 .add_source(&RawOrigin::Memory { id }, &[(memory, spread(7))])
2141 .unwrap();
2142 index.commit().unwrap();
2143 }
2144
2145 type TopK = Vec<Vec<(u64, f32)>>;
2146
2147 fn top_k(index: &RawIndex) -> TopK {
2148 let mut out: TopK = (0..8)
2149 .map(|i| index.vector_search(&ramp(i), 4, None).unwrap())
2150 .collect();
2151 out.push(index.bm25_search(MARKER, 10, None).unwrap());
2152 out.push(
2153 index
2154 .bm25_search("number 3", 10, Some("-srv-garden"))
2155 .unwrap(),
2156 );
2157 assert!(out.iter().all(|hits| !hits.is_empty()));
2158 out
2159 }
2160
2161 fn stored_json(index: &RawIndex) -> String {
2162 let searcher = index.reader.searcher();
2163 let schema = index.index.schema();
2164 let mut out = String::new();
2165 for (ord, segment) in searcher.segment_readers().iter().enumerate() {
2166 for doc in 0..segment.max_doc() {
2167 let document: TantivyDocument =
2168 searcher.doc(DocAddress::new(ord as u32, doc)).unwrap();
2169 out.push_str(&document.to_json(&schema));
2170 out.push('\n');
2171 }
2172 }
2173 out
2174 }
2175
2176 #[test]
2177 fn a_keyed_index_does_not_open_without_its_key_or_under_another() {
2178 let dir = tempfile::tempdir().unwrap();
2179 populate(&mut RawIndex::open_keyed(dir.path(), &seal_keys(1)).unwrap());
2180
2181 for opened in [
2182 RawIndex::open(dir.path()),
2183 RawIndex::open_readonly(dir.path()),
2184 ] {
2185 let err = opened.err().expect("a sealed index opened without keys");
2186 assert_eq!(seal_error(&err), Some(&SealError::KeyRequired), "{err:#}");
2187 }
2188 for opened in [
2189 RawIndex::open_keyed(dir.path(), &seal_keys(2)),
2190 RawIndex::open_readonly_keyed(dir.path(), &seal_keys(2)),
2191 ] {
2192 let err = opened
2193 .err()
2194 .expect("a sealed index opened under another key");
2195 assert!(
2196 matches!(seal_error(&err), Some(SealError::UnknownKey(_))),
2197 "{err:#}"
2198 );
2199 }
2200 let reopened = RawIndex::open_readonly_keyed(dir.path(), &seal_keys(1)).unwrap();
2201 assert_eq!(reopened.len(), 6);
2202 assert_eq!(
2203 reopened.seal_keys().unwrap().current_id(),
2204 seal_keys(1).current_id()
2205 );
2206 }
2207
2208 #[test]
2209 fn a_plaintext_index_opened_keyed_searches_the_same_and_reseal_seals_every_file() {
2210 let dir = tempfile::tempdir().unwrap();
2211 populate(&mut RawIndex::open(dir.path()).unwrap());
2212 let baseline = top_k(&RawIndex::open_readonly(dir.path()).unwrap());
2213
2214 let keys = seal_keys(3);
2215 let mut index = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2216 assert_eq!(top_k(&index), baseline);
2217 assert!(index.reseal().unwrap() > 0);
2218 assert_all_sealed_under(dir.path(), keys.current_id());
2219 assert_eq!(top_k(&index), baseline);
2220 assert_eq!(index.reseal().unwrap(), 0);
2221 drop(index);
2222
2223 assert_eq!(
2224 top_k(&RawIndex::open_readonly_keyed(dir.path(), &keys).unwrap()),
2225 baseline
2226 );
2227 let reopened = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2228 assert_eq!(top_k(&reopened), baseline);
2229 let hit = reopened.bm25_search("number 3", 1, None).unwrap()[0].0;
2230 assert_eq!(
2231 reopened.passage(hit).unwrap().unwrap().text,
2232 format!("{MARKER} passage number 3")
2233 );
2234 }
2235
2236 #[test]
2237 fn no_passage_text_reaches_a_file_of_a_keyed_index() {
2238 let dir = tempfile::tempdir().unwrap();
2239 let keys = seal_keys(4);
2240 let mut index = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2241 populate(&mut index);
2242 index.compact().unwrap();
2243
2244 assert!(!holds_bytes(dir.path(), MARKER));
2245 assert_all_sealed_under(dir.path(), keys.current_id());
2246 let hit = index.bm25_search("number 3", 1, None).unwrap()[0].0;
2247 assert_eq!(
2248 index.passage(hit).unwrap().unwrap().text,
2249 format!("{MARKER} passage number 3")
2250 );
2251 }
2252
2253 #[test]
2254 fn a_rotation_reseals_every_file_under_the_new_key_only() {
2255 let dir = tempfile::tempdir().unwrap();
2256 populate(&mut RawIndex::open_keyed(dir.path(), &SealKeys::single(seal_key(5))).unwrap());
2257 let mut index = RawIndex::open_keyed(dir.path(), &SealKeys::single(seal_key(5))).unwrap();
2258 let baseline = top_k(&index);
2259
2260 index
2261 .set_seal_keys(SealKeys::new(seal_key(6), vec![seal_key(5)]))
2262 .unwrap();
2263 assert!(index.reseal().unwrap() > 0);
2264
2265 assert_all_sealed_under(dir.path(), seal_key(6).id());
2266 assert_eq!(top_k(&index), baseline);
2267 index
2268 .add_source(
2269 &transcript("bb"),
2270 &[(passage("one more passage", None), unit(9))],
2271 )
2272 .unwrap();
2273 index.commit().unwrap();
2274 drop(index);
2275 let reopened = RawIndex::open_keyed(dir.path(), &SealKeys::single(seal_key(6))).unwrap();
2276 assert_eq!(reopened.len(), 7);
2277 assert_all_sealed_under(dir.path(), seal_key(6).id());
2278 }
2279
2280 #[test]
2281 fn a_rewrite_drops_withdrawn_vectors_from_sealed_segments_and_keeps_its_journal_sealed() {
2282 let dir = tempfile::tempdir().unwrap();
2283 let keys = seal_keys(7);
2284 let (gone, crashed) = {
2285 let mut index = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2286 index.segment_capacity = 1;
2287 for (sha, text, i) in [
2288 ("aa", "sealed away early", 1),
2289 ("bb", "withdrawn before the crash", 2),
2290 ("cc", "kept passage", 3),
2291 ] {
2292 index
2293 .add_source(&transcript(sha), &[(passage(text, None), unit(i))])
2294 .unwrap();
2295 }
2296 index.commit().unwrap();
2297 let gone = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
2298 assert!(holds_key(&index, gone, &unit(1)));
2299 purge(&mut index, &[transcript("aa")], &[], &[]);
2300 assert!(!holds_key(&index, gone, &unit(1)));
2301
2302 let crashed = index.matching(&[transcript("bb")], &[], &[]).unwrap()[0].key;
2303 index.withdraw(&[crashed]).unwrap();
2304 index.forget_sources(&[transcript("bb")]).unwrap();
2305 index.seal_for_rewrite(&[crashed]).unwrap();
2306 index.commit().unwrap();
2307 (gone, crashed)
2308 };
2309 let journal = dir.path().join(REWRITE_JOURNAL);
2310 assert_eq!(sealed_key_id(&journal).unwrap(), Some(keys.current_id()));
2311 let err = RawIndex::open(dir.path())
2312 .err()
2313 .expect("a sealed journal read without keys");
2314 assert_eq!(seal_error(&err), Some(&SealError::KeyRequired), "{err:#}");
2315 assert!(journal.exists());
2316
2317 let reopened = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2318
2319 assert!(!journal.exists());
2320 assert!(!holds_key(&reopened, gone, &unit(1)));
2321 assert!(!holds_key(&reopened, crashed, &unit(2)));
2322 assert_eq!(reopened.len(), 1);
2323 for n in segment_files(dir.path()).unwrap() {
2324 assert_eq!(
2325 sealed_key_id(&segment_path(dir.path(), n)).unwrap(),
2326 Some(keys.current_id())
2327 );
2328 }
2329 let hit = reopened.vector_search(&unit(3), 1, None).unwrap()[0].0;
2330 assert_eq!(reopened.passage(hit).unwrap().unwrap().text, "kept passage");
2331 }
2332
2333 #[test]
2334 fn a_rewrite_pending_when_the_index_is_reopened_keyed_writes_sealed() {
2335 let dir = tempfile::tempdir().unwrap();
2336 let keys = seal_keys(8);
2337 let mut index = RawIndex::open(dir.path()).unwrap();
2338 index.segment_capacity = 1;
2339 for (sha, text, i) in [
2340 ("aa", "withdrawn while unkeyed", 1),
2341 ("bb", "kept passage", 2),
2342 ] {
2343 index
2344 .add_source(&transcript(sha), &[(passage(text, None), unit(i))])
2345 .unwrap();
2346 }
2347 index.commit().unwrap();
2348 let gone = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
2349 index.withdraw(&[gone]).unwrap();
2350 index.forget_sources(&[transcript("aa")]).unwrap();
2351 let rewrite = index.seal_for_rewrite(&[gone]).unwrap();
2352 index.commit().unwrap();
2353
2354 index.set_seal_keys(keys.clone()).unwrap();
2355 index.reseal().unwrap();
2356 rewrite.run().unwrap();
2357
2358 for n in segment_files(dir.path()).unwrap() {
2359 assert_eq!(
2360 sealed_key_id(&segment_path(dir.path(), n)).unwrap(),
2361 Some(keys.current_id()),
2362 "segment {n}"
2363 );
2364 }
2365 let reopened = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2366 assert!(!holds_key(&reopened, gone, &unit(1)));
2367 assert_eq!(reopened.len(), 1);
2368 }
2369
2370 #[test]
2371 fn an_archived_memorys_passage_is_searchable_but_keeps_no_stored_text() {
2372 let dir = tempfile::tempdir().unwrap();
2373 let mut index = RawIndex::open(dir.path()).unwrap();
2374 let id = Uuid::new_v4();
2375 let text = "the quokka ferry leaves at dawn";
2376 let mut memory = passage(text, None);
2377 memory.memory = Some(id);
2378 index
2379 .add_source(&RawOrigin::Memory { id }, &[(memory.clone(), unit(1))])
2380 .unwrap();
2381 index
2382 .add_source(
2383 &transcript("aa"),
2384 &[(
2385 passage("the lantern hangs by the north door", None),
2386 unit(2),
2387 )],
2388 )
2389 .unwrap();
2390 index.commit().unwrap();
2391
2392 let stored = stored_json(&index);
2393 assert!(!stored.contains("quokka"), "{stored}");
2394 assert!(stored.contains("the lantern hangs by the north door"));
2395 let key = index.bm25_search("quokka ferry", 5, None).unwrap()[0].0;
2396 assert_eq!(index.vector_search(&unit(1), 1, None).unwrap()[0].0, key);
2397 let hit = index.passage(key).unwrap().unwrap();
2398 assert_eq!(hit.origin, RawOrigin::Memory { id });
2399 assert_eq!(hit.memory, Some(id));
2400 assert!(hit.text.is_empty());
2401 assert_eq!(hit.text_hash, text_hash(text));
2402 assert!(index.has_text(text));
2403 assert_eq!(index.matching(&[], &[], &[text]).unwrap()[0].key, key);
2404
2405 index
2406 .rekey(key, &transcript("bb"), &passage(text, None))
2407 .unwrap();
2408 index.commit().unwrap();
2409 assert_eq!(index.passage(key).unwrap().unwrap().text, text);
2410 index
2411 .rekey(key, &RawOrigin::Memory { id }, &memory)
2412 .unwrap();
2413 index.compact().unwrap();
2414 assert!(!stored_json(&index).contains("quokka"));
2415 assert_eq!(index.bm25_search("quokka", 5, None).unwrap()[0].0, key);
2416 }
2417
2418 type LegacyDoc = (RawOrigin, &'static str, Option<Uuid>);
2419
2420 fn write_legacy_index(dir: &Path, docs: &[LegacyDoc]) -> Vec<u64> {
2423 let text_dir = dir.join(TEXT_DIR);
2424 std::fs::create_dir_all(&text_dir).unwrap();
2425 let schema = legacy_schema();
2426 let field = |name: &str| schema.get_field(name).unwrap();
2427 let index = Index::create(
2428 MmapDirectory::open(&text_dir).unwrap(),
2429 schema.clone(),
2430 IndexSettings::default(),
2431 )
2432 .unwrap();
2433 let mut writer: IndexWriter = index.writer_with_num_threads(1, WRITER_HEAP_BYTES).unwrap();
2434 let mut vectors =
2435 VectorIndex::open_with_kind(&segment_path(dir, 0), ScalarKind::F16).unwrap();
2436 let mut keys = Vec::new();
2437 for (i, (origin, text, memory)) in docs.iter().enumerate() {
2438 let key = i as u64;
2439 let mut document = doc!(
2440 field("key") => key,
2441 field("body") => *text,
2442 field("text_hash") => text_hash(text),
2443 field("source") => origin.source_key(),
2444 field("project") => "-srv-garden",
2445 field("at") => 1_790_000_000i64,
2446 );
2447 if let Some(memory) = memory {
2448 document.add_text(field("memory"), memory.to_string());
2449 }
2450 writer.add_document(document).unwrap();
2451 vectors.upsert(key, &unit(i)).unwrap();
2452 keys.push(key);
2453 }
2454 let withdrawn = docs.len() as u64;
2455 writer
2456 .add_document(doc!(
2457 field("key") => withdrawn,
2458 field("body") => "a withdrawn line",
2459 field("text_hash") => text_hash("a withdrawn line"),
2460 field("source") => transcript("aa").source_key(),
2461 ))
2462 .unwrap();
2463 writer.delete_term(Term::from_field_u64(field("key"), withdrawn));
2464 writer
2465 .add_document(doc!(field("source") => transcript("empty").source_key()))
2466 .unwrap();
2467 writer.commit().unwrap();
2468 writer.wait_merging_threads().unwrap();
2469 vectors.save().unwrap();
2470 keys
2471 }
2472
2473 fn legacy_docs(memory: Uuid) -> Vec<LegacyDoc> {
2474 vec![
2475 (
2476 transcript("aa"),
2477 "the lantern hangs by the north door",
2478 None,
2479 ),
2480 (
2481 RawOrigin::Memory { id: memory },
2482 "the quokka ferry leaves at dawn",
2483 Some(memory),
2484 ),
2485 (
2486 transcript("aa"),
2487 "a third line of the transcript",
2488 Some(Uuid::new_v4()),
2489 ),
2490 ]
2491 }
2492
2493 fn assert_migrated(dir: &Path, index: &RawIndex, docs: &[LegacyDoc], keys: &[u64]) {
2494 assert_eq!(index.index.schema(), schema().0);
2495 assert_eq!(index.len(), docs.len());
2496 for (i, ((origin, text, memory), key)) in docs.iter().zip(keys).enumerate() {
2497 assert_eq!(index.vector_search(&unit(i), 1, None).unwrap()[0].0, *key);
2498 let hit = index.passage(*key).unwrap().unwrap();
2499 assert_eq!(&hit.origin, origin);
2500 assert_eq!(hit.memory, *memory);
2501 assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
2502 assert_eq!(hit.at.unwrap().timestamp(), 1_790_000_000);
2503 assert_eq!(hit.text_hash, text_hash(text));
2504 let stored = match origin {
2505 RawOrigin::Memory { .. } => "",
2506 RawOrigin::Transcript { .. } => text,
2507 };
2508 assert_eq!(hit.text, stored);
2509 assert!(index.has_text(text));
2510 }
2511 assert_eq!(index.bm25_search("quokka", 5, None).unwrap()[0].0, keys[1]);
2512 assert!(!index.has_text("a withdrawn line"));
2513 assert!(index.has_source(&transcript("empty")));
2514 assert!(!stored_json(index).contains("quokka"));
2515 for leftover in [MIGRATED_TEXT_DIR, REPLACED_TEXT_DIR, MIGRATION_JOURNAL] {
2516 assert!(!dir.join(leftover).exists(), "{leftover} was left behind");
2517 }
2518 }
2519
2520 #[test]
2521 fn an_index_written_with_the_old_schema_migrates_keeping_every_passage_and_vector() {
2522 let dir = tempfile::tempdir().unwrap();
2523 let docs = legacy_docs(Uuid::new_v4());
2524 let keys = write_legacy_index(dir.path(), &docs);
2525
2526 let mut index = RawIndex::open(dir.path()).unwrap();
2527
2528 assert_migrated(dir.path(), &index, &docs, &keys);
2529 assert_migrated(
2530 dir.path(),
2531 &RawIndex::open_readonly(dir.path()).unwrap(),
2532 &docs,
2533 &keys,
2534 );
2535 index
2536 .add_source(
2537 &transcript("bb"),
2538 &[(passage("one more passage", None), unit(9))],
2539 )
2540 .unwrap();
2541 index.commit().unwrap();
2542 assert_eq!(index.len(), docs.len() + 1);
2543 let key = index.vector_search(&unit(9), 1, None).unwrap()[0].0;
2544 assert_eq!(
2545 index.passage(key).unwrap().unwrap().text,
2546 "one more passage"
2547 );
2548 assert!(!keys.contains(&key));
2549 }
2550
2551 #[test]
2552 fn an_old_schema_index_opened_keyed_migrates_into_sealed_text_files() {
2553 let dir = tempfile::tempdir().unwrap();
2554 let docs = legacy_docs(Uuid::new_v4());
2555 let keys = write_legacy_index(dir.path(), &docs);
2556 let sealing = seal_keys(8);
2557
2558 let index = RawIndex::open_keyed(dir.path(), &sealing).unwrap();
2559
2560 assert_migrated(dir.path(), &index, &docs, &keys);
2561 for path in files_under(&dir.path().join(TEXT_DIR)) {
2562 assert_eq!(
2563 sealed_key_id(&path).unwrap(),
2564 Some(sealing.current_id()),
2565 "{}",
2566 path.display()
2567 );
2568 }
2569 assert!(!holds_bytes(dir.path(), "lantern"));
2570 }
2571
2572 #[test]
2573 fn a_migration_a_crash_cut_short_is_redone_or_finished_at_open() {
2574 let before_the_journal = tempfile::tempdir().unwrap();
2575 let docs = legacy_docs(Uuid::new_v4());
2576 let keys = write_legacy_index(before_the_journal.path(), &docs);
2577 let partial = before_the_journal.path().join(MIGRATED_TEXT_DIR);
2578 std::fs::create_dir_all(&partial).unwrap();
2579 std::fs::write(partial.join(META_FILE), "a copy cut short").unwrap();
2580 let index = RawIndex::open(before_the_journal.path()).unwrap();
2581 assert_migrated(before_the_journal.path(), &index, &docs, &keys);
2582
2583 let mid_swap = tempfile::tempdir().unwrap();
2584 let keys = write_legacy_index(mid_swap.path(), &docs);
2585 build_migrated_text(mid_swap.path(), None).unwrap();
2586 sealed::write_file(
2587 &mid_swap.path().join(MIGRATION_JOURNAL),
2588 None,
2589 MIGRATION_JOURNAL,
2590 b"",
2591 )
2592 .unwrap();
2593 std::fs::rename(
2594 mid_swap.path().join(TEXT_DIR),
2595 mid_swap.path().join(REPLACED_TEXT_DIR),
2596 )
2597 .unwrap();
2598 let index = RawIndex::open(mid_swap.path()).unwrap();
2599 assert_migrated(mid_swap.path(), &index, &docs, &keys);
2600 }
2601}