Skip to main content

second_brain_core/index/
raw.rs

1use std::collections::{BTreeMap, BTreeSet, HashSet};
2use std::path::{Path, PathBuf};
3use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
4
5use anyhow::{Context, Result, anyhow, bail, ensure};
6use chrono::{DateTime, Utc};
7use tantivy::collector::{DocSetCollector, TopDocs};
8use tantivy::directory::{Directory, MmapDirectory};
9use tantivy::merge_policy::{DefaultMergePolicy, NoMergePolicy};
10use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
11use tantivy::schema::{
12    Field, INDEXED, IndexRecordOption, STORED, STRING, Schema, TEXT, TantivyDocument, Value,
13};
14use tantivy::{
15    DocAddress, DocSet, FutureResult, Index, IndexReader, IndexSettings, IndexWriter, ReloadPolicy,
16    Searcher, SegmentMeta, TERMINATED, Term, doc,
17};
18use usearch::{IndexOptions, MetricKind, ScalarKind};
19use uuid::Uuid;
20
21use super::vector::{EMBEDDING_DIM, VectorIndex};
22use crate::sealed::{self, SealKeys, SealedDirectory};
23
24const SEGMENT_CAPACITY: u64 = 100_000;
25const WRITER_HEAP_BYTES: usize = 15_000_000;
26const OVERFETCH: usize = 4;
27const TEXT_DIR: &str = "text";
28const MIGRATED_TEXT_DIR: &str = "text.migrated";
29const REPLACED_TEXT_DIR: &str = "text.replaced";
30const MIGRATION_JOURNAL: &str = "text-migration.pending";
31const REWRITE_JOURNAL: &str = "rewrite.pending";
32const META_FILE: &str = "meta.json";
33
34/// The raw transcript index directory that belongs to the database at `db_path`.
35pub fn dir_for_db(db_path: &Path) -> PathBuf {
36    db_path.with_extension("raw-index")
37}
38
39/// The hash passages are matched by, of the trimmed text. Every passage
40/// keeps it, including one whose text the index does not store.
41pub fn text_hash(text: &str) -> String {
42    use sha2::{Digest, Sha256};
43    let digest = Sha256::digest(text.trim().as_bytes());
44    digest.iter().take(16).map(|b| format!("{b:02x}")).collect()
45}
46
47/// Where a raw passage came from, as stored in the index.
48#[derive(Debug, Clone, PartialEq, Eq)]
49pub enum RawOrigin {
50    /// A transcript in the archive, by the sha256 of its blob.
51    Transcript { sha256: String },
52    /// An archived raw memory.
53    Memory { id: Uuid },
54}
55
56impl RawOrigin {
57    /// The key a source is recorded under once indexed.
58    pub fn source_key(&self) -> String {
59        match self {
60            Self::Transcript { sha256 } => format!("t:{sha256}"),
61            Self::Memory { id } => format!("m:{id}"),
62        }
63    }
64
65    fn parse(key: &str) -> Option<Self> {
66        match key.split_once(':')? {
67            ("t", sha256) => Some(Self::Transcript {
68                sha256: sha256.to_string(),
69            }),
70            ("m", id) => Uuid::parse_str(id).ok().map(|id| Self::Memory { id }),
71            _ => None,
72        }
73    }
74}
75
76#[derive(Debug, Clone)]
77pub struct RawPassage {
78    pub text: String,
79    pub at: Option<DateTime<Utc>>,
80    pub project: Option<String>,
81    /// The raw memory ingest stores this passage as, so forgetting that
82    /// memory also withdraws the passage.
83    pub memory: Option<Uuid>,
84}
85
86#[derive(Debug, Clone)]
87pub struct RawHit {
88    pub key: u64,
89    /// Empty for an archived memory's passage: its text is the store's to
90    /// keep, a cold row's in its sealed segment.
91    pub text: String,
92    /// `text_hash` of the passage's text.
93    pub text_hash: String,
94    pub at: Option<DateTime<Utc>>,
95    pub project: Option<String>,
96    pub memory: Option<Uuid>,
97    pub origin: RawOrigin,
98}
99
100struct Fields {
101    key: Field,
102    body: Field,
103    text: Field,
104    text_hash: Field,
105    source: Field,
106    project: Field,
107    memory: Field,
108    at: Field,
109}
110
111/// The keys files are sealed at rest under, None for a plaintext index.
112/// Shared with each `SealedRewrite`, which holds the lock while it writes
113/// segment files, so a reseal never rewrites one under it.
114type AtRestKeys = Arc<Mutex<Option<SealKeys>>>;
115
116fn lock_keys(keys: &AtRestKeys) -> MutexGuard<'_, Option<SealKeys>> {
117    keys.lock().unwrap_or_else(PoisonError::into_inner)
118}
119
120/// Passages that left the hot indexes, searchable by vector and BM25. The
121/// vectors live in fixed-size segments so only the one being filled is
122/// writable; the full ones are mmapped views, or copies loaded into RAM
123/// when the index is sealed at rest, since a sealed file cannot be mmapped.
124/// Tantivy holds each passage's metadata under its vector key, so no SQLite
125/// table maps keys.
126pub struct RawIndex {
127    dir: PathBuf,
128    full_segments: Vec<(u64, VectorIndex)>,
129    active: Option<VectorIndex>,
130    active_segment: u64,
131    next_ordinal: u64,
132    segment_capacity: u64,
133    index: Index,
134    reader: IndexReader,
135    writer: Option<IndexWriter>,
136    fields: Fields,
137    writable: bool,
138    text_bytes: u64,
139    pending_texts: HashSet<String>,
140    pending_sources: HashSet<String>,
141    purge_generation: u64,
142    at_rest: AtRestKeys,
143    text_at_rest: Option<SealedDirectory>,
144}
145
146/// Withdrawn keys whose vectors still sit in full segment files. `run`
147/// rebuilds each such file without them; a usearch removal only marks a
148/// slot free and leaves the embedding in the file. The keys are journaled
149/// in the index directory until `run` finishes, and `RawIndex::open`
150/// finishes a journal a crash left behind.
151pub struct SealedRewrite {
152    dir: PathBuf,
153    keys: BTreeSet<u64>,
154    at_rest: AtRestKeys,
155}
156
157/// A merge of every text segment, scheduled so that it runs while recall
158/// keeps reading the segments it replaces.
159pub struct Compaction {
160    merge: Option<FutureResult<Option<SegmentMeta>>>,
161}
162
163/// Only a transcript's passages store their text. An archived memory's
164/// passage keeps what search needs (its terms, its hash and its metadata),
165/// since storing its text beside its cold segment would keep a plaintext
166/// copy the cold tier exists to drop.
167fn schema() -> (Schema, Fields) {
168    let mut builder = Schema::builder();
169    let fields = Fields {
170        key: builder.add_u64_field("key", INDEXED | STORED),
171        body: builder.add_text_field("body", TEXT),
172        text: builder.add_text_field("text", STORED),
173        text_hash: builder.add_text_field("text_hash", STRING | STORED),
174        source: builder.add_text_field("source", STRING | STORED),
175        project: builder.add_text_field("project", STRING | STORED),
176        memory: builder.add_text_field("memory", STRING | STORED),
177        at: builder.add_i64_field("at", STORED),
178    };
179    (builder.build(), fields)
180}
181
182/// The schema before passages of archived memories stopped storing their
183/// text, which `RawIndex::open` migrates from.
184fn legacy_schema() -> Schema {
185    let mut builder = Schema::builder();
186    builder.add_u64_field("key", INDEXED | STORED);
187    builder.add_text_field("body", TEXT | STORED);
188    builder.add_text_field("text_hash", STRING);
189    builder.add_text_field("source", STRING | STORED);
190    builder.add_text_field("project", STRING | STORED);
191    builder.add_text_field("memory", STRING | STORED);
192    builder.add_i64_field("at", STORED);
193    builder.build()
194}
195
196fn stores_text(source: &str) -> bool {
197    !matches!(RawOrigin::parse(source), Some(RawOrigin::Memory { .. }))
198}
199
200fn passage_document(
201    fields: &Fields,
202    key: u64,
203    source: &str,
204    passage: &RawPassage,
205) -> TantivyDocument {
206    let mut document = doc!(
207        fields.key => key,
208        fields.body => passage.text.as_str(),
209        fields.text_hash => text_hash(&passage.text),
210        fields.source => source,
211    );
212    if stores_text(source) {
213        document.add_text(fields.text, &passage.text);
214    }
215    if let Some(project) = &passage.project {
216        document.add_text(fields.project, project);
217    }
218    if let Some(memory) = passage.memory {
219        document.add_text(fields.memory, memory.to_string());
220    }
221    if let Some(at) = passage.at {
222        document.add_i64(fields.at, at.timestamp());
223    }
224    document
225}
226
227fn segment_name(segment: u64) -> String {
228    format!("vectors-{segment:05}.usearch")
229}
230
231fn segment_path(dir: &Path, segment: u64) -> PathBuf {
232    dir.join(segment_name(segment))
233}
234
235fn segment_files(dir: &Path) -> Result<Vec<u64>> {
236    let mut segments = Vec::new();
237    for entry in std::fs::read_dir(dir).with_context(|| format!("listing {}", dir.display()))? {
238        let name = entry?.file_name();
239        let Some(name) = name.to_str() else { continue };
240        if let Some(n) = name
241            .strip_prefix("vectors-")
242            .and_then(|rest| rest.strip_suffix(".usearch"))
243            .and_then(|n| n.parse().ok())
244        {
245            segments.push(n);
246        }
247    }
248    segments.sort_unstable();
249    Ok(segments)
250}
251
252fn open_writable_segment(path: &Path, at_rest: Option<&SealKeys>) -> Result<VectorIndex> {
253    match at_rest {
254        Some(keys) => VectorIndex::open_with_kind_keyed(path, ScalarKind::F16, keys),
255        None => VectorIndex::open_with_kind(path, ScalarKind::F16),
256    }
257}
258
259fn open_full_segment(path: &Path, at_rest: Option<&SealKeys>) -> Result<VectorIndex> {
260    match at_rest {
261        Some(keys) => VectorIndex::open_loaded_with_kind(path, ScalarKind::F16, keys),
262        None => VectorIndex::open_view_with_kind(path, ScalarKind::F16),
263    }
264}
265
266/// Files a write killed midway leaves in the index directory: temporary
267/// copies of segments and journals, which may hold plaintext vectors.
268fn is_scratch_file(name: &str) -> bool {
269    sealed::is_tmp_name(name) || name.ends_with(".tmp") || name.ends_with(".rebuilt.usearch")
270}
271
272/// Regular files directly in `dir`, by name.
273fn top_level_files(dir: &Path) -> Result<Vec<String>> {
274    let mut names = Vec::new();
275    for entry in std::fs::read_dir(dir).with_context(|| format!("listing {}", dir.display()))? {
276        let entry = entry?;
277        if entry.file_type()?.is_file()
278            && let Some(name) = entry.file_name().to_str()
279        {
280            names.push(name.to_string());
281        }
282    }
283    Ok(names)
284}
285
286fn remove_scratch_files(dir: &Path) -> Result<()> {
287    for name in top_level_files(dir)? {
288        if is_scratch_file(&name) {
289            let path = dir.join(&name);
290            std::fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?;
291        }
292    }
293    Ok(())
294}
295
296fn dir_bytes(dir: &Path) -> u64 {
297    std::fs::read_dir(dir)
298        .map(|entries| {
299            entries
300                .filter_map(|e| e.ok()?.metadata().ok())
301                .map(|m| m.len())
302                .sum()
303        })
304        .unwrap_or(0)
305}
306
307fn sync_dir(dir: &Path) -> Result<()> {
308    std::fs::File::open(dir)
309        .and_then(|d| d.sync_all())
310        .with_context(|| format!("syncing {}", dir.display()))
311}
312
313/// Whether a document not withdrawn holds `term`, stopping at the first.
314/// `doc_freq` still counts withdrawn documents until a merge drops them.
315fn holds_live(searcher: &Searcher, term: Term) -> Result<bool> {
316    for segment in searcher.segment_readers() {
317        let Some(mut postings) = segment
318            .inverted_index(term.field())?
319            .read_postings(&term, IndexRecordOption::Basic)?
320        else {
321            continue;
322        };
323        let mut doc = postings.doc();
324        while doc != TERMINATED {
325            if !segment.is_deleted(doc) {
326                return Ok(true);
327            }
328            doc = postings.advance();
329        }
330    }
331    Ok(false)
332}
333
334fn read_journal(dir: &Path, at_rest: Option<&SealKeys>) -> Result<BTreeSet<u64>> {
335    let path = dir.join(REWRITE_JOURNAL);
336    if !path.exists() {
337        return Ok(BTreeSet::new());
338    }
339    let bytes = sealed::read_file(&path, at_rest, REWRITE_JOURNAL)?;
340    let text = String::from_utf8(bytes).with_context(|| format!("reading {}", path.display()))?;
341    text.lines()
342        .map(str::trim)
343        .filter(|line| !line.is_empty())
344        .map(|line| {
345            line.parse::<u64>()
346                .with_context(|| format!("reading {}", path.display()))
347        })
348        .collect()
349}
350
351fn write_journal(dir: &Path, keys: &BTreeSet<u64>, at_rest: Option<&SealKeys>) -> Result<()> {
352    let body: String = keys.iter().map(|k| format!("{k}\n")).collect();
353    sealed::write_file(
354        &dir.join(REWRITE_JOURNAL),
355        at_rest,
356        REWRITE_JOURNAL,
357        body.as_bytes(),
358    )
359}
360
361fn segment_options() -> IndexOptions {
362    IndexOptions {
363        dimensions: EMBEDDING_DIM,
364        metric: MetricKind::Cos,
365        quantization: ScalarKind::F16,
366        ..IndexOptions::default()
367    }
368}
369
370/// A segment file's vectors: mmapped when no keys are held, read whole and
371/// opened when they are, a plaintext file included.
372fn segment_vectors(path: &Path, name: &str, at_rest: Option<&SealKeys>) -> Result<usearch::Index> {
373    let index = usearch::Index::new(&segment_options())
374        .map_err(|e| anyhow!("creating usearch index: {e}"))?;
375    let kind = match at_rest {
376        Some(keys) => {
377            let bytes = sealed::read_file(path, Some(keys), name)?;
378            let kind = usearch::Index::metadata_from_buffer(&bytes)
379                .map_err(|e| anyhow!("reading usearch header at {}: {e}", path.display()))?
380                .quantization;
381            if kind == ScalarKind::F16 {
382                index
383                    .load_from_buffer(&bytes)
384                    .map_err(|e| anyhow!("loading usearch index from {}: {e}", path.display()))?;
385            }
386            Some(kind)
387        }
388        None => {
389            let kind = VectorIndex::on_disk_kind(path)?;
390            if kind == Some(ScalarKind::F16) {
391                let path_str = path.to_str().context("vector index path is not utf-8")?;
392                index
393                    .view(path_str)
394                    .map_err(|e| anyhow!("viewing usearch index at {path_str}: {e}"))?;
395            }
396            kind
397        }
398    };
399    if kind != Some(ScalarKind::F16) {
400        bail!("raw vector segment {} is not F16", path.display());
401    }
402    Ok(index)
403}
404
405/// Rebuilds the segment file from its vectors other than `keys`, sealed
406/// under `at_rest` when given. Every key in a segment is its number shifted
407/// left 32 bits plus an ordinal below the segment capacity, so the live keys
408/// are found by probing ordinals.
409fn rewrite_segment(
410    dir: &Path,
411    segment: u64,
412    keys: &HashSet<u64>,
413    at_rest: Option<&SealKeys>,
414) -> Result<()> {
415    let path = segment_path(dir, segment);
416    if !path.exists() {
417        return Ok(());
418    }
419    let name = segment_name(segment);
420    let source = segment_vectors(&path, &name, at_rest)?;
421    let live = source.size();
422    let rebuilt = usearch::Index::new(&segment_options())
423        .map_err(|e| anyhow!("creating usearch index: {e}"))?;
424    rebuilt
425        .reserve(live.max(1))
426        .map_err(|e| anyhow!("reserving usearch capacity: {e}"))?;
427    let mut seen = 0;
428    let mut vector = vec![0.0f32; EMBEDDING_DIM];
429    for ordinal in 0..SEGMENT_CAPACITY {
430        if seen == live {
431            break;
432        }
433        let key = (segment << 32) | ordinal;
434        if !source.contains(key) {
435            continue;
436        }
437        seen += 1;
438        if keys.contains(&key) {
439            continue;
440        }
441        source
442            .get(key, vector.as_mut_slice())
443            .map_err(|e| anyhow!("reading key {key} from {}: {e}", path.display()))?;
444        rebuilt
445            .add(key, vector.as_slice())
446            .map_err(|e| anyhow!("adding key {key}: {e}"))?;
447    }
448    if seen < live {
449        bail!(
450            "raw vector segment {} holds keys outside its range",
451            path.display()
452        );
453    }
454    let mut bytes = vec![0u8; rebuilt.serialized_length()];
455    rebuilt
456        .save_to_buffer(&mut bytes)
457        .map_err(|e| anyhow!("serializing usearch index: {e}"))?;
458    drop(rebuilt);
459    drop(source);
460    sealed::write_file(&path, at_rest, &name, &bytes)
461}
462
463/// Opens the tantivy index in `text_dir`, creating it when `create` is set
464/// and none is there. A sealed `meta.json` is opened first, so a missing or
465/// unknown key surfaces as a `SealError` rather than as tantivy's error.
466fn open_text_dir(
467    text_dir: &Path,
468    at_rest: Option<&SealKeys>,
469    create: bool,
470) -> Result<(Index, Option<SealedDirectory>)> {
471    let meta = text_dir.join(META_FILE);
472    if meta.exists() {
473        sealed::read_file(&meta, at_rest, META_FILE).context("opening raw text index")?;
474    }
475    let (directory, text_at_rest): (Box<dyn Directory>, _) = match at_rest {
476        Some(keys) => {
477            let sealed = SealedDirectory::open(text_dir, keys.clone())?;
478            if create {
479                sealed.remove_leftover_tmp_files()?;
480            }
481            (Box::new(sealed.clone()), Some(sealed))
482        }
483        None => (
484            Box::new(
485                MmapDirectory::open(text_dir)
486                    .with_context(|| format!("opening raw text index in {}", text_dir.display()))?,
487            ),
488            None,
489        ),
490    };
491    let index = if create && !Index::exists(directory.as_ref())? {
492        Index::create(directory, schema().0, IndexSettings::default())
493    } else {
494        Index::open(directory)
495    }
496    .context("opening raw text index")?;
497    Ok((index, text_at_rest))
498}
499
500/// Opens the text index, migrating one written with `legacy_schema` first
501/// when `writable`.
502fn open_text(
503    dir: &Path,
504    at_rest: Option<&SealKeys>,
505    writable: bool,
506) -> Result<(Index, Option<SealedDirectory>)> {
507    let text_dir = dir.join(TEXT_DIR);
508    let opened = open_text_dir(&text_dir, at_rest, writable)?;
509    let found = opened.0.schema();
510    if found == schema().0 {
511        return Ok(opened);
512    }
513    if found != legacy_schema() {
514        bail!("raw text index in {} has an unknown schema", dir.display());
515    }
516    if !writable {
517        bail!(
518            "raw text index in {} predates the current schema; open it for writing to migrate it",
519            dir.display()
520        );
521    }
522    drop(opened);
523    migrate_text(dir, at_rest)?;
524    let opened = open_text_dir(&text_dir, at_rest, true)?;
525    ensure!(
526        opened.0.schema() == schema().0,
527        "raw text index in {} did not migrate",
528        dir.display()
529    );
530    Ok(opened)
531}
532
533fn migrate_text(dir: &Path, at_rest: Option<&SealKeys>) -> Result<()> {
534    build_migrated_text(dir, at_rest)?;
535    sealed::write_file(
536        &dir.join(MIGRATION_JOURNAL),
537        at_rest,
538        MIGRATION_JOURNAL,
539        b"",
540    )?;
541    finish_text_swap(dir)
542}
543
544/// Copies every live document of the legacy text index into a fresh one
545/// with the current schema beside it, reindexing each passage from its
546/// stored text, which `passage_document` keeps only for a transcript's.
547fn build_migrated_text(dir: &Path, at_rest: Option<&SealKeys>) -> Result<()> {
548    let migrated_dir = dir.join(MIGRATED_TEXT_DIR);
549    if migrated_dir.exists() {
550        std::fs::remove_dir_all(&migrated_dir)
551            .with_context(|| format!("removing {}", migrated_dir.display()))?;
552    }
553    std::fs::create_dir_all(&migrated_dir)
554        .with_context(|| format!("creating {}", migrated_dir.display()))?;
555    let (legacy, _) = open_text_dir(&dir.join(TEXT_DIR), at_rest, false)?;
556    let (migrated, _) = open_text_dir(&migrated_dir, at_rest, true)?;
557    let legacy_schema = legacy.schema();
558    let legacy_field = |name: &str| {
559        legacy_schema
560            .get_field(name)
561            .with_context(|| format!("the legacy raw text index has no {name} field"))
562    };
563    let (key, body, source, project, memory, at) = (
564        legacy_field("key")?,
565        legacy_field("body")?,
566        legacy_field("source")?,
567        legacy_field("project")?,
568        legacy_field("memory")?,
569        legacy_field("at")?,
570    );
571    let (_, fields) = schema();
572    let reader: IndexReader = legacy
573        .reader_builder()
574        .reload_policy(ReloadPolicy::Manual)
575        .try_into()
576        .context("creating legacy raw text reader")?;
577    let searcher = reader.searcher();
578    let mut writer: IndexWriter = migrated
579        .writer_with_num_threads(1, WRITER_HEAP_BYTES)
580        .context("creating migrated raw text writer")?;
581    let mut copied = 0u64;
582    for (ord, segment) in searcher.segment_readers().iter().enumerate() {
583        for doc_id in segment.doc_ids_alive() {
584            let old: TantivyDocument = searcher.doc(DocAddress::new(ord as u32, doc_id))?;
585            let text = |field| {
586                old.get_first(field)
587                    .and_then(|v| v.as_str())
588                    .map(str::to_string)
589            };
590            let source_key = text(source).context("a raw text document has no source")?;
591            let document = match old.get_first(key).and_then(|v| v.as_u64()) {
592                Some(key) => passage_document(
593                    &fields,
594                    key,
595                    &source_key,
596                    &RawPassage {
597                        text: text(body).unwrap_or_default(),
598                        at: old
599                            .get_first(at)
600                            .and_then(|v| v.as_i64())
601                            .and_then(|s| DateTime::from_timestamp(s, 0)),
602                        project: text(project),
603                        memory: text(memory).and_then(|m| Uuid::parse_str(&m).ok()),
604                    },
605                ),
606                None => doc!(fields.source => source_key.as_str()),
607            };
608            writer.add_document(document)?;
609            copied += 1;
610        }
611    }
612    writer.commit().context("migrated raw text commit")?;
613    writer
614        .wait_merging_threads()
615        .context("waiting for migrated raw text merges")?;
616    let reader: IndexReader = migrated
617        .reader_builder()
618        .reload_policy(ReloadPolicy::Manual)
619        .try_into()
620        .context("creating migrated raw text reader")?;
621    let landed = reader.searcher().num_docs();
622    ensure!(
623        landed == copied && copied == searcher.num_docs(),
624        "raw text migration copied {copied} of {} documents and {landed} landed",
625        searcher.num_docs()
626    );
627    Ok(())
628}
629
630/// Moves a migrated text index into place once the journal says it is
631/// whole, or drops a copy a crash cut short. Each step leaves a state this
632/// recognises, so a crash at any point is finished at the next open.
633fn finish_text_swap(dir: &Path) -> Result<()> {
634    let text = dir.join(TEXT_DIR);
635    let migrated = dir.join(MIGRATED_TEXT_DIR);
636    let replaced = dir.join(REPLACED_TEXT_DIR);
637    let journal = dir.join(MIGRATION_JOURNAL);
638    let rename = |from: &Path, to: &Path| {
639        std::fs::rename(from, to)
640            .with_context(|| format!("renaming {} to {}", from.display(), to.display()))
641    };
642    if journal.exists() {
643        if migrated.exists() {
644            if text.exists() {
645                if replaced.exists() {
646                    std::fs::remove_dir_all(&replaced)
647                        .with_context(|| format!("removing {}", replaced.display()))?;
648                }
649                rename(&text, &replaced)?;
650            }
651            rename(&migrated, &text)?;
652            sync_dir(dir)?;
653        }
654    } else if migrated.exists() {
655        std::fs::remove_dir_all(&migrated)
656            .with_context(|| format!("removing {}", migrated.display()))?;
657    }
658    if replaced.exists() {
659        std::fs::remove_dir_all(&replaced)
660            .with_context(|| format!("removing {}", replaced.display()))?;
661    }
662    if journal.exists() {
663        std::fs::remove_file(&journal)
664            .with_context(|| format!("removing {}", journal.display()))?;
665        sync_dir(dir)?;
666    }
667    Ok(())
668}
669
670impl SealedRewrite {
671    pub fn is_empty(&self) -> bool {
672        self.keys.is_empty()
673    }
674
675    /// Rewrites the segment files without the withdrawn keys, sealed under
676    /// the index's current keys when it has any. Segments read before keep
677    /// the old contents until `RawIndex::reload_sealed`, so it needs no lock
678    /// on the index.
679    pub fn run(&self) -> Result<()> {
680        if self.keys.is_empty() {
681            return Ok(());
682        }
683        let at_rest = lock_keys(&self.at_rest);
684        let mut by_segment: BTreeMap<u64, HashSet<u64>> = BTreeMap::new();
685        for key in &self.keys {
686            by_segment.entry(key >> 32).or_default().insert(*key);
687        }
688        for (segment, keys) in &by_segment {
689            rewrite_segment(&self.dir, *segment, keys, at_rest.as_ref())?;
690        }
691        let journal = self.dir.join(REWRITE_JOURNAL);
692        if journal.exists() {
693            std::fs::remove_file(&journal)
694                .with_context(|| format!("removing {}", journal.display()))?;
695        }
696        Ok(())
697    }
698}
699
700impl Compaction {
701    pub fn wait(self) -> Result<()> {
702        match self.merge {
703            Some(merge) => merge.wait().map(|_| ()).context("raw text merge"),
704            None => Ok(()),
705        }
706    }
707}
708
709impl RawIndex {
710    /// Opens or creates the index for writing: every full segment as a view,
711    /// the last one loaded so passages can be added to it. A segment rewrite
712    /// or a text migration a crash interrupted is finished first.
713    pub fn open(dir: &Path) -> Result<Self> {
714        Self::open_writable(dir, None)
715    }
716
717    /// Same as `open`, with every file read and written sealed under `keys`
718    /// and the full segments loaded into RAM. Files written before sealing
719    /// open as they are until `reseal`.
720    pub fn open_keyed(dir: &Path, keys: &SealKeys) -> Result<Self> {
721        Self::open_writable(dir, Some(keys))
722    }
723
724    fn open_writable(dir: &Path, at_rest: Option<&SealKeys>) -> Result<Self> {
725        finish_text_swap(dir)?;
726        std::fs::create_dir_all(dir.join(TEXT_DIR))
727            .with_context(|| format!("creating raw index dir {}", dir.display()))?;
728        remove_scratch_files(dir)?;
729        let keys: AtRestKeys = Arc::new(Mutex::new(at_rest.cloned()));
730        SealedRewrite {
731            dir: dir.to_path_buf(),
732            keys: read_journal(dir, at_rest)?,
733            at_rest: keys.clone(),
734        }
735        .run()?;
736        let segments = segment_files(dir)?;
737        let mut full_segments = Vec::new();
738        let mut active = None;
739        for (i, n) in segments.iter().enumerate() {
740            let path = segment_path(dir, *n);
741            if i + 1 == segments.len() {
742                let loaded = open_writable_segment(&path, at_rest)?;
743                if (loaded.len() as u64) < SEGMENT_CAPACITY {
744                    active = Some((*n, loaded));
745                    continue;
746                }
747            }
748            full_segments.push((*n, open_full_segment(&path, at_rest)?));
749        }
750        let (segment, active) = match active {
751            Some(found) => found,
752            None => {
753                let next = segments.last().map_or(0, |n| n + 1);
754                (
755                    next,
756                    open_writable_segment(&segment_path(dir, next), at_rest)?,
757                )
758            }
759        };
760        let mut index = Self::with_text(dir, full_segments, Some(active), keys, true)?;
761        index.active_segment = segment;
762        index.next_ordinal = index.ordinal_after(segment)?;
763        Ok(index)
764    }
765
766    /// Opens an existing index for search only: every segment is a view and
767    /// no tantivy writer is ever created.
768    pub fn open_readonly(dir: &Path) -> Result<Self> {
769        Self::open_search_only(dir, None)
770    }
771
772    /// Same as `open_readonly` over an index sealed under `keys`; the
773    /// segments load into RAM, since a sealed file cannot be mmapped.
774    pub fn open_readonly_keyed(dir: &Path, keys: &SealKeys) -> Result<Self> {
775        Self::open_search_only(dir, Some(keys))
776    }
777
778    fn open_search_only(dir: &Path, at_rest: Option<&SealKeys>) -> Result<Self> {
779        let full_segments = segment_files(dir)?
780            .into_iter()
781            .map(|n| open_full_segment(&segment_path(dir, n), at_rest).map(|s| (n, s)))
782            .collect::<Result<Vec<_>>>()?;
783        let keys = Arc::new(Mutex::new(at_rest.cloned()));
784        Self::with_text(dir, full_segments, None, keys, false)
785    }
786
787    fn with_text(
788        dir: &Path,
789        full_segments: Vec<(u64, VectorIndex)>,
790        active: Option<VectorIndex>,
791        at_rest: AtRestKeys,
792        writable: bool,
793    ) -> Result<Self> {
794        let keys = lock_keys(&at_rest).clone();
795        let (index, text_at_rest) = open_text(dir, keys.as_ref(), writable)?;
796        let (_, fields) = schema();
797        let reader = index
798            .reader_builder()
799            .reload_policy(ReloadPolicy::Manual)
800            .try_into()
801            .context("creating raw text reader")?;
802        Ok(Self {
803            dir: dir.to_path_buf(),
804            full_segments,
805            active,
806            active_segment: 0,
807            next_ordinal: 0,
808            segment_capacity: SEGMENT_CAPACITY,
809            index,
810            reader,
811            writer: None,
812            fields,
813            writable,
814            text_bytes: dir_bytes(&dir.join(TEXT_DIR)),
815            pending_texts: HashSet::new(),
816            pending_sources: HashSet::new(),
817            purge_generation: 0,
818            at_rest,
819            text_at_rest,
820        })
821    }
822
823    /// The keys the index is sealed under at rest; None when it was opened
824    /// without keys.
825    pub fn seal_keys(&self) -> Option<SealKeys> {
826        lock_keys(&self.at_rest).clone()
827    }
828
829    /// Files written from now on are sealed under `keys`' current key, and
830    /// files under its retired keys still open. A writable index opened
831    /// without keys is committed and reopened under them, so `reseal` can
832    /// seal what it wrote in plaintext; a read-only one errors.
833    pub fn set_seal_keys(&mut self, keys: SealKeys) -> Result<()> {
834        let Some(text) = &self.text_at_rest else {
835            return self.reopen_keyed(&keys);
836        };
837        text.set_keys(keys.clone());
838        if let Some(active) = self.active.as_mut() {
839            active.set_seal_keys(keys.clone());
840        }
841        for (_, segment) in &mut self.full_segments {
842            segment.set_seal_keys(keys.clone());
843        }
844        *lock_keys(&self.at_rest) = Some(keys);
845        Ok(())
846    }
847
848    fn reopen_keyed(&mut self, keys: &SealKeys) -> Result<()> {
849        if !self.writable {
850            bail!(
851                "raw index at {} was opened read-only without keys; reopen it keyed to seal it",
852                self.dir.display()
853            );
854        }
855        let shared = self.at_rest.clone();
856        let mut held = lock_keys(&shared);
857        self.commit()?;
858        if let Some(writer) = self.writer.take() {
859            writer
860                .wait_merging_threads()
861                .context("waiting for raw text merges")?;
862        }
863        let generation = self.purge_generation + 1;
864        let mut reopened = Self::open_keyed(&self.dir.clone(), keys)?;
865        *held = Some(keys.clone());
866        reopened.at_rest = shared.clone();
867        reopened.purge_generation = generation;
868        drop(held);
869        *self = reopened;
870        Ok(())
871    }
872
873    /// Commits, then rewrites every file of the index not sealed under the
874    /// current key (plaintext or under a retired key) and removes the
875    /// scratch files a killed write left; returns how many were rewritten.
876    /// The tantivy writer is closed first, once its merges finish.
877    pub fn reseal(&mut self) -> Result<usize> {
878        self.require_writable()?;
879        let Some(text) = self.text_at_rest.clone() else {
880            bail!(
881                "raw index at {} was opened without keys; reopen it keyed to seal it",
882                self.dir.display()
883            );
884        };
885        self.commit()?;
886        if let Some(writer) = self.writer.take() {
887            writer
888                .wait_merging_threads()
889                .context("waiting for raw text merges")?;
890        }
891        let mut rewritten = text.reseal_all()?;
892        let at_rest = self.at_rest.clone();
893        let keys = lock_keys(&at_rest);
894        let keys = keys
895            .as_ref()
896            .context("a raw index with a sealed text directory holds its keys")?;
897        for name in top_level_files(&self.dir)? {
898            let path = self.dir.join(&name);
899            if is_scratch_file(&name) {
900                std::fs::remove_file(&path)
901                    .with_context(|| format!("removing {}", path.display()))?;
902            } else if sealed::reseal_file(&path, keys, &name)? {
903                rewritten += 1;
904            }
905        }
906        Ok(rewritten)
907    }
908
909    /// The first ordinal of `segment` no vector or passage has used: a
910    /// removal shrinks the loaded segment's length, so the length alone
911    /// could hand out a key a live passage still holds.
912    fn ordinal_after(&self, segment: u64) -> Result<u64> {
913        let mut next = self.active.as_ref().map_or(0, |a| a.len() as u64);
914        let low = (segment << 32).to_be_bytes();
915        let high = ((segment + 1) << 32).to_be_bytes();
916        for reader in self.reader.searcher().segment_readers() {
917            let inverted = reader.inverted_index(self.fields.key)?;
918            let mut terms = inverted.terms().range().ge(low).lt(high).into_stream()?;
919            while terms.advance() {
920                if let Ok(bytes) = <[u8; 8]>::try_from(terms.key()) {
921                    next = next.max((u64::from_be_bytes(bytes) & u64::from(u32::MAX)) + 1);
922                }
923            }
924        }
925        Ok(next)
926    }
927
928    fn segments(&self) -> impl Iterator<Item = &VectorIndex> {
929        self.full_segments
930            .iter()
931            .map(|(_, segment)| segment)
932            .chain(self.active.iter())
933    }
934
935    pub fn len(&self) -> usize {
936        self.segments().map(VectorIndex::len).sum()
937    }
938
939    pub fn is_empty(&self) -> bool {
940        self.len() == 0
941    }
942
943    /// An upper bound on the bytes this index can keep resident: the loaded
944    /// segments and the views' own bookkeeping, plus the size of the text
945    /// index's files as of the last open or commit. Does no I/O.
946    pub fn resident_bytes(&self) -> u64 {
947        let vectors: usize = self.segments().map(VectorIndex::memory_usage).sum();
948        vectors as u64 + self.text_bytes
949    }
950
951    /// Whether a passage with this text is indexed or waiting to be.
952    pub fn has_text(&self, text: &str) -> bool {
953        let hash = text_hash(text);
954        if self.pending_texts.contains(&hash) {
955            return true;
956        }
957        let term = Term::from_field_text(self.fields.text_hash, &hash);
958        holds_live(&self.reader.searcher(), term).unwrap_or(false)
959    }
960
961    /// Whether `source` was indexed, even if it held no passage.
962    pub fn has_source(&self, source: &RawOrigin) -> bool {
963        let key = source.source_key();
964        if self.pending_sources.contains(&key) {
965            return true;
966        }
967        let term = Term::from_field_text(self.fields.source, &key);
968        holds_live(&self.reader.searcher(), term).unwrap_or(false)
969    }
970
971    /// Changes whenever passages are withdrawn or rekeyed, so a caller that
972    /// filtered passages by `has_text` before releasing its lock can tell
973    /// that a text it skipped may no longer be indexed.
974    pub fn purge_generation(&self) -> u64 {
975        self.purge_generation
976    }
977
978    /// Indexes one source's passages, each with its embedding, skipping any
979    /// whose text is already indexed from another source. A source with no
980    /// new passage is still recorded, so it is not read again. Nothing is
981    /// searchable until `commit`.
982    pub fn add_source(
983        &mut self,
984        origin: &RawOrigin,
985        passages: &[(RawPassage, Vec<f32>)],
986    ) -> Result<usize> {
987        self.require_writable()?;
988        let source = origin.source_key();
989        let searcher = self.reader.searcher();
990        let mut added = 0;
991        for (passage, embedding) in passages {
992            if embedding.len() != EMBEDDING_DIM || passage.text.trim().is_empty() {
993                continue;
994            }
995            let hash = text_hash(&passage.text);
996            let term = Term::from_field_text(self.fields.text_hash, &hash);
997            if self.pending_texts.contains(&hash) || holds_live(&searcher, term)? {
998                continue;
999            }
1000            self.pending_texts.insert(hash);
1001            let key = self.next_key()?;
1002            self.active_mut()?.upsert(key, embedding)?;
1003            let document = passage_document(&self.fields, key, &source, passage);
1004            self.writer_mut()?.add_document(document)?;
1005            added += 1;
1006        }
1007        self.pending_sources.insert(source.clone());
1008        if added == 0 {
1009            let marker = doc!(self.fields.source => source.as_str());
1010            self.writer_mut()?.add_document(marker)?;
1011        }
1012        Ok(added)
1013    }
1014
1015    /// Saves the vectors, then commits the text, so a crash between the two
1016    /// leaves vectors whose keys resolve to nothing rather than text without
1017    /// vectors.
1018    pub fn commit(&mut self) -> Result<()> {
1019        self.require_writable()?;
1020        if let Some(active) = &self.active {
1021            active.save()?;
1022        }
1023        if let Some(writer) = self.writer.as_mut() {
1024            writer.commit().context("raw text commit")?;
1025        }
1026        self.reader.reload().context("raw text reader reload")?;
1027        self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
1028        self.pending_texts.clear();
1029        self.pending_sources.clear();
1030        Ok(())
1031    }
1032
1033    /// Commits without letting the commit start a background merge, which
1034    /// would hold segments a following `start_compaction` needs.
1035    pub fn commit_without_merging(&mut self) -> Result<()> {
1036        self.require_writable()?;
1037        self.writer_mut()?.set_merge_policy(Box::new(NoMergePolicy));
1038        let committed = self.commit();
1039        if let Some(writer) = &self.writer {
1040            writer.set_merge_policy(Box::new(DefaultMergePolicy::default()));
1041        }
1042        committed
1043    }
1044
1045    /// Picks up segments and text another process committed.
1046    pub fn reload(&mut self) -> Result<()> {
1047        if self.writable {
1048            return Ok(());
1049        }
1050        let at_rest = self.seal_keys();
1051        let segments = segment_files(&self.dir)?;
1052        let mut full_segments = Vec::with_capacity(segments.len());
1053        for n in segments {
1054            full_segments.push((
1055                n,
1056                open_full_segment(&segment_path(&self.dir, n), at_rest.as_ref())?,
1057            ));
1058        }
1059        self.full_segments = full_segments;
1060        self.reader.reload().context("raw text reader reload")?;
1061        self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
1062        Ok(())
1063    }
1064
1065    /// Keys and cosine similarity, best first, across every segment; with a
1066    /// project, only passages filed under it.
1067    pub fn vector_search(
1068        &self,
1069        embedding: &[f32],
1070        k: usize,
1071        project: Option<&str>,
1072    ) -> Result<Vec<(u64, f32)>> {
1073        let fetch = if project.is_some() { k * OVERFETCH } else { k };
1074        let mut hits = Vec::new();
1075        for segment in self.segments() {
1076            if segment.len() > 0 {
1077                hits.extend(segment.search(embedding, fetch)?);
1078            }
1079        }
1080        hits.sort_by(|a, b| b.1.total_cmp(&a.1));
1081        let mut out = Vec::with_capacity(k);
1082        for (key, score) in hits {
1083            if out.len() == k {
1084                break;
1085            }
1086            let Some(hit) = self.passage(key)? else {
1087                continue;
1088            };
1089            if project.is_none() || hit.project.as_deref() == project {
1090                out.push((key, score));
1091            }
1092        }
1093        Ok(out)
1094    }
1095
1096    /// Keys and BM25 score, best first; with a project, only passages filed
1097    /// under it.
1098    pub fn bm25_search(
1099        &self,
1100        query_text: &str,
1101        k: usize,
1102        project: Option<&str>,
1103    ) -> Result<Vec<(u64, f32)>> {
1104        let searcher = self.reader.searcher();
1105        let parser = QueryParser::for_index(&self.index, vec![self.fields.body]);
1106        let (text_query, _errors) = parser.parse_query_lenient(query_text);
1107        let query: Box<dyn Query> = match project {
1108            Some(project) => Box::new(BooleanQuery::new(vec![
1109                (Occur::Must, text_query),
1110                (
1111                    Occur::Must,
1112                    Box::new(TermQuery::new(
1113                        Term::from_field_text(self.fields.project, project),
1114                        IndexRecordOption::Basic,
1115                    )),
1116                ),
1117            ])),
1118            None => text_query,
1119        };
1120        let top = searcher.search(&query, &TopDocs::with_limit(k.max(1)).order_by_score())?;
1121        let mut out = Vec::with_capacity(top.len());
1122        for (score, address) in top {
1123            let document: TantivyDocument = searcher.doc(address)?;
1124            if let Some(key) = document.get_first(self.fields.key).and_then(|v| v.as_u64()) {
1125                out.push((key, score));
1126            }
1127        }
1128        Ok(out)
1129    }
1130
1131    pub fn passage(&self, key: u64) -> Result<Option<RawHit>> {
1132        let searcher = self.reader.searcher();
1133        let query = TermQuery::new(
1134            Term::from_field_u64(self.fields.key, key),
1135            IndexRecordOption::Basic,
1136        );
1137        let Some((_, address)) = searcher
1138            .search(&query, &TopDocs::with_limit(1).order_by_score())?
1139            .into_iter()
1140            .next()
1141        else {
1142            return Ok(None);
1143        };
1144        let document: TantivyDocument = searcher.doc(address)?;
1145        Ok(self.hit(&document))
1146    }
1147
1148    fn hit(&self, document: &TantivyDocument) -> Option<RawHit> {
1149        let text = |field| {
1150            document
1151                .get_first(field)
1152                .and_then(|v| v.as_str())
1153                .map(str::to_string)
1154        };
1155        let key = document
1156            .get_first(self.fields.key)
1157            .and_then(|v| v.as_u64())?;
1158        let origin = text(self.fields.source)
1159            .as_deref()
1160            .and_then(RawOrigin::parse)?;
1161        Some(RawHit {
1162            key,
1163            text: text(self.fields.text).unwrap_or_default(),
1164            text_hash: text(self.fields.text_hash).unwrap_or_default(),
1165            at: document
1166                .get_first(self.fields.at)
1167                .and_then(|v| v.as_i64())
1168                .and_then(|s| DateTime::from_timestamp(s, 0)),
1169            project: text(self.fields.project),
1170            memory: text(self.fields.memory).and_then(|m| Uuid::parse_str(&m).ok()),
1171            origin,
1172        })
1173    }
1174
1175    /// Every source the committed index records, including any whose
1176    /// removal has not been compacted away yet.
1177    pub fn sources(&self) -> Result<Vec<RawOrigin>> {
1178        Ok(self
1179            .terms(self.fields.source)?
1180            .iter()
1181            .filter_map(|key| RawOrigin::parse(key))
1182            .collect())
1183    }
1184
1185    /// The memories committed passages stand for.
1186    pub fn passage_memories(&self) -> Result<Vec<Uuid>> {
1187        Ok(self
1188            .terms(self.fields.memory)?
1189            .iter()
1190            .filter_map(|id| Uuid::parse_str(id).ok())
1191            .collect())
1192    }
1193
1194    /// Every committed passage of `sources`, standing for one of `memories`
1195    /// or with the text of one of `texts`, each once.
1196    pub fn matching(
1197        &self,
1198        sources: &[RawOrigin],
1199        memories: &[Uuid],
1200        texts: &[&str],
1201    ) -> Result<Vec<RawHit>> {
1202        let hashes: BTreeSet<String> = texts.iter().map(|t| text_hash(t)).collect();
1203        let terms = sources
1204            .iter()
1205            .map(|s| Term::from_field_text(self.fields.source, &s.source_key()))
1206            .chain(
1207                memories
1208                    .iter()
1209                    .map(|m| Term::from_field_text(self.fields.memory, &m.to_string())),
1210            )
1211            .chain(
1212                hashes
1213                    .iter()
1214                    .map(|h| Term::from_field_text(self.fields.text_hash, h)),
1215            );
1216        let searcher = self.reader.searcher();
1217        let mut seen = HashSet::new();
1218        let mut hits = Vec::new();
1219        for term in terms {
1220            let query = TermQuery::new(term, IndexRecordOption::Basic);
1221            for address in searcher.search(&query, &DocSetCollector)? {
1222                let document: TantivyDocument = searcher.doc(address)?;
1223                if let Some(hit) = self.hit(&document)
1224                    && seen.insert(hit.key)
1225                {
1226                    hits.push(hit);
1227                }
1228            }
1229        }
1230        Ok(hits)
1231    }
1232
1233    /// Withdraws the passages under `keys` at the next commit. Their vectors
1234    /// in the loaded segment are marked free; `seal_for_rewrite` takes them
1235    /// out of the segment files.
1236    pub fn withdraw(&mut self, keys: &[u64]) -> Result<()> {
1237        self.require_writable()?;
1238        if keys.is_empty() {
1239            return Ok(());
1240        }
1241        self.purge_generation += 1;
1242        let active_segment = self.active_segment;
1243        if let Some(active) = self.active.as_mut() {
1244            for key in keys.iter().filter(|k| *k >> 32 == active_segment) {
1245                active.remove(*key)?;
1246            }
1247        }
1248        let field = self.fields.key;
1249        let writer = self.writer_mut()?;
1250        for key in keys {
1251            writer.delete_term(Term::from_field_u64(field, *key));
1252        }
1253        Ok(())
1254    }
1255
1256    /// Files the passage under `key` under `origin` with `passage`'s
1257    /// metadata at the next commit, keeping its key and so its vector. The
1258    /// text is stored when `origin` is a transcript.
1259    pub fn rekey(&mut self, key: u64, origin: &RawOrigin, passage: &RawPassage) -> Result<()> {
1260        self.require_writable()?;
1261        self.purge_generation += 1;
1262        let document = passage_document(&self.fields, key, &origin.source_key(), passage);
1263        let term = Term::from_field_u64(self.fields.key, key);
1264        let writer = self.writer_mut()?;
1265        writer.delete_term(term);
1266        writer.add_document(document)?;
1267        Ok(())
1268    }
1269
1270    /// Drops every committed document recorded under `sources`, the marker
1271    /// of a source with no passage included, at the next commit.
1272    pub fn forget_sources(&mut self, sources: &[RawOrigin]) -> Result<()> {
1273        self.require_writable()?;
1274        let field = self.fields.source;
1275        let writer = self.writer_mut()?;
1276        for source in sources {
1277            writer.delete_term(Term::from_field_text(field, &source.source_key()));
1278        }
1279        Ok(())
1280    }
1281
1282    /// Journals `keys` for a rewrite of the segment files holding them,
1283    /// along with any a failed rewrite left. The loaded segment is closed
1284    /// first when it holds one, so every file the rewrite touches is one
1285    /// nothing writes to.
1286    pub fn seal_for_rewrite(&mut self, keys: &[u64]) -> Result<SealedRewrite> {
1287        self.require_writable()?;
1288        let at_rest = self.seal_keys();
1289        let mut pending = read_journal(&self.dir, at_rest.as_ref())?;
1290        pending.extend(keys.iter().copied());
1291        if pending.iter().any(|k| k >> 32 == self.active_segment) {
1292            self.close_active()?;
1293        }
1294        if !pending.is_empty() {
1295            write_journal(&self.dir, &pending, at_rest.as_ref())?;
1296        }
1297        Ok(SealedRewrite {
1298            dir: self.dir.clone(),
1299            keys: pending,
1300            at_rest: self.at_rest.clone(),
1301        })
1302    }
1303
1304    /// Reads back the segment files `rewrite` replaced.
1305    pub fn reload_sealed(&mut self, rewrite: &SealedRewrite) -> Result<()> {
1306        let segments: HashSet<u64> = rewrite.keys.iter().map(|k| k >> 32).collect();
1307        for (n, segment) in &mut self.full_segments {
1308            if segments.contains(n) {
1309                segment.reload_view()?;
1310            }
1311        }
1312        Ok(())
1313    }
1314
1315    /// Schedules a merge of every committed text segment, so the documents
1316    /// withdrawn so far leave the files once it finishes. Readers keep the
1317    /// old segments until `finish_compaction`.
1318    pub fn start_compaction(&mut self) -> Result<Compaction> {
1319        self.require_writable()?;
1320        let segments = self
1321            .index
1322            .searchable_segment_ids()
1323            .context("listing raw text segments")?;
1324        if segments.is_empty() {
1325            return Ok(Compaction { merge: None });
1326        }
1327        let merge = self.writer_mut()?.merge(&segments);
1328        Ok(Compaction { merge: Some(merge) })
1329    }
1330
1331    /// Deletes the files the merge replaced and moves readers onto the
1332    /// merged segment. A merge tantivy refused, because a background merge
1333    /// held one of the segments, is redone by `compact`.
1334    pub fn finish_compaction(&mut self, merged: Result<()>) -> Result<()> {
1335        self.require_writable()?;
1336        if let Err(refused) = merged {
1337            return self
1338                .compact()
1339                .with_context(|| format!("compacting after the merge failed: {refused:#}"));
1340        }
1341        if let Some(writer) = &self.writer {
1342            writer
1343                .garbage_collect_files()
1344                .wait()
1345                .context("raw text garbage collection")?;
1346        }
1347        self.reader.reload().context("raw text reader reload")?;
1348        self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
1349        Ok(())
1350    }
1351
1352    /// Commits, then merges the text index into fresh segments so removed
1353    /// passages no longer sit in its files, holding `self` throughout.
1354    pub fn compact(&mut self) -> Result<()> {
1355        self.commit()?;
1356        if let Some(writer) = self.writer.take() {
1357            writer
1358                .wait_merging_threads()
1359                .context("waiting for raw text merges")?;
1360        }
1361        let segments = self
1362            .index
1363            .searchable_segment_ids()
1364            .context("listing raw text segments")?;
1365        let writer = self.writer_mut()?;
1366        if !segments.is_empty() {
1367            writer.merge(&segments).wait().context("raw text merge")?;
1368        }
1369        writer
1370            .garbage_collect_files()
1371            .wait()
1372            .context("raw text garbage collection")?;
1373        self.reader.reload().context("raw text reader reload")?;
1374        self.text_bytes = dir_bytes(&self.dir.join(TEXT_DIR));
1375        Ok(())
1376    }
1377
1378    fn terms(&self, field: Field) -> Result<Vec<String>> {
1379        let searcher = self.reader.searcher();
1380        let mut found = std::collections::BTreeSet::new();
1381        for segment in searcher.segment_readers() {
1382            let inverted = segment.inverted_index(field)?;
1383            let mut stream = inverted.terms().stream()?;
1384            while stream.advance() {
1385                found.insert(String::from_utf8_lossy(stream.key()).into_owned());
1386            }
1387        }
1388        Ok(found.into_iter().collect())
1389    }
1390
1391    fn next_key(&mut self) -> Result<u64> {
1392        if self.next_ordinal >= self.segment_capacity {
1393            self.close_active()?;
1394        }
1395        let ordinal = self.next_ordinal;
1396        self.next_ordinal += 1;
1397        Ok((self.active_segment << 32) | ordinal)
1398    }
1399
1400    /// Saves the loaded segment, reopens it read-only as a full segment and
1401    /// starts the next one.
1402    fn close_active(&mut self) -> Result<()> {
1403        let active = self
1404            .active
1405            .take()
1406            .context("raw index has no active segment")?;
1407        active.save()?;
1408        drop(active);
1409        let at_rest = self.seal_keys();
1410        let path = segment_path(&self.dir, self.active_segment);
1411        self.full_segments.push((
1412            self.active_segment,
1413            open_full_segment(&path, at_rest.as_ref())?,
1414        ));
1415        self.active_segment += 1;
1416        self.next_ordinal = 0;
1417        self.active = Some(open_writable_segment(
1418            &segment_path(&self.dir, self.active_segment),
1419            at_rest.as_ref(),
1420        )?);
1421        Ok(())
1422    }
1423
1424    fn active_mut(&mut self) -> Result<&mut VectorIndex> {
1425        self.active
1426            .as_mut()
1427            .context("raw index has no active segment")
1428    }
1429
1430    fn writer_mut(&mut self) -> Result<&mut IndexWriter> {
1431        if self.writer.is_none() {
1432            self.writer = Some(
1433                self.index
1434                    .writer_with_num_threads(1, WRITER_HEAP_BYTES)
1435                    .context("creating raw text writer")?,
1436            );
1437        }
1438        Ok(self.writer.as_mut().expect("writer just created"))
1439    }
1440
1441    fn require_writable(&self) -> Result<()> {
1442        if self.writable {
1443            Ok(())
1444        } else {
1445            Err(anyhow!("raw index is open read-only"))
1446        }
1447    }
1448}
1449
1450#[cfg(test)]
1451mod tests {
1452    use super::*;
1453    use crate::keys::KeyGuard;
1454    use crate::sealed::{SealError, SealKey, SealKeyId, seal_error, sealed_key_id};
1455    use tantivy::{DocAddress, Document, IndexSettings};
1456
1457    fn unit(i: usize) -> Vec<f32> {
1458        let mut v = vec![0.0; EMBEDDING_DIM];
1459        v[i % EMBEDDING_DIM] = 1.0;
1460        v
1461    }
1462
1463    fn passage(text: &str, project: Option<&str>) -> RawPassage {
1464        RawPassage {
1465            text: text.to_string(),
1466            at: DateTime::from_timestamp(1_790_000_000, 0),
1467            project: project.map(str::to_string),
1468            memory: None,
1469        }
1470    }
1471
1472    fn transcript(sha: &str) -> RawOrigin {
1473        RawOrigin::Transcript {
1474            sha256: sha.to_string(),
1475        }
1476    }
1477
1478    fn purge(
1479        index: &mut RawIndex,
1480        sources: &[RawOrigin],
1481        memories: &[Uuid],
1482        texts: &[&str],
1483    ) -> usize {
1484        let keys: Vec<u64> = index
1485            .matching(sources, memories, texts)
1486            .unwrap()
1487            .iter()
1488            .map(|hit| hit.key)
1489            .collect();
1490        index.withdraw(&keys).unwrap();
1491        index.forget_sources(sources).unwrap();
1492        let rewrite = index.seal_for_rewrite(&keys).unwrap();
1493        index.commit().unwrap();
1494        rewrite.run().unwrap();
1495        index.reload_sealed(&rewrite).unwrap();
1496        keys.len()
1497    }
1498
1499    fn holds_key(index: &RawIndex, key: u64, near: &[f32]) -> bool {
1500        index
1501            .segments()
1502            .filter(|segment| segment.len() > 0)
1503            .any(|segment| {
1504                segment
1505                    .search(near, 10)
1506                    .unwrap()
1507                    .iter()
1508                    .any(|(k, _)| *k == key)
1509            })
1510    }
1511
1512    #[test]
1513    fn a_passage_keeps_the_raw_memory_it_maps_to_and_its_text_is_known_before_commit() {
1514        let dir = tempfile::tempdir().unwrap();
1515        let mut index = RawIndex::open(dir.path()).unwrap();
1516        let memory = Uuid::new_v4();
1517        let mut mapped = passage("the mirror job runs at four", None);
1518        mapped.memory = Some(memory);
1519        assert!(!index.has_text("the mirror job runs at four"));
1520
1521        index
1522            .add_source(&transcript("aa"), &[(mapped, unit(1))])
1523            .unwrap();
1524        assert!(index.has_text("the mirror job runs at four"));
1525        index.commit().unwrap();
1526
1527        assert!(index.has_text("  the mirror job runs at four \n"));
1528        assert!(!index.has_text("the mirror job runs at five"));
1529        let key = index.vector_search(&unit(1), 1, None).unwrap()[0].0;
1530        assert_eq!(index.passage(key).unwrap().unwrap().memory, Some(memory));
1531        assert!(index.resident_bytes() > 0);
1532    }
1533
1534    #[test]
1535    fn a_passage_is_found_by_vector_and_by_words_with_its_metadata() {
1536        let dir = tempfile::tempdir().unwrap();
1537        let mut index = RawIndex::open(dir.path()).unwrap();
1538        let source = transcript("aa");
1539        index
1540            .add_source(
1541                &source,
1542                &[
1543                    (
1544                        passage(
1545                            "the garden daemon listens on port 7200",
1546                            Some("-srv-garden"),
1547                        ),
1548                        unit(1),
1549                    ),
1550                    (passage("lunch was soup", Some("-srv-garden")), unit(2)),
1551                ],
1552            )
1553            .unwrap();
1554        index.commit().unwrap();
1555
1556        let by_vector = index.vector_search(&unit(1), 1, None).unwrap();
1557        let hit = index.passage(by_vector[0].0).unwrap().unwrap();
1558        assert_eq!(hit.text, "the garden daemon listens on port 7200");
1559        assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
1560        assert_eq!(hit.origin, source);
1561        assert_eq!(hit.at.unwrap().timestamp(), 1_790_000_000);
1562
1563        let by_words = index.bm25_search("daemon port", 5, None).unwrap();
1564        assert_eq!(by_words.len(), 1);
1565        assert_eq!(by_words[0].0, by_vector[0].0);
1566    }
1567
1568    #[test]
1569    fn a_project_filter_keeps_only_that_projects_passages() {
1570        let dir = tempfile::tempdir().unwrap();
1571        let mut index = RawIndex::open(dir.path()).unwrap();
1572        index
1573            .add_source(
1574                &transcript("aa"),
1575                &[
1576                    (
1577                        passage("deploy uses argo here", Some("-srv-garden")),
1578                        unit(1),
1579                    ),
1580                    (
1581                        passage("deploy uses argo there", Some("-srv-orchard")),
1582                        unit(1),
1583                    ),
1584                ],
1585            )
1586            .unwrap();
1587        index.commit().unwrap();
1588
1589        for hits in [
1590            index
1591                .vector_search(&unit(1), 5, Some("-srv-orchard"))
1592                .unwrap(),
1593            index
1594                .bm25_search("deploy argo", 5, Some("-srv-orchard"))
1595                .unwrap(),
1596        ] {
1597            assert_eq!(hits.len(), 1);
1598            let hit = index.passage(hits[0].0).unwrap().unwrap();
1599            assert_eq!(hit.project.as_deref(), Some("-srv-orchard"));
1600        }
1601    }
1602
1603    #[test]
1604    fn the_same_text_from_two_sources_is_stored_once_and_both_sources_are_recorded() {
1605        let dir = tempfile::tempdir().unwrap();
1606        let mut index = RawIndex::open(dir.path()).unwrap();
1607        let memory = RawOrigin::Memory { id: Uuid::new_v4() };
1608        let text = "the port moved to 7200";
1609        assert_eq!(
1610            index
1611                .add_source(&transcript("aa"), &[(passage(text, None), unit(1))])
1612                .unwrap(),
1613            1
1614        );
1615        index.commit().unwrap();
1616        assert_eq!(
1617            index
1618                .add_source(&memory, &[(passage(text, None), unit(1))])
1619                .unwrap(),
1620            0
1621        );
1622        index.commit().unwrap();
1623
1624        assert_eq!(index.len(), 1);
1625        assert!(index.has_source(&transcript("aa")));
1626        assert!(index.has_source(&memory));
1627        assert!(!index.has_source(&transcript("bb")));
1628    }
1629
1630    #[test]
1631    fn a_passage_without_a_full_embedding_is_skipped() {
1632        let dir = tempfile::tempdir().unwrap();
1633        let mut index = RawIndex::open(dir.path()).unwrap();
1634        let added = index
1635            .add_source(
1636                &transcript("aa"),
1637                &[(passage("short vector", None), vec![1.0; 3])],
1638            )
1639            .unwrap();
1640        index.commit().unwrap();
1641        assert_eq!(added, 0);
1642        assert!(index.is_empty());
1643        assert!(index.has_source(&transcript("aa")));
1644    }
1645
1646    #[test]
1647    fn full_segments_are_sealed_and_everything_survives_a_reopen() {
1648        let dir = tempfile::tempdir().unwrap();
1649        {
1650            let mut index = RawIndex::open(dir.path()).unwrap();
1651            index.segment_capacity = 2;
1652            let passages: Vec<_> = (0..5)
1653                .map(|i| (passage(&format!("passage number {i}"), None), unit(i)))
1654                .collect();
1655            index.add_source(&transcript("aa"), &passages).unwrap();
1656            index.commit().unwrap();
1657            assert_eq!(index.full_segments.len(), 2);
1658        }
1659        assert_eq!(segment_files(dir.path()).unwrap(), vec![0, 1, 2]);
1660
1661        let mut reopened = RawIndex::open(dir.path()).unwrap();
1662        assert_eq!(reopened.len(), 5);
1663        let hit = reopened.vector_search(&unit(3), 1, None).unwrap()[0].0;
1664        assert_eq!(
1665            reopened.passage(hit).unwrap().unwrap().text,
1666            "passage number 3"
1667        );
1668        reopened
1669            .add_source(
1670                &transcript("bb"),
1671                &[(passage("one more passage", None), unit(9))],
1672            )
1673            .unwrap();
1674        reopened.commit().unwrap();
1675
1676        let readonly = RawIndex::open_readonly(dir.path()).unwrap();
1677        assert_eq!(readonly.len(), 6);
1678        let hit = readonly.bm25_search("one more", 1, None).unwrap()[0].0;
1679        assert_eq!(
1680            readonly.passage(hit).unwrap().unwrap().text,
1681            "one more passage"
1682        );
1683    }
1684
1685    #[test]
1686    fn a_read_only_index_refuses_writes() {
1687        let dir = tempfile::tempdir().unwrap();
1688        RawIndex::open(dir.path()).unwrap().commit().unwrap();
1689        let mut readonly = RawIndex::open_readonly(dir.path()).unwrap();
1690        assert!(readonly.add_source(&transcript("aa"), &[]).is_err());
1691    }
1692
1693    #[test]
1694    fn origins_round_trip_through_their_source_key() {
1695        for origin in [transcript("ab12"), RawOrigin::Memory { id: Uuid::new_v4() }] {
1696            assert_eq!(RawOrigin::parse(&origin.source_key()), Some(origin));
1697        }
1698        assert_eq!(RawOrigin::parse("x:1"), None);
1699    }
1700
1701    #[test]
1702    fn removing_a_source_withdraws_its_passages_and_their_text_from_the_files() {
1703        let dir = tempfile::tempdir().unwrap();
1704        let mut index = RawIndex::open(dir.path()).unwrap();
1705        index
1706            .add_source(
1707                &transcript("aa"),
1708                &[
1709                    (passage("the gate code sits under a stone", None), unit(1)),
1710                    (
1711                        passage("a second line of the same transcript", None),
1712                        unit(2),
1713                    ),
1714                ],
1715            )
1716            .unwrap();
1717        index
1718            .add_source(
1719                &transcript("bb"),
1720                &[(passage("an unrelated line", None), unit(3))],
1721            )
1722            .unwrap();
1723        index.commit().unwrap();
1724        assert_eq!(index.sources().unwrap().len(), 2);
1725
1726        assert_eq!(purge(&mut index, &[transcript("aa")], &[], &[]), 2);
1727        index.commit().unwrap();
1728        index.compact().unwrap();
1729
1730        assert!(
1731            index
1732                .bm25_search("gate code stone", 5, None)
1733                .unwrap()
1734                .is_empty()
1735        );
1736        assert!(
1737            index
1738                .vector_search(&unit(1), 5, None)
1739                .unwrap()
1740                .iter()
1741                .all(|(k, _)| { index.passage(*k).unwrap().unwrap().text == "an unrelated line" })
1742        );
1743        assert!(!index.has_source(&transcript("aa")));
1744        assert!(!index.has_text("the gate code sits under a stone"));
1745        assert_eq!(index.sources().unwrap(), vec![transcript("bb")]);
1746        assert_eq!(index.len(), 1);
1747        let leftovers: Vec<String> = std::fs::read_dir(dir.path().join(TEXT_DIR))
1748            .unwrap()
1749            .filter_map(|e| std::fs::read(e.ok()?.path()).ok())
1750            .map(|b| String::from_utf8_lossy(&b).into_owned())
1751            .filter(|t| t.contains("stone"))
1752            .collect();
1753        assert!(
1754            leftovers.is_empty(),
1755            "the removed text left the index files"
1756        );
1757    }
1758
1759    #[test]
1760    fn removing_a_memory_withdraws_the_transcript_passages_that_stand_for_it() {
1761        let dir = tempfile::tempdir().unwrap();
1762        let mut index = RawIndex::open(dir.path()).unwrap();
1763        let gone = Uuid::new_v4();
1764        let mut mapped = passage("passage the memory stands for", None);
1765        mapped.memory = Some(gone);
1766        index
1767            .add_source(
1768                &transcript("aa"),
1769                &[
1770                    (mapped, unit(1)),
1771                    (passage("a passage of nobody", None), unit(2)),
1772                ],
1773            )
1774            .unwrap();
1775        index
1776            .add_source(
1777                &RawOrigin::Memory { id: gone },
1778                &[(passage("the archived memory itself", None), unit(3))],
1779            )
1780            .unwrap();
1781        index.commit().unwrap();
1782        assert_eq!(index.passage_memories().unwrap(), vec![gone]);
1783
1784        let removed = purge(&mut index, &[RawOrigin::Memory { id: gone }], &[gone], &[]);
1785        index.commit().unwrap();
1786
1787        assert_eq!(removed, 2);
1788        assert!(index.bm25_search("memory", 5, None).unwrap().is_empty());
1789        assert_eq!(index.bm25_search("nobody", 5, None).unwrap().len(), 1);
1790        assert_eq!(index.len(), 1);
1791    }
1792
1793    #[test]
1794    fn removing_by_text_withdraws_the_passage_whichever_source_holds_it() {
1795        let dir = tempfile::tempdir().unwrap();
1796        let mut index = RawIndex::open(dir.path()).unwrap();
1797        index
1798            .add_source(
1799                &transcript("old"),
1800                &[(passage("a line the user typed", None), unit(1))],
1801            )
1802            .unwrap();
1803        index
1804            .add_source(
1805                &transcript("new"),
1806                &[(passage("a later line", None), unit(2))],
1807            )
1808            .unwrap();
1809        index.commit().unwrap();
1810
1811        let removed = purge(
1812            &mut index,
1813            &[transcript("new")],
1814            &[],
1815            &["a line the user typed", "a later line"],
1816        );
1817        index.commit().unwrap();
1818
1819        assert_eq!(removed, 2);
1820        assert!(index.bm25_search("line", 5, None).unwrap().is_empty());
1821    }
1822
1823    #[test]
1824    fn a_passage_in_a_sealed_segment_stops_resolving_once_removed() {
1825        let dir = tempfile::tempdir().unwrap();
1826        let mut index = RawIndex::open(dir.path()).unwrap();
1827        index.segment_capacity = 1;
1828        index
1829            .add_source(
1830                &transcript("aa"),
1831                &[(passage("sealed away early", None), unit(1))],
1832            )
1833            .unwrap();
1834        index
1835            .add_source(
1836                &transcript("bb"),
1837                &[(passage("kept passage", None), unit(2))],
1838            )
1839            .unwrap();
1840        index.commit().unwrap();
1841        assert!(!index.full_segments.is_empty());
1842
1843        purge(&mut index, &[transcript("aa")], &[], &[]);
1844        index.commit().unwrap();
1845
1846        let hits = index.vector_search(&unit(1), 5, None).unwrap();
1847        assert!(
1848            hits.iter()
1849                .all(|(k, _)| { index.passage(*k).unwrap().unwrap().text == "kept passage" })
1850        );
1851    }
1852
1853    #[test]
1854    fn a_withdrawn_key_leaves_the_sealed_segment_file_that_held_it() {
1855        let dir = tempfile::tempdir().unwrap();
1856        let mut index = RawIndex::open(dir.path()).unwrap();
1857        index.segment_capacity = 1;
1858        index
1859            .add_source(
1860                &transcript("aa"),
1861                &[(passage("sealed away early", None), unit(1))],
1862            )
1863            .unwrap();
1864        index
1865            .add_source(
1866                &transcript("bb"),
1867                &[(passage("kept passage", None), unit(2))],
1868            )
1869            .unwrap();
1870        index.commit().unwrap();
1871        let key = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
1872        assert_eq!(key >> 32, 0);
1873        assert!(holds_key(&index, key, &unit(1)));
1874        let before = std::fs::metadata(segment_path(dir.path(), 0))
1875            .unwrap()
1876            .len();
1877
1878        purge(&mut index, &[transcript("aa")], &[], &[]);
1879
1880        assert!(!holds_key(&index, key, &unit(1)));
1881        drop(index);
1882        let after = std::fs::metadata(segment_path(dir.path(), 0))
1883            .unwrap()
1884            .len();
1885        assert!(after < before, "the embedding stayed in the file");
1886        assert!(!dir.path().join(REWRITE_JOURNAL).exists());
1887        let reopened = RawIndex::open(dir.path()).unwrap();
1888        assert!(!holds_key(&reopened, key, &unit(1)));
1889        assert_eq!(reopened.len(), 1);
1890        let hit = reopened.vector_search(&unit(2), 1, None).unwrap()[0].0;
1891        assert_eq!(reopened.passage(hit).unwrap().unwrap().text, "kept passage");
1892    }
1893
1894    #[test]
1895    fn a_rewrite_a_crash_interrupted_is_finished_at_open() {
1896        let dir = tempfile::tempdir().unwrap();
1897        let key = {
1898            let mut index = RawIndex::open(dir.path()).unwrap();
1899            index
1900                .add_source(
1901                    &transcript("aa"),
1902                    &[(passage("withdrawn before the crash", None), unit(1))],
1903                )
1904                .unwrap();
1905            index
1906                .add_source(
1907                    &transcript("bb"),
1908                    &[(passage("kept passage", None), unit(2))],
1909                )
1910                .unwrap();
1911            index.commit().unwrap();
1912            let key = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
1913            index.withdraw(&[key]).unwrap();
1914            index.forget_sources(&[transcript("aa")]).unwrap();
1915            index.seal_for_rewrite(&[key]).unwrap();
1916            index.commit().unwrap();
1917            key
1918        };
1919        assert!(dir.path().join(REWRITE_JOURNAL).exists());
1920
1921        let reopened = RawIndex::open(dir.path()).unwrap();
1922
1923        assert!(!dir.path().join(REWRITE_JOURNAL).exists());
1924        assert!(!holds_key(&reopened, key, &unit(1)));
1925        assert_eq!(reopened.len(), 1);
1926    }
1927
1928    #[test]
1929    fn a_key_withdrawn_from_the_loaded_segment_is_never_handed_out_again() {
1930        let dir = tempfile::tempdir().unwrap();
1931        let mut index = RawIndex::open(dir.path()).unwrap();
1932        let passages: Vec<_> = (0..3)
1933            .map(|i| (passage(&format!("passage number {i}"), None), unit(i)))
1934            .collect();
1935        index.add_source(&transcript("aa"), &passages).unwrap();
1936        index.commit().unwrap();
1937        let first = index.matching(&[], &[], &["passage number 0"]).unwrap()[0].key;
1938        index.withdraw(&[first]).unwrap();
1939        index.commit().unwrap();
1940
1941        index
1942            .add_source(&transcript("bb"), &[(passage("one more", None), unit(7))])
1943            .unwrap();
1944        index.commit().unwrap();
1945        drop(index);
1946        let mut reopened = RawIndex::open(dir.path()).unwrap();
1947        reopened
1948            .add_source(
1949                &transcript("cc"),
1950                &[(passage("and another", None), unit(8))],
1951            )
1952            .unwrap();
1953        reopened.commit().unwrap();
1954
1955        let all = reopened
1956            .matching(
1957                &[transcript("aa"), transcript("bb"), transcript("cc")],
1958                &[],
1959                &[],
1960            )
1961            .unwrap();
1962        assert_eq!(all.len(), 4);
1963        for (text, near) in [
1964            ("passage number 1", unit(1)),
1965            ("passage number 2", unit(2)),
1966            ("one more", unit(7)),
1967            ("and another", unit(8)),
1968        ] {
1969            let key = reopened.vector_search(&near, 1, None).unwrap()[0].0;
1970            assert_eq!(reopened.passage(key).unwrap().unwrap().text, text);
1971        }
1972    }
1973
1974    #[test]
1975    fn a_withdrawn_text_reads_as_not_indexed_before_any_merge() {
1976        let dir = tempfile::tempdir().unwrap();
1977        let mut index = RawIndex::open(dir.path()).unwrap();
1978        index
1979            .add_source(
1980                &transcript("aa"),
1981                &[
1982                    (
1983                        passage("the lantern hangs by the north door", None),
1984                        unit(1),
1985                    ),
1986                    (passage("the gate is painted red", None), unit(2)),
1987                ],
1988            )
1989            .unwrap();
1990        index.commit().unwrap();
1991        let generation = index.purge_generation();
1992        let key = index
1993            .matching(&[], &[], &["the lantern hangs by the north door"])
1994            .unwrap()[0]
1995            .key;
1996
1997        index.withdraw(&[key]).unwrap();
1998        index.commit_without_merging().unwrap();
1999
2000        let term = Term::from_field_text(
2001            index.fields.text_hash,
2002            &text_hash("the lantern hangs by the north door"),
2003        );
2004        assert_eq!(index.reader.searcher().doc_freq(&term).unwrap(), 1);
2005        assert_ne!(index.purge_generation(), generation);
2006        assert!(!index.has_text("the lantern hangs by the north door"));
2007        assert!(index.has_text("the gate is painted red"));
2008        assert!(index.has_source(&transcript("aa")));
2009        assert_eq!(
2010            index
2011                .add_source(
2012                    &transcript("bb"),
2013                    &[(
2014                        passage("the lantern hangs by the north door", None),
2015                        unit(1)
2016                    )],
2017                )
2018                .unwrap(),
2019            1
2020        );
2021
2022        let rest = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
2023        index.withdraw(&[rest]).unwrap();
2024        index.forget_sources(&[transcript("aa")]).unwrap();
2025        index.commit_without_merging().unwrap();
2026        assert!(!index.has_source(&transcript("aa")));
2027        assert!(index.has_source(&transcript("bb")));
2028    }
2029
2030    #[test]
2031    fn a_rekeyed_passage_keeps_its_vector_under_its_new_source() {
2032        let dir = tempfile::tempdir().unwrap();
2033        let mut index = RawIndex::open(dir.path()).unwrap();
2034        index
2035            .add_source(&transcript("gone"), &[(passage("yes", None), unit(1))])
2036            .unwrap();
2037        index.commit().unwrap();
2038        let key = index.matching(&[transcript("gone")], &[], &[]).unwrap()[0].key;
2039
2040        index
2041            .rekey(
2042                key,
2043                &transcript("kept"),
2044                &passage("yes", Some("-srv-garden")),
2045            )
2046            .unwrap();
2047        index.forget_sources(&[transcript("gone")]).unwrap();
2048        index.commit().unwrap();
2049        index.compact().unwrap();
2050
2051        let hit = index.vector_search(&unit(1), 1, None).unwrap()[0].0;
2052        assert_eq!(hit, key);
2053        let hit = index.passage(hit).unwrap().unwrap();
2054        assert_eq!(hit.origin, transcript("kept"));
2055        assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
2056        assert_eq!(index.sources().unwrap(), vec![transcript("kept")]);
2057        assert!(index.has_text("yes"));
2058    }
2059
2060    const MARKER: &str = "zanzibarquokka";
2061
2062    fn seal_key(byte: u8) -> SealKey {
2063        SealKey::from_bytes(KeyGuard::new([byte; 32])).unwrap()
2064    }
2065
2066    fn seal_keys(byte: u8) -> SealKeys {
2067        SealKeys::single(seal_key(byte))
2068    }
2069
2070    fn files_under(dir: &Path) -> Vec<PathBuf> {
2071        let mut out = Vec::new();
2072        for entry in std::fs::read_dir(dir).unwrap() {
2073            let path = entry.unwrap().path();
2074            let name = path.file_name().unwrap().to_string_lossy().into_owned();
2075            if path.is_dir() {
2076                out.extend(files_under(&path));
2077            } else if !(name.starts_with(".tantivy-") && name.ends_with(".lock")) {
2078                out.push(path);
2079            }
2080        }
2081        out
2082    }
2083
2084    fn assert_all_sealed_under(dir: &Path, id: SealKeyId) {
2085        let files = files_under(dir);
2086        assert!(files.iter().any(|p| p.starts_with(dir.join(TEXT_DIR))));
2087        assert!(
2088            files
2089                .iter()
2090                .any(|p| p.extension().is_some_and(|e| e == "usearch"))
2091        );
2092        for path in files {
2093            assert_eq!(
2094                sealed_key_id(&path).unwrap(),
2095                Some(id),
2096                "{} is not sealed under the current key",
2097                path.display()
2098            );
2099        }
2100    }
2101
2102    fn holds_bytes(dir: &Path, needle: &str) -> bool {
2103        files_under(dir).iter().any(|p| {
2104            std::fs::read(p)
2105                .unwrap()
2106                .windows(needle.len())
2107                .any(|w| w == needle.as_bytes())
2108        })
2109    }
2110
2111    fn spread(i: usize) -> Vec<f32> {
2112        let mut v = unit(i);
2113        v[(i + 1) % EMBEDDING_DIM] = 0.5;
2114        v
2115    }
2116
2117    fn ramp(i: usize) -> Vec<f32> {
2118        let mut v: Vec<f32> = (0..EMBEDDING_DIM)
2119            .map(|j| (j + 1) as f32 / EMBEDDING_DIM as f32)
2120            .collect();
2121        v[i] += 5.0;
2122        v
2123    }
2124
2125    fn populate(index: &mut RawIndex) {
2126        index.segment_capacity = 2;
2127        let passages: Vec<_> = (0..5)
2128            .map(|i| {
2129                (
2130                    passage(&format!("{MARKER} passage number {i}"), Some("-srv-garden")),
2131                    spread(i),
2132                )
2133            })
2134            .collect();
2135        index.add_source(&transcript("aa"), &passages).unwrap();
2136        let id = Uuid::new_v4();
2137        let mut memory = passage(&format!("{MARKER} archived memory"), None);
2138        memory.memory = Some(id);
2139        index
2140            .add_source(&RawOrigin::Memory { id }, &[(memory, spread(7))])
2141            .unwrap();
2142        index.commit().unwrap();
2143    }
2144
2145    type TopK = Vec<Vec<(u64, f32)>>;
2146
2147    fn top_k(index: &RawIndex) -> TopK {
2148        let mut out: TopK = (0..8)
2149            .map(|i| index.vector_search(&ramp(i), 4, None).unwrap())
2150            .collect();
2151        out.push(index.bm25_search(MARKER, 10, None).unwrap());
2152        out.push(
2153            index
2154                .bm25_search("number 3", 10, Some("-srv-garden"))
2155                .unwrap(),
2156        );
2157        assert!(out.iter().all(|hits| !hits.is_empty()));
2158        out
2159    }
2160
2161    fn stored_json(index: &RawIndex) -> String {
2162        let searcher = index.reader.searcher();
2163        let schema = index.index.schema();
2164        let mut out = String::new();
2165        for (ord, segment) in searcher.segment_readers().iter().enumerate() {
2166            for doc in 0..segment.max_doc() {
2167                let document: TantivyDocument =
2168                    searcher.doc(DocAddress::new(ord as u32, doc)).unwrap();
2169                out.push_str(&document.to_json(&schema));
2170                out.push('\n');
2171            }
2172        }
2173        out
2174    }
2175
2176    #[test]
2177    fn a_keyed_index_does_not_open_without_its_key_or_under_another() {
2178        let dir = tempfile::tempdir().unwrap();
2179        populate(&mut RawIndex::open_keyed(dir.path(), &seal_keys(1)).unwrap());
2180
2181        for opened in [
2182            RawIndex::open(dir.path()),
2183            RawIndex::open_readonly(dir.path()),
2184        ] {
2185            let err = opened.err().expect("a sealed index opened without keys");
2186            assert_eq!(seal_error(&err), Some(&SealError::KeyRequired), "{err:#}");
2187        }
2188        for opened in [
2189            RawIndex::open_keyed(dir.path(), &seal_keys(2)),
2190            RawIndex::open_readonly_keyed(dir.path(), &seal_keys(2)),
2191        ] {
2192            let err = opened
2193                .err()
2194                .expect("a sealed index opened under another key");
2195            assert!(
2196                matches!(seal_error(&err), Some(SealError::UnknownKey(_))),
2197                "{err:#}"
2198            );
2199        }
2200        let reopened = RawIndex::open_readonly_keyed(dir.path(), &seal_keys(1)).unwrap();
2201        assert_eq!(reopened.len(), 6);
2202        assert_eq!(
2203            reopened.seal_keys().unwrap().current_id(),
2204            seal_keys(1).current_id()
2205        );
2206    }
2207
2208    #[test]
2209    fn a_plaintext_index_opened_keyed_searches_the_same_and_reseal_seals_every_file() {
2210        let dir = tempfile::tempdir().unwrap();
2211        populate(&mut RawIndex::open(dir.path()).unwrap());
2212        let baseline = top_k(&RawIndex::open_readonly(dir.path()).unwrap());
2213
2214        let keys = seal_keys(3);
2215        let mut index = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2216        assert_eq!(top_k(&index), baseline);
2217        assert!(index.reseal().unwrap() > 0);
2218        assert_all_sealed_under(dir.path(), keys.current_id());
2219        assert_eq!(top_k(&index), baseline);
2220        assert_eq!(index.reseal().unwrap(), 0);
2221        drop(index);
2222
2223        assert_eq!(
2224            top_k(&RawIndex::open_readonly_keyed(dir.path(), &keys).unwrap()),
2225            baseline
2226        );
2227        let reopened = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2228        assert_eq!(top_k(&reopened), baseline);
2229        let hit = reopened.bm25_search("number 3", 1, None).unwrap()[0].0;
2230        assert_eq!(
2231            reopened.passage(hit).unwrap().unwrap().text,
2232            format!("{MARKER} passage number 3")
2233        );
2234    }
2235
2236    #[test]
2237    fn no_passage_text_reaches_a_file_of_a_keyed_index() {
2238        let dir = tempfile::tempdir().unwrap();
2239        let keys = seal_keys(4);
2240        let mut index = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2241        populate(&mut index);
2242        index.compact().unwrap();
2243
2244        assert!(!holds_bytes(dir.path(), MARKER));
2245        assert_all_sealed_under(dir.path(), keys.current_id());
2246        let hit = index.bm25_search("number 3", 1, None).unwrap()[0].0;
2247        assert_eq!(
2248            index.passage(hit).unwrap().unwrap().text,
2249            format!("{MARKER} passage number 3")
2250        );
2251    }
2252
2253    #[test]
2254    fn a_rotation_reseals_every_file_under_the_new_key_only() {
2255        let dir = tempfile::tempdir().unwrap();
2256        populate(&mut RawIndex::open_keyed(dir.path(), &SealKeys::single(seal_key(5))).unwrap());
2257        let mut index = RawIndex::open_keyed(dir.path(), &SealKeys::single(seal_key(5))).unwrap();
2258        let baseline = top_k(&index);
2259
2260        index
2261            .set_seal_keys(SealKeys::new(seal_key(6), vec![seal_key(5)]))
2262            .unwrap();
2263        assert!(index.reseal().unwrap() > 0);
2264
2265        assert_all_sealed_under(dir.path(), seal_key(6).id());
2266        assert_eq!(top_k(&index), baseline);
2267        index
2268            .add_source(
2269                &transcript("bb"),
2270                &[(passage("one more passage", None), unit(9))],
2271            )
2272            .unwrap();
2273        index.commit().unwrap();
2274        drop(index);
2275        let reopened = RawIndex::open_keyed(dir.path(), &SealKeys::single(seal_key(6))).unwrap();
2276        assert_eq!(reopened.len(), 7);
2277        assert_all_sealed_under(dir.path(), seal_key(6).id());
2278    }
2279
2280    #[test]
2281    fn a_rewrite_drops_withdrawn_vectors_from_sealed_segments_and_keeps_its_journal_sealed() {
2282        let dir = tempfile::tempdir().unwrap();
2283        let keys = seal_keys(7);
2284        let (gone, crashed) = {
2285            let mut index = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2286            index.segment_capacity = 1;
2287            for (sha, text, i) in [
2288                ("aa", "sealed away early", 1),
2289                ("bb", "withdrawn before the crash", 2),
2290                ("cc", "kept passage", 3),
2291            ] {
2292                index
2293                    .add_source(&transcript(sha), &[(passage(text, None), unit(i))])
2294                    .unwrap();
2295            }
2296            index.commit().unwrap();
2297            let gone = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
2298            assert!(holds_key(&index, gone, &unit(1)));
2299            purge(&mut index, &[transcript("aa")], &[], &[]);
2300            assert!(!holds_key(&index, gone, &unit(1)));
2301
2302            let crashed = index.matching(&[transcript("bb")], &[], &[]).unwrap()[0].key;
2303            index.withdraw(&[crashed]).unwrap();
2304            index.forget_sources(&[transcript("bb")]).unwrap();
2305            index.seal_for_rewrite(&[crashed]).unwrap();
2306            index.commit().unwrap();
2307            (gone, crashed)
2308        };
2309        let journal = dir.path().join(REWRITE_JOURNAL);
2310        assert_eq!(sealed_key_id(&journal).unwrap(), Some(keys.current_id()));
2311        let err = RawIndex::open(dir.path())
2312            .err()
2313            .expect("a sealed journal read without keys");
2314        assert_eq!(seal_error(&err), Some(&SealError::KeyRequired), "{err:#}");
2315        assert!(journal.exists());
2316
2317        let reopened = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2318
2319        assert!(!journal.exists());
2320        assert!(!holds_key(&reopened, gone, &unit(1)));
2321        assert!(!holds_key(&reopened, crashed, &unit(2)));
2322        assert_eq!(reopened.len(), 1);
2323        for n in segment_files(dir.path()).unwrap() {
2324            assert_eq!(
2325                sealed_key_id(&segment_path(dir.path(), n)).unwrap(),
2326                Some(keys.current_id())
2327            );
2328        }
2329        let hit = reopened.vector_search(&unit(3), 1, None).unwrap()[0].0;
2330        assert_eq!(reopened.passage(hit).unwrap().unwrap().text, "kept passage");
2331    }
2332
2333    #[test]
2334    fn a_rewrite_pending_when_the_index_is_reopened_keyed_writes_sealed() {
2335        let dir = tempfile::tempdir().unwrap();
2336        let keys = seal_keys(8);
2337        let mut index = RawIndex::open(dir.path()).unwrap();
2338        index.segment_capacity = 1;
2339        for (sha, text, i) in [
2340            ("aa", "withdrawn while unkeyed", 1),
2341            ("bb", "kept passage", 2),
2342        ] {
2343            index
2344                .add_source(&transcript(sha), &[(passage(text, None), unit(i))])
2345                .unwrap();
2346        }
2347        index.commit().unwrap();
2348        let gone = index.matching(&[transcript("aa")], &[], &[]).unwrap()[0].key;
2349        index.withdraw(&[gone]).unwrap();
2350        index.forget_sources(&[transcript("aa")]).unwrap();
2351        let rewrite = index.seal_for_rewrite(&[gone]).unwrap();
2352        index.commit().unwrap();
2353
2354        index.set_seal_keys(keys.clone()).unwrap();
2355        index.reseal().unwrap();
2356        rewrite.run().unwrap();
2357
2358        for n in segment_files(dir.path()).unwrap() {
2359            assert_eq!(
2360                sealed_key_id(&segment_path(dir.path(), n)).unwrap(),
2361                Some(keys.current_id()),
2362                "segment {n}"
2363            );
2364        }
2365        let reopened = RawIndex::open_keyed(dir.path(), &keys).unwrap();
2366        assert!(!holds_key(&reopened, gone, &unit(1)));
2367        assert_eq!(reopened.len(), 1);
2368    }
2369
2370    #[test]
2371    fn an_archived_memorys_passage_is_searchable_but_keeps_no_stored_text() {
2372        let dir = tempfile::tempdir().unwrap();
2373        let mut index = RawIndex::open(dir.path()).unwrap();
2374        let id = Uuid::new_v4();
2375        let text = "the quokka ferry leaves at dawn";
2376        let mut memory = passage(text, None);
2377        memory.memory = Some(id);
2378        index
2379            .add_source(&RawOrigin::Memory { id }, &[(memory.clone(), unit(1))])
2380            .unwrap();
2381        index
2382            .add_source(
2383                &transcript("aa"),
2384                &[(
2385                    passage("the lantern hangs by the north door", None),
2386                    unit(2),
2387                )],
2388            )
2389            .unwrap();
2390        index.commit().unwrap();
2391
2392        let stored = stored_json(&index);
2393        assert!(!stored.contains("quokka"), "{stored}");
2394        assert!(stored.contains("the lantern hangs by the north door"));
2395        let key = index.bm25_search("quokka ferry", 5, None).unwrap()[0].0;
2396        assert_eq!(index.vector_search(&unit(1), 1, None).unwrap()[0].0, key);
2397        let hit = index.passage(key).unwrap().unwrap();
2398        assert_eq!(hit.origin, RawOrigin::Memory { id });
2399        assert_eq!(hit.memory, Some(id));
2400        assert!(hit.text.is_empty());
2401        assert_eq!(hit.text_hash, text_hash(text));
2402        assert!(index.has_text(text));
2403        assert_eq!(index.matching(&[], &[], &[text]).unwrap()[0].key, key);
2404
2405        index
2406            .rekey(key, &transcript("bb"), &passage(text, None))
2407            .unwrap();
2408        index.commit().unwrap();
2409        assert_eq!(index.passage(key).unwrap().unwrap().text, text);
2410        index
2411            .rekey(key, &RawOrigin::Memory { id }, &memory)
2412            .unwrap();
2413        index.compact().unwrap();
2414        assert!(!stored_json(&index).contains("quokka"));
2415        assert_eq!(index.bm25_search("quokka", 5, None).unwrap()[0].0, key);
2416    }
2417
2418    type LegacyDoc = (RawOrigin, &'static str, Option<Uuid>);
2419
2420    /// Writes an index the way the schema before `schema` did, with a
2421    /// withdrawn passage among its documents and one source with none.
2422    fn write_legacy_index(dir: &Path, docs: &[LegacyDoc]) -> Vec<u64> {
2423        let text_dir = dir.join(TEXT_DIR);
2424        std::fs::create_dir_all(&text_dir).unwrap();
2425        let schema = legacy_schema();
2426        let field = |name: &str| schema.get_field(name).unwrap();
2427        let index = Index::create(
2428            MmapDirectory::open(&text_dir).unwrap(),
2429            schema.clone(),
2430            IndexSettings::default(),
2431        )
2432        .unwrap();
2433        let mut writer: IndexWriter = index.writer_with_num_threads(1, WRITER_HEAP_BYTES).unwrap();
2434        let mut vectors =
2435            VectorIndex::open_with_kind(&segment_path(dir, 0), ScalarKind::F16).unwrap();
2436        let mut keys = Vec::new();
2437        for (i, (origin, text, memory)) in docs.iter().enumerate() {
2438            let key = i as u64;
2439            let mut document = doc!(
2440                field("key") => key,
2441                field("body") => *text,
2442                field("text_hash") => text_hash(text),
2443                field("source") => origin.source_key(),
2444                field("project") => "-srv-garden",
2445                field("at") => 1_790_000_000i64,
2446            );
2447            if let Some(memory) = memory {
2448                document.add_text(field("memory"), memory.to_string());
2449            }
2450            writer.add_document(document).unwrap();
2451            vectors.upsert(key, &unit(i)).unwrap();
2452            keys.push(key);
2453        }
2454        let withdrawn = docs.len() as u64;
2455        writer
2456            .add_document(doc!(
2457                field("key") => withdrawn,
2458                field("body") => "a withdrawn line",
2459                field("text_hash") => text_hash("a withdrawn line"),
2460                field("source") => transcript("aa").source_key(),
2461            ))
2462            .unwrap();
2463        writer.delete_term(Term::from_field_u64(field("key"), withdrawn));
2464        writer
2465            .add_document(doc!(field("source") => transcript("empty").source_key()))
2466            .unwrap();
2467        writer.commit().unwrap();
2468        writer.wait_merging_threads().unwrap();
2469        vectors.save().unwrap();
2470        keys
2471    }
2472
2473    fn legacy_docs(memory: Uuid) -> Vec<LegacyDoc> {
2474        vec![
2475            (
2476                transcript("aa"),
2477                "the lantern hangs by the north door",
2478                None,
2479            ),
2480            (
2481                RawOrigin::Memory { id: memory },
2482                "the quokka ferry leaves at dawn",
2483                Some(memory),
2484            ),
2485            (
2486                transcript("aa"),
2487                "a third line of the transcript",
2488                Some(Uuid::new_v4()),
2489            ),
2490        ]
2491    }
2492
2493    fn assert_migrated(dir: &Path, index: &RawIndex, docs: &[LegacyDoc], keys: &[u64]) {
2494        assert_eq!(index.index.schema(), schema().0);
2495        assert_eq!(index.len(), docs.len());
2496        for (i, ((origin, text, memory), key)) in docs.iter().zip(keys).enumerate() {
2497            assert_eq!(index.vector_search(&unit(i), 1, None).unwrap()[0].0, *key);
2498            let hit = index.passage(*key).unwrap().unwrap();
2499            assert_eq!(&hit.origin, origin);
2500            assert_eq!(hit.memory, *memory);
2501            assert_eq!(hit.project.as_deref(), Some("-srv-garden"));
2502            assert_eq!(hit.at.unwrap().timestamp(), 1_790_000_000);
2503            assert_eq!(hit.text_hash, text_hash(text));
2504            let stored = match origin {
2505                RawOrigin::Memory { .. } => "",
2506                RawOrigin::Transcript { .. } => text,
2507            };
2508            assert_eq!(hit.text, stored);
2509            assert!(index.has_text(text));
2510        }
2511        assert_eq!(index.bm25_search("quokka", 5, None).unwrap()[0].0, keys[1]);
2512        assert!(!index.has_text("a withdrawn line"));
2513        assert!(index.has_source(&transcript("empty")));
2514        assert!(!stored_json(index).contains("quokka"));
2515        for leftover in [MIGRATED_TEXT_DIR, REPLACED_TEXT_DIR, MIGRATION_JOURNAL] {
2516            assert!(!dir.join(leftover).exists(), "{leftover} was left behind");
2517        }
2518    }
2519
2520    #[test]
2521    fn an_index_written_with_the_old_schema_migrates_keeping_every_passage_and_vector() {
2522        let dir = tempfile::tempdir().unwrap();
2523        let docs = legacy_docs(Uuid::new_v4());
2524        let keys = write_legacy_index(dir.path(), &docs);
2525
2526        let mut index = RawIndex::open(dir.path()).unwrap();
2527
2528        assert_migrated(dir.path(), &index, &docs, &keys);
2529        assert_migrated(
2530            dir.path(),
2531            &RawIndex::open_readonly(dir.path()).unwrap(),
2532            &docs,
2533            &keys,
2534        );
2535        index
2536            .add_source(
2537                &transcript("bb"),
2538                &[(passage("one more passage", None), unit(9))],
2539            )
2540            .unwrap();
2541        index.commit().unwrap();
2542        assert_eq!(index.len(), docs.len() + 1);
2543        let key = index.vector_search(&unit(9), 1, None).unwrap()[0].0;
2544        assert_eq!(
2545            index.passage(key).unwrap().unwrap().text,
2546            "one more passage"
2547        );
2548        assert!(!keys.contains(&key));
2549    }
2550
2551    #[test]
2552    fn an_old_schema_index_opened_keyed_migrates_into_sealed_text_files() {
2553        let dir = tempfile::tempdir().unwrap();
2554        let docs = legacy_docs(Uuid::new_v4());
2555        let keys = write_legacy_index(dir.path(), &docs);
2556        let sealing = seal_keys(8);
2557
2558        let index = RawIndex::open_keyed(dir.path(), &sealing).unwrap();
2559
2560        assert_migrated(dir.path(), &index, &docs, &keys);
2561        for path in files_under(&dir.path().join(TEXT_DIR)) {
2562            assert_eq!(
2563                sealed_key_id(&path).unwrap(),
2564                Some(sealing.current_id()),
2565                "{}",
2566                path.display()
2567            );
2568        }
2569        assert!(!holds_bytes(dir.path(), "lantern"));
2570    }
2571
2572    #[test]
2573    fn a_migration_a_crash_cut_short_is_redone_or_finished_at_open() {
2574        let before_the_journal = tempfile::tempdir().unwrap();
2575        let docs = legacy_docs(Uuid::new_v4());
2576        let keys = write_legacy_index(before_the_journal.path(), &docs);
2577        let partial = before_the_journal.path().join(MIGRATED_TEXT_DIR);
2578        std::fs::create_dir_all(&partial).unwrap();
2579        std::fs::write(partial.join(META_FILE), "a copy cut short").unwrap();
2580        let index = RawIndex::open(before_the_journal.path()).unwrap();
2581        assert_migrated(before_the_journal.path(), &index, &docs, &keys);
2582
2583        let mid_swap = tempfile::tempdir().unwrap();
2584        let keys = write_legacy_index(mid_swap.path(), &docs);
2585        build_migrated_text(mid_swap.path(), None).unwrap();
2586        sealed::write_file(
2587            &mid_swap.path().join(MIGRATION_JOURNAL),
2588            None,
2589            MIGRATION_JOURNAL,
2590            b"",
2591        )
2592        .unwrap();
2593        std::fs::rename(
2594            mid_swap.path().join(TEXT_DIR),
2595            mid_swap.path().join(REPLACED_TEXT_DIR),
2596        )
2597        .unwrap();
2598        let index = RawIndex::open(mid_swap.path()).unwrap();
2599        assert_migrated(mid_swap.path(), &index, &docs, &keys);
2600    }
2601}