Skip to main content

reflex/
snapshot.rs

1//! The index snapshot: which store files make up the index, and one reader over them.
2//!
3//! A snapshot is named by `.reflex/manifest.json`, the single commit point of an
4//! index run. It is:
5//!
6//! - a **base**: `content.<g>.bin` + `trigrams.<g>.bin` (+ planning sizes) from the
7//!   last full build, file ids `0..N-1`;
8//! - optionally a **delta**: the same pair for files added or modified since, with
9//!   their own ids `0..k-1`, seen through the snapshot as `N..N+k-1`;
10//! - optionally a **recent** segment on top of the delta, ids after the delta's: the
11//!   files of the latest updates. An update rebuilds only this small segment; when
12//!   it passes its limits it is folded into a new delta (two tiers, so an update
13//!   near the merge limit does not rebuild the whole delta);
14//! - **tombstones**: base and delta ids that are deleted or superseded.
15//!
16//! Every file a manifest names is generation-suffixed and never modified once
17//! written: a new run writes new files, then renames a new manifest into place. A
18//! reader that opened one generation keeps reading it; one that opens after the
19//! rename sees the next. No reader can pair files from two runs, which the
20//! two-rename publish before the manifest allowed.
21//!
22//! `content.bin` and `trigrams.bin` exist as hard links to the base only while the
23//! base alone is the snapshot (no delta, no tombstones), for older Reflex binaries
24//! (which read only those names) and for tools. While a delta holds changes they
25//! are absent, so an older binary stops instead of answering from the base alone.
26//!
27//! Without a manifest (a cache written before it existed) the fixed names are the
28//! snapshot, as before.
29
30use std::path::{Path, PathBuf};
31
32use anyhow::{Context, Result};
33use serde::{Deserialize, Serialize};
34
35use crate::content_store::ContentReader;
36use crate::errors::ReflexError;
37use crate::trigram::{
38    EXOTIC_LONG_S_RANGE, FileLocation, ListPart, Trigram, TrigramIndex, TrigramList,
39    exotic_singles, fold_windows, pattern_trigrams, plan_fold_intersection, plan_intersection,
40    union_lists,
41};
42
43/// File name of the manifest in the cache directory.
44pub const MANIFEST: &str = "manifest.json";
45/// The names older binaries read; hard links to the base while it is the snapshot.
46pub const FIXED_CONTENT: &str = "content.bin";
47pub const FIXED_TRIGRAMS: &str = "trigrams.bin";
48
49/// Manifest format this binary writes and reads.
50const MANIFEST_FORMAT: u32 = 2;
51
52/// One pair of stores (content + trigrams) named by a manifest.
53#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
54pub struct SegmentFiles {
55    /// `content.bin`-format file name, relative to the cache directory.
56    pub content: String,
57    /// `trigrams.bin`-format file name.
58    pub trigrams: String,
59    /// Planning sizes of `trigrams` (see `trigram::plan_size`), when written.
60    #[serde(default, skip_serializing_if = "Option::is_none")]
61    pub plan: Option<String>,
62    /// Files in the segment (both stores hold this many).
63    pub files: u64,
64    /// Byte sizes of the two files, checked at open.
65    pub content_bytes: u64,
66    pub trigrams_bytes: u64,
67}
68
69impl SegmentFiles {
70    fn names(&self) -> Vec<&str> {
71        let mut out = vec![self.content.as_str(), self.trigrams.as_str()];
72        out.extend(self.plan.as_deref());
73        out
74    }
75}
76
77/// The commit point of an index run.
78#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
79pub struct Manifest {
80    pub format: u32,
81    /// Bumped at every publish; every file this manifest names carries it or an
82    /// earlier generation's number.
83    pub generation: u64,
84    pub base: SegmentFiles,
85    /// Files added or modified since the base was built.
86    #[serde(default, skip_serializing_if = "Option::is_none")]
87    pub delta: Option<SegmentFiles>,
88    /// Files of the latest updates, above the delta (ids after the delta's).
89    #[serde(default, skip_serializing_if = "Option::is_none")]
90    pub recent: Option<SegmentFiles>,
91    /// Base and delta ids that are deleted or superseded, ascending.
92    #[serde(default, skip_serializing_if = "Vec::is_empty")]
93    pub tombstones: Vec<u32>,
94    /// `(trigram, planning size)` of the tombstoned base files' postings: what the
95    /// base's planning sizes over-count (see [`TombPlan`]).
96    #[serde(default, skip_serializing_if = "Option::is_none")]
97    pub tomb: Option<String>,
98    /// The same for the tombstoned delta files' postings.
99    #[serde(default, skip_serializing_if = "Option::is_none")]
100    pub tomb_delta: Option<String>,
101    /// Distinct trigrams with at least one posting in a live file.
102    pub live_trigrams: u64,
103    /// Unique per publish (random): what in-process caches of a publish's state
104    /// key on, since a generation number repeats after the cache is cleared.
105    #[serde(default)]
106    pub publish_id: u64,
107    /// Bytes of text of the live files (what `content.bin` of a full build of the
108    /// same tree holds).
109    pub live_corpus_bytes: u64,
110    /// blake3 of this manifest serialized with an empty `checksum`.
111    #[serde(default)]
112    pub checksum: String,
113}
114
115impl Manifest {
116    /// A manifest for a freshly built base.
117    pub fn for_base(
118        generation: u64,
119        base: SegmentFiles,
120        live_trigrams: u64,
121        live_corpus_bytes: u64,
122    ) -> Self {
123        Self {
124            format: MANIFEST_FORMAT,
125            generation,
126            base,
127            delta: None,
128            recent: None,
129            tombstones: Vec::new(),
130            tomb: None,
131            tomb_delta: None,
132            live_trigrams,
133            publish_id: 0,
134            live_corpus_bytes,
135            checksum: String::new(),
136        }
137        .sealed()
138    }
139
140    /// This manifest with the current format and its checksum (after any field
141    /// was set).
142    pub fn sealed(mut self) -> Self {
143        self.format = MANIFEST_FORMAT;
144        self.publish_id = new_publish_id();
145        self.checksum = self.compute_checksum();
146        self
147    }
148
149    fn compute_checksum(&self) -> String {
150        let mut unsigned = self.clone();
151        unsigned.checksum = String::new();
152        let body = serde_json::to_vec(&unsigned).expect("manifest serializes");
153        blake3::hash(&body).to_hex().to_string()
154    }
155
156    /// Every file name this manifest refers to.
157    pub fn files(&self) -> Vec<&str> {
158        let mut out = self.base.names();
159        for segment in [&self.delta, &self.recent].into_iter().flatten() {
160            out.extend(segment.names());
161        }
162        out.extend(self.tomb.as_deref());
163        out.extend(self.tomb_delta.as_deref());
164        out
165    }
166
167    /// Whether the base alone is the snapshot (no delta, nothing tombstoned).
168    pub fn base_only(&self) -> bool {
169        self.delta.is_none() && self.recent.is_none() && self.tombstones.is_empty()
170    }
171
172    /// Files of the snapshot: what a full build of the same tree holds.
173    pub fn live_files(&self) -> u64 {
174        self.base.files - self.tombstones.len() as u64
175            + self.delta.as_ref().map_or(0, |d| d.files)
176            + self.recent.as_ref().map_or(0, |d| d.files)
177    }
178}
179
180/// A random nonzero id: the clock, the process id and a counter, mixed by blake3.
181fn new_publish_id() -> u64 {
182    static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
183    let mut h = blake3::Hasher::new();
184    let nanos = std::time::SystemTime::now()
185        .duration_since(std::time::UNIX_EPOCH)
186        .map_or(0, |d| d.as_nanos());
187    h.update(&nanos.to_le_bytes());
188    h.update(&std::process::id().to_le_bytes());
189    h.update(
190        &COUNTER
191            .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
192            .to_le_bytes(),
193    );
194    let bytes = h.finalize();
195    u64::from_le_bytes(bytes.as_bytes()[..8].try_into().expect("8 bytes")).max(1)
196}
197
198/// `content.<g>.bin` / `trigrams.<g>.bin` / `trigrams.<g>.plan` of a base built
199/// at generation `g`.
200pub fn base_file_names(generation: u64) -> (String, String, String) {
201    (
202        format!("content.{generation}.bin"),
203        format!("trigrams.{generation}.bin"),
204        format!("trigrams.{generation}.plan"),
205    )
206}
207
208/// The files of a delta built at generation `g`: content, trigrams, planning
209/// sizes, tombstoned planning sizes.
210pub fn delta_file_names(generation: u64) -> (String, String, String, String) {
211    (
212        format!("delta.{generation}.content.bin"),
213        format!("delta.{generation}.trigrams.bin"),
214        format!("delta.{generation}.plan"),
215        format!("delta.{generation}.tomb"),
216    )
217}
218
219/// The files of a recent segment built at generation `g` (content, trigrams,
220/// planning sizes), and of the delta's tombstoned planning sizes.
221pub fn recent_file_names(generation: u64) -> (String, String, String, String) {
222    (
223        format!("recent.{generation}.content.bin"),
224        format!("recent.{generation}.trigrams.bin"),
225        format!("recent.{generation}.plan"),
226        format!("delta.{generation}.dtomb"),
227    )
228}
229
230/// Whether `name` is a generation file this module writes (so cleanup may delete
231/// it when no manifest names it).
232pub fn is_generation_file(name: &str) -> bool {
233    let parts: Vec<&str> = name.split('.').collect();
234    let digits = |g: &str| !g.is_empty() && g.bytes().all(|b| b.is_ascii_digit());
235    match parts.as_slice() {
236        ["content" | "trigrams", g, "bin"] => digits(g),
237        ["trigrams", g, "plan"] => digits(g),
238        ["delta" | "recent", g, "content" | "trigrams", "bin"] => digits(g),
239        ["delta" | "recent", g, "plan"] => digits(g),
240        ["delta", g, "tomb" | "dtomb"] => digits(g),
241        _ => false,
242    }
243}
244
245/// Read the manifest; `Ok(None)` when there is none (a cache from before it).
246pub fn read_manifest(cache_dir: &Path) -> Result<Option<Manifest>> {
247    let path = cache_dir.join(MANIFEST);
248    let bytes = match std::fs::read(&path) {
249        Ok(b) => b,
250        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
251        Err(e) => return Err(e).with_context(|| format!("Failed to read {}", path.display())),
252    };
253    let manifest: Manifest = serde_json::from_slice(&bytes)
254        .with_context(|| format!("{} is not a valid manifest", path.display()))?;
255    if manifest.format != MANIFEST_FORMAT {
256        anyhow::bail!(
257            "{} has format {}, this binary reads {}",
258            path.display(),
259            manifest.format,
260            MANIFEST_FORMAT
261        );
262    }
263    if manifest.checksum != manifest.compute_checksum() {
264        anyhow::bail!("{} failed its checksum", path.display());
265    }
266    Ok(Some(manifest))
267}
268
269/// Publish `manifest`: tmp + fsync + rename + directory fsync. After this returns,
270/// every new reader sees the snapshot it names.
271pub fn write_manifest(cache_dir: &Path, manifest: &Manifest) -> Result<()> {
272    use std::io::Write;
273    let path = cache_dir.join(MANIFEST);
274    let tmp = crate::atomic_write::tmp_path_for(&path);
275    let body = serde_json::to_vec_pretty(manifest).context("Failed to serialize the manifest")?;
276    {
277        let mut f = std::fs::File::create(&tmp)
278            .with_context(|| format!("Failed to create {}", tmp.display()))?;
279        f.write_all(&body)?;
280        f.sync_all()?;
281    }
282    crate::atomic_write::atomic_replace(&tmp, &path)
283        .with_context(|| format!("Failed to move {} into place", path.display()))?;
284    sync_dir(cache_dir);
285    Ok(())
286}
287
288/// fsync a directory so a rename in it is durable (best effort; not on Windows).
289pub fn sync_dir(dir: &Path) {
290    #[cfg(unix)]
291    if let Ok(d) = std::fs::File::open(dir) {
292        let _ = d.sync_all();
293    }
294    #[cfg(not(unix))]
295    let _ = dir;
296}
297
298/// Point `content.bin` / `trigrams.bin` at the manifest's base (hard links), for
299/// older binaries, when the base alone is the snapshot. Otherwise, or where hard
300/// links are not supported, the fixed names are removed: an older binary then
301/// stops rather than answering from a base that lacks the delta.
302pub fn link_fixed_names(cache_dir: &Path, manifest: &Manifest) {
303    if !manifest.base_only() {
304        unlink_fixed_names(cache_dir);
305        return;
306    }
307    for (generation_file, fixed) in [
308        (&manifest.base.content, FIXED_CONTENT),
309        (&manifest.base.trigrams, FIXED_TRIGRAMS),
310    ] {
311        let target = cache_dir.join(fixed);
312        let tmp = crate::atomic_write::tmp_path_for(&target);
313        let _ = std::fs::remove_file(&tmp);
314        let linked = std::fs::hard_link(cache_dir.join(generation_file), &tmp)
315            .and_then(|_| crate::atomic_write::atomic_replace(&tmp, &target));
316        if let Err(e) = linked {
317            log::debug!(
318                "Cannot link {} to {}: {}; removing it",
319                fixed,
320                generation_file,
321                e
322            );
323            let _ = std::fs::remove_file(&tmp);
324            let _ = std::fs::remove_file(&target);
325        }
326    }
327}
328
329/// Remove `content.bin` / `trigrams.bin` (before a manifest with a delta is
330/// published, so no older binary reads the base alone once it is incomplete).
331pub fn unlink_fixed_names(cache_dir: &Path) {
332    let mut removed = false;
333    for fixed in [FIXED_CONTENT, FIXED_TRIGRAMS] {
334        match std::fs::remove_file(cache_dir.join(fixed)) {
335            Ok(()) => removed = true,
336            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
337            Err(e) => log::warn!("Could not remove {}: {}", fixed, e),
338        }
339    }
340    // Already absent (a delta was live): nothing to make durable.
341    if removed {
342        sync_dir(cache_dir);
343    }
344}
345
346/// Delete generation files that neither `current` nor `previous` names. A reader
347/// that read the previous manifest a moment ago can still open its files; older
348/// ones are gone. Failures (a file still mapped on Windows) are retried next run.
349pub fn remove_unreferenced(cache_dir: &Path, current: &Manifest, previous: Option<&Manifest>) {
350    let mut keep: std::collections::HashSet<&str> = current.files().into_iter().collect();
351    if let Some(prev) = previous {
352        keep.extend(prev.files());
353    }
354    let Ok(entries) = std::fs::read_dir(cache_dir) else {
355        return;
356    };
357    for entry in entries.flatten() {
358        let name = entry.file_name();
359        let Some(name) = name.to_str() else { continue };
360        if is_generation_file(name)
361            && !keep.contains(name)
362            && let Err(e) = std::fs::remove_file(entry.path())
363        {
364            log::debug!("Could not remove {}: {}", name, e);
365        }
366    }
367}
368
369const TOMB_MAGIC: &[u8; 4] = b"RFTB";
370const TOMB_VERSION: u32 = 1;
371const TOMB_HEADER: usize = 16;
372
373/// `delta.<g>.tomb`: for each trigram with postings in tombstoned base files, the
374/// planning size of those postings. The base's planning size minus this, plus the
375/// delta's, is the planning size a full build of the same tree has. Header: magic
376/// `RFTB`, version u32, entry count u64; then `(trigram u32, size u32)` sorted.
377pub struct TombPlan {
378    mmap: Option<memmap2::Mmap>,
379    count: usize,
380}
381
382impl TombPlan {
383    fn load(path: &Path) -> Result<Self> {
384        let file = std::fs::File::open(path)
385            .with_context(|| format!("Failed to open {}", path.display()))?;
386        let len = file.metadata()?.len() as usize;
387        if len == TOMB_HEADER {
388            // An empty map cannot be mapped on every platform.
389            return Ok(Self {
390                mmap: None,
391                count: 0,
392            });
393        }
394        let mmap = unsafe {
395            memmap2::Mmap::map(&file)
396                .with_context(|| format!("Failed to mmap {}", path.display()))?
397        };
398        if mmap.len() < TOMB_HEADER || &mmap[0..4] != TOMB_MAGIC {
399            anyhow::bail!("{} is not a tombstone planning file", path.display());
400        }
401        if u32::from_le_bytes(mmap[4..8].try_into().unwrap()) != TOMB_VERSION {
402            anyhow::bail!("{} has an unsupported version", path.display());
403        }
404        let count = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
405        if mmap.len() != TOMB_HEADER + count * 8 {
406            anyhow::bail!("{} is truncated", path.display());
407        }
408        Ok(Self {
409            mmap: Some(mmap),
410            count,
411        })
412    }
413
414    #[inline]
415    fn entry(&self, i: usize) -> (Trigram, u32) {
416        let mmap = self.mmap.as_ref().expect("non-empty");
417        let off = TOMB_HEADER + i * 8;
418        (
419            u32::from_le_bytes(mmap[off..off + 4].try_into().unwrap()),
420            u32::from_le_bytes(mmap[off + 4..off + 8].try_into().unwrap()),
421        )
422    }
423
424    /// The tombstoned planning size of `trigram` (0 when none).
425    pub fn get(&self, trigram: Trigram) -> u32 {
426        let (mut lo, mut hi) = (0usize, self.count);
427        while lo < hi {
428            let mid = lo + (hi - lo) / 2;
429            let (t, size) = self.entry(mid);
430            match t.cmp(&trigram) {
431                std::cmp::Ordering::Less => lo = mid + 1,
432                std::cmp::Ordering::Greater => hi = mid,
433                std::cmp::Ordering::Equal => return size,
434            }
435        }
436        0
437    }
438
439    /// Every `(trigram, size)`, ascending.
440    pub fn entries(&self) -> impl Iterator<Item = (Trigram, u32)> + '_ {
441        (0..self.count).map(|i| self.entry(i))
442    }
443}
444
445/// Write a tombstone planning file (`entries` sorted by trigram; tmp + fsync + rename).
446pub fn write_tomb_file(path: &Path, entries: &[(Trigram, u32)]) -> Result<()> {
447    use std::io::Write;
448    let tmp = crate::atomic_write::tmp_path_for(path);
449    {
450        let mut w = std::io::BufWriter::new(
451            std::fs::File::create(&tmp)
452                .with_context(|| format!("Failed to create {}", tmp.display()))?,
453        );
454        w.write_all(TOMB_MAGIC)?;
455        w.write_all(&TOMB_VERSION.to_le_bytes())?;
456        w.write_all(&(entries.len() as u64).to_le_bytes())?;
457        for (t, size) in entries {
458            w.write_all(&t.to_le_bytes())?;
459            w.write_all(&size.to_le_bytes())?;
460        }
461        w.flush()?;
462        w.get_ref().sync_all()?;
463    }
464    crate::atomic_write::atomic_replace(&tmp, path)
465        .with_context(|| format!("Failed to move {} into place", path.display()))
466}
467
468/// The first 32 bytes of a store file (both store formats have a 32-byte header).
469fn read_header(cache_dir: &Path, logical: &str, name: &str) -> Result<[u8; 32]> {
470    use std::io::Read;
471    let mut header = [0u8; 32];
472    std::fs::File::open(cache_dir.join(name))
473        .and_then(|mut f| f.read_exact(&mut header))
474        .map_err(|e| {
475            if e.kind() == std::io::ErrorKind::NotFound {
476                missing(logical, name)
477            } else {
478                ReflexError::CacheCorrupted(format!("{logical}: {name}: {e}")).into()
479            }
480        })?;
481    Ok(header)
482}
483
484/// Whether the snapshot the manifest names is complete, without opening it: every
485/// named file exists at the size the manifest recorded, each store's header has its
486/// magic, version and the manifest's file count, and every tombstone is in range.
487/// Reads one header per store, where [`IndexSnapshot::open`] maps the stores and
488/// walks their path tables (an index run that changes nothing only needs this).
489/// `Ok(None)` without a manifest.
490pub fn check_published(cache_dir: &Path) -> Result<Option<Manifest>> {
491    let Some(m) = read_manifest(cache_dir)? else {
492        return Ok(None);
493    };
494    let u32_at = |h: &[u8; 32], at: usize| u32::from_le_bytes(h[at..at + 4].try_into().unwrap());
495    let u64_at = |h: &[u8; 32], at: usize| u64::from_le_bytes(h[at..at + 8].try_into().unwrap());
496    for segment in [Some(&m.base), m.delta.as_ref(), m.recent.as_ref()]
497        .into_iter()
498        .flatten()
499    {
500        check_size(
501            cache_dir,
502            "content.bin",
503            &segment.content,
504            segment.content_bytes,
505        )?;
506        check_size(
507            cache_dir,
508            "trigrams.bin",
509            &segment.trigrams,
510            segment.trigrams_bytes,
511        )?;
512        let content = read_header(cache_dir, "content.bin", &segment.content)?;
513        if &content[0..4] != b"RFCT"
514            || u32_at(&content, 4) != 2
515            || u64_at(&content, 8) != segment.files
516        {
517            return Err(ReflexError::CacheCorrupted(format!(
518                "content.bin: {} does not hold the manifest's {} files",
519                segment.content, segment.files
520            ))
521            .into());
522        }
523        let trigrams = read_header(cache_dir, "trigrams.bin", &segment.trigrams)?;
524        if &trigrams[0..4] != crate::trigram::MAGIC
525            || u32_at(&trigrams, 4) != crate::trigram::VERSION
526            || u64_at(&trigrams, 16) != segment.files
527        {
528            return Err(ReflexError::CacheCorrupted(format!(
529                "trigrams.bin: {} does not hold the manifest's {} files",
530                segment.trigrams, segment.files
531            ))
532            .into());
533        }
534        if let Some(plan) = &segment.plan
535            && !cache_dir.join(plan).exists()
536        {
537            return Err(missing("trigrams.bin", plan));
538        }
539    }
540    for tomb in [&m.tomb, &m.tomb_delta].into_iter().flatten() {
541        if !cache_dir.join(tomb).exists() {
542            return Err(missing("trigrams.bin", tomb));
543        }
544    }
545    let bound = m.base.files + m.delta.as_ref().map_or(0, |d| d.files);
546    if m.tombstones.iter().any(|&id| id as u64 >= bound) {
547        return Err(ReflexError::CacheCorrupted(
548            "content.bin: a tombstone is past the base and delta's files".to_string(),
549        )
550        .into());
551    }
552    Ok(Some(m))
553}
554
555/// One pair of opened stores.
556pub struct Segment {
557    pub content: ContentReader,
558    pub trigrams: TrigramIndex,
559}
560
561/// Every reader of the stores goes through this: file ids, content, paths and
562/// candidate lookups over the snapshot a manifest (or the legacy fixed names) names.
563pub struct IndexSnapshot {
564    base: Segment,
565    delta: Option<Segment>,
566    recent: Option<Segment>,
567    /// Tombstoned base and delta ids, one bit each.
568    dead: Vec<u64>,
569    dead_count: usize,
570    tomb: Option<TombPlan>,
571    tomb_delta: Option<TombPlan>,
572    /// Files in the base: the delta's ids start here.
573    base_len: u32,
574    /// Files in the delta: the recent segment's ids start at `base_len + delta_len`.
575    delta_len: u32,
576    /// The manifest this snapshot was opened from; `None` for a legacy cache.
577    manifest: Option<Manifest>,
578}
579
580impl std::fmt::Debug for IndexSnapshot {
581    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
582        f.debug_struct("IndexSnapshot")
583            .field("files", &self.live_file_count())
584            .field("generation", &self.generation())
585            .finish()
586    }
587}
588
589/// Whether an error chain carries an `io::ErrorKind::NotFound`.
590fn is_not_found(err: &anyhow::Error) -> bool {
591    err.chain().any(|c| {
592        c.downcast_ref::<std::io::Error>()
593            .is_some_and(|e| e.kind() == std::io::ErrorKind::NotFound)
594    })
595}
596
597/// A missing manifest-named file: a publish race (retried) or a damaged cache.
598/// `logical` is the name an error shows (`content.bin` / `trigrams.bin`).
599fn missing(logical: &str, name: &str) -> anyhow::Error {
600    anyhow::Error::new(std::io::Error::new(
601        std::io::ErrorKind::NotFound,
602        format!("{} is missing", name),
603    ))
604    .context(ReflexError::CacheCorrupted(format!(
605        "{}: {} is missing",
606        logical, name
607    )))
608}
609
610/// Check a manifest-named file's size (a truncated or replaced file is corruption).
611fn check_size(cache_dir: &Path, logical: &str, name: &str, expected: u64) -> Result<()> {
612    match std::fs::metadata(cache_dir.join(name)) {
613        Ok(md) if md.len() == expected => Ok(()),
614        Ok(md) => Err(ReflexError::CacheCorrupted(format!(
615            "{}: {} is {} bytes, the manifest says {}",
616            logical,
617            name,
618            md.len(),
619            expected
620        ))
621        .into()),
622        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(missing(logical, name)),
623        Err(e) => Err(e).with_context(|| format!("Failed to stat {}", name)),
624    }
625}
626
627/// Open one manifest-named segment.
628fn open_segment(cache_dir: &Path, files: &SegmentFiles) -> Result<Segment> {
629    check_size(
630        cache_dir,
631        "content.bin",
632        &files.content,
633        files.content_bytes,
634    )?;
635    check_size(
636        cache_dir,
637        "trigrams.bin",
638        &files.trigrams,
639        files.trigrams_bytes,
640    )?;
641    let content = ContentReader::open(cache_dir.join(&files.content))
642        .map_err(|e| ReflexError::CacheCorrupted(format!("content.bin: {e:#}")))?;
643    let mut trigrams = TrigramIndex::load(cache_dir.join(&files.trigrams))
644        .map_err(|e| ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}")))?;
645    // Planning sizes: the early-stop planner reads them instead of on-disk
646    // bytes (an index without them plans by bytes, as before).
647    if let Some(plan) = &files.plan {
648        let plan_path = cache_dir.join(plan);
649        if !plan_path.exists() {
650            return Err(missing("trigrams.bin", plan));
651        }
652        trigrams
653            .attach_plan(&plan_path)
654            .map_err(|e| ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}")))?;
655    }
656    check_counts(&content, &trigrams)?;
657    if content.file_count() as u64 != files.files {
658        return Err(ReflexError::CacheCorrupted(format!(
659            "content.bin holds {} files but the manifest lists {}",
660            content.file_count(),
661            files.files
662        ))
663        .into());
664    }
665    Ok(Segment { content, trigrams })
666}
667
668fn check_counts(content: &ContentReader, trigrams: &TrigramIndex) -> Result<()> {
669    if trigrams.file_count() != content.file_count() {
670        return Err(ReflexError::CacheCorrupted(format!(
671            "trigrams.bin lists {} files but content.bin holds {} (index written by two runs?)",
672            trigrams.file_count(),
673            content.file_count()
674        ))
675        .into());
676    }
677    Ok(())
678}
679
680impl IndexSnapshot {
681    /// Open the snapshot the manifest names (retrying when a concurrent publish
682    /// removed a file between reading the manifest and opening it), or the legacy
683    /// fixed names when there is no manifest.
684    pub fn open(cache_dir: &Path) -> Result<Self> {
685        Self::open_reading(cache_dir, || read_manifest(cache_dir))
686    }
687
688    /// [`Self::open`] with the manifest read by `read` (each attempt reads it again).
689    fn open_reading(
690        cache_dir: &Path,
691        mut read: impl FnMut() -> Result<Option<Manifest>>,
692    ) -> Result<Self> {
693        let mut last: Option<anyhow::Error> = None;
694        for attempt in 0..5u64 {
695            let manifest =
696                read().map_err(|e| ReflexError::CacheCorrupted(format!("content.bin: {e:#}")))?;
697            let result = match manifest {
698                None => return Self::open_legacy(cache_dir),
699                Some(m) => Self::open_manifest(cache_dir, m),
700            };
701            match result {
702                Ok(s) => return Ok(s),
703                Err(e) if is_not_found(&e) => {
704                    last = Some(e);
705                    std::thread::sleep(std::time::Duration::from_millis(10 * (attempt + 1)));
706                }
707                Err(e) => return Err(e),
708            }
709        }
710        Err(last.expect("at least one attempt"))
711    }
712
713    fn open_manifest(cache_dir: &Path, manifest: Manifest) -> Result<Self> {
714        let base = open_segment(cache_dir, &manifest.base)?;
715        let base_len = base.content.file_count() as u32;
716        let delta = match &manifest.delta {
717            Some(d) => Some(open_segment(cache_dir, d)?),
718            None => None,
719        };
720        let delta_len = delta.as_ref().map_or(0, |d| d.content.file_count() as u32);
721        let recent = match &manifest.recent {
722            Some(d) => Some(open_segment(cache_dir, d)?),
723            None => None,
724        };
725        let load_tomb = |name: &Option<String>| -> Result<Option<TombPlan>> {
726            let Some(name) = name else { return Ok(None) };
727            let tomb_path = cache_dir.join(name);
728            if !tomb_path.exists() {
729                return Err(missing("trigrams.bin", name));
730            }
731            Ok(Some(TombPlan::load(&tomb_path).map_err(|e| {
732                ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}"))
733            })?))
734        };
735        let tomb = load_tomb(&manifest.tomb)?;
736        let tomb_delta = load_tomb(&manifest.tomb_delta)?;
737        let dead_bound = base_len + delta_len;
738        let mut dead = vec![0u64; (dead_bound as usize).div_ceil(64)];
739        for &id in &manifest.tombstones {
740            if id >= dead_bound {
741                return Err(ReflexError::CacheCorrupted(format!(
742                    "content.bin: tombstone {} is past the base and delta's {} files",
743                    id, dead_bound
744                ))
745                .into());
746            }
747            dead[(id / 64) as usize] |= 1u64 << (id % 64);
748        }
749        Ok(Self {
750            base,
751            delta,
752            recent,
753            dead,
754            dead_count: manifest.tombstones.len(),
755            tomb,
756            tomb_delta,
757            base_len,
758            delta_len,
759            manifest: Some(manifest),
760        })
761    }
762
763    fn open_legacy(cache_dir: &Path) -> Result<Self> {
764        let content = ContentReader::open(cache_dir.join(FIXED_CONTENT))
765            .map_err(|e| ReflexError::CacheCorrupted(format!("content.bin: {e:#}")))?;
766        let trigrams_path = cache_dir.join(FIXED_TRIGRAMS);
767        let trigrams = if trigrams_path.exists() {
768            match TrigramIndex::load(&trigrams_path) {
769                Ok(index) => index,
770                // A format from another Reflex version is not corruption (see
771                // `ReflexError::CacheVersionMismatch`): serve from an in-memory
772                // rebuild, and let the schema-hash check report the index stale
773                // so the caller re-indexes.
774                Err(e) if e.to_string().contains("Unsupported trigrams.bin version") => {
775                    log::warn!("{}; rebuilding trigram index in memory for this process", e);
776                    crate::query::result::rebuild_trigram_index(&content)?
777                }
778                Err(e) => {
779                    return Err(ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}")).into());
780                }
781            }
782        } else {
783            log::debug!("trigrams.bin not found, rebuilding from content store");
784            crate::query::result::rebuild_trigram_index(&content)?
785        };
786        check_counts(&content, &trigrams)?;
787        let base_len = content.file_count() as u32;
788        Ok(Self {
789            base: Segment { content, trigrams },
790            delta: None,
791            recent: None,
792            dead: Vec::new(),
793            dead_count: 0,
794            tomb: None,
795            tomb_delta: None,
796            base_len,
797            delta_len: 0,
798            manifest: None,
799        })
800    }
801
802    /// The generation of the manifest this snapshot was opened from.
803    pub fn generation(&self) -> Option<u64> {
804        self.manifest.as_ref().map(|m| m.generation)
805    }
806
807    /// The manifest this snapshot was opened from (`None` for a legacy cache).
808    pub fn manifest(&self) -> Option<&Manifest> {
809        self.manifest.as_ref()
810    }
811
812    /// The base segment.
813    pub fn base(&self) -> &Segment {
814        &self.base
815    }
816
817    /// The delta segment, if any.
818    pub fn delta(&self) -> Option<&Segment> {
819        self.delta.as_ref()
820    }
821
822    /// The recent segment, if any.
823    pub fn recent(&self) -> Option<&Segment> {
824        self.recent.as_ref()
825    }
826
827    /// Files in the base (the delta's first id).
828    pub fn base_len(&self) -> u32 {
829        self.base_len
830    }
831
832    /// Files in the delta (the recent segment's first id is `base_len + delta_len`).
833    pub fn delta_len(&self) -> u32 {
834        self.delta_len
835    }
836
837    fn recent_len(&self) -> u32 {
838        self.recent
839            .as_ref()
840            .map_or(0, |d| d.content.file_count() as u32)
841    }
842
843    /// The tombstoned base postings' planning sizes.
844    pub fn tomb(&self) -> Option<&TombPlan> {
845        self.tomb.as_ref()
846    }
847
848    /// The tombstoned delta postings' planning sizes.
849    pub fn tomb_delta(&self) -> Option<&TombPlan> {
850        self.tomb_delta.as_ref()
851    }
852
853    /// Whether id `id` (base or delta) is tombstoned.
854    #[inline]
855    pub fn is_dead(&self, id: u32) -> bool {
856        id < self.base_len + self.delta_len
857            && self
858                .dead
859                .get((id / 64) as usize)
860                .is_some_and(|w| w & (1u64 << (id % 64)) != 0)
861    }
862
863    /// Upper bound of file ids (`0..id_bound()`); some ids below it may be dead.
864    pub fn id_bound(&self) -> u32 {
865        self.base_len + self.delta_len + self.recent_len()
866    }
867
868    /// Number of live files (what a fresh build of the same tree holds).
869    pub fn live_file_count(&self) -> usize {
870        self.id_bound() as usize - self.dead_count
871    }
872
873    /// Whether `id` names a live file.
874    pub fn is_live(&self, id: u32) -> bool {
875        id < self.id_bound() && !self.is_dead(id)
876    }
877
878    /// Every live file id, ascending.
879    pub fn live_ids(&self) -> impl Iterator<Item = u32> + '_ {
880        (0..self.id_bound()).filter(|&id| !self.is_dead(id))
881    }
882
883    /// The segment holding `id`, and the id within it.
884    #[inline]
885    fn route(&self, id: u32) -> (&Segment, u32) {
886        if id < self.base_len {
887            return (&self.base, id);
888        }
889        match (&self.delta, &self.recent) {
890            (Some(delta), _) if id < self.base_len + self.delta_len => (delta, id - self.base_len),
891            (_, Some(recent)) => (recent, id - self.base_len - self.delta_len),
892            _ => (&self.base, id),
893        }
894    }
895
896    /// Content of a live file.
897    pub fn get_file_content(&self, file_id: u32) -> Result<&str> {
898        let (segment, local) = self.route(file_id);
899        segment.content.get_file_content(local)
900    }
901
902    /// Drop the resident pages of every content store (see
903    /// [`ContentReader::release_pages`]).
904    pub fn release_pages(&self) {
905        for segment in [Some(&self.base), self.delta.as_ref(), self.recent.as_ref()]
906            .into_iter()
907            .flatten()
908        {
909            segment.content.release_pages();
910        }
911    }
912
913    /// Length of a file's text, from the entry table (no content page touched).
914    pub fn file_len(&self, file_id: u32) -> Option<u64> {
915        let (segment, local) = self.route(file_id);
916        segment.content.file_len(local)
917    }
918
919    /// Path of a live file (`None` for an id that is out of range or dead).
920    pub fn get_file_path(&self, file_id: u32) -> Option<&Path> {
921        if !self.is_live(file_id) {
922            return None;
923        }
924        let (segment, local) = self.route(file_id);
925        segment.content.get_file_path(local)
926    }
927
928    /// Lines around `line_number` of a live file.
929    pub fn get_context_by_line(
930        &self,
931        file_id: u32,
932        line_number: usize,
933        context_lines: usize,
934    ) -> Result<(Vec<String>, Vec<String>)> {
935        let (segment, local) = self.route(file_id);
936        segment
937            .content
938            .get_context_by_line(local, line_number, context_lines)
939    }
940
941    /// Live id of `path` (a leading `./` is ignored), by linear scan. Callers on a
942    /// hot path use `OpenIndex::file_id_for`, which caches the map.
943    pub fn get_file_id_by_path(&self, path: &str) -> Option<u32> {
944        let normalized = path.strip_prefix("./").unwrap_or(path);
945        self.live_ids().find(|&id| {
946            self.get_file_path(id)
947                .and_then(|p| p.to_str())
948                .is_some_and(|p| p.strip_prefix("./").unwrap_or(p) == normalized)
949        })
950    }
951
952    /// Whether candidate lookups can go straight to the base index.
953    fn base_alone(&self) -> bool {
954        self.delta.is_none() && self.recent.is_none() && self.dead_count == 0
955    }
956
957    /// Planning size of `trigram`'s live postings (0: none).
958    pub fn live_plan(&self, trigram: Trigram) -> u64 {
959        self.list(trigram).map_or(0, |l| l.plan)
960    }
961
962    /// `trigram`'s posting list over the base and the delta (tombstones left out)
963    /// and the recent segment, with the planning size a full build of the same
964    /// tree has.
965    fn list(&self, trigram: Trigram) -> Option<TrigramList<'_>> {
966        let mut parts: Vec<ListPart<'_>> = Vec::with_capacity(3);
967        let mut plan: i64 = 0;
968        let dead: Option<&[u64]> = (self.dead_count > 0).then_some(self.dead.as_slice());
969        if let Some((mut part, base_plan)) = self.base.trigrams.list_part(trigram) {
970            part.dead = dead;
971            parts.push(part);
972            plan += base_plan as i64;
973        }
974        for tomb in [&self.tomb, &self.tomb_delta].into_iter().flatten() {
975            plan -= tomb.get(trigram) as i64;
976        }
977        if let Some(delta) = &self.delta
978            && let Some((mut part, delta_plan)) = delta.trigrams.list_part(trigram)
979        {
980            part.id_offset = self.base_len;
981            part.dead = dead;
982            parts.push(part);
983            plan += delta_plan as i64;
984        }
985        if let Some(recent) = &self.recent
986            && let Some((mut part, recent_plan)) = recent.trigrams.list_part(trigram)
987        {
988            part.id_offset = self.base_len + self.delta_len;
989            parts.push(part);
990            plan += recent_plan as i64;
991        }
992        // Every live file block adds at least 2 to the planning size, so 0 means
993        // every file holding the trigram is tombstoned: a fresh build has no list.
994        if parts.is_empty() || plan <= 0 {
995            return None;
996        }
997        Some(TrigramList {
998            trigram,
999            plan: plan as u64,
1000            parts,
1001        })
1002    }
1003
1004    /// Candidate lines for `pattern` (see [`TrigramIndex::search_candidates`]).
1005    pub fn search_candidates(&self, pattern: &str) -> Vec<FileLocation> {
1006        if self.base_alone() {
1007            return self.base.trigrams.search_candidates(pattern);
1008        }
1009        let lists = pattern_trigrams(pattern)
1010            .into_iter()
1011            .map(|t| self.list(t))
1012            .collect::<Vec<_>>();
1013        if lists.is_empty() {
1014            return vec![];
1015        }
1016        plan_intersection(lists, true)
1017    }
1018
1019    /// Case-insensitive candidate lines (see [`TrigramIndex::search_candidates_fold`]).
1020    pub fn search_candidates_fold(&self, literal: &[u8]) -> Vec<FileLocation> {
1021        if self.base_alone() {
1022            return self.base.trigrams.search_candidates_fold(literal);
1023        }
1024        let Some(windows) = fold_windows(literal) else {
1025            return vec![];
1026        };
1027        let lists = windows
1028            .iter()
1029            .map(|variants| variants.iter().filter_map(|&t| self.list(t)).collect())
1030            .collect();
1031        plan_fold_intersection(lists)
1032    }
1033
1034    /// Lines with a Kelvin sign or long s (see [`TrigramIndex::exotic_fold_lines`]).
1035    pub fn exotic_fold_lines(&self) -> Vec<FileLocation> {
1036        if self.base_alone() {
1037            return self.base.trigrams.exotic_fold_lines();
1038        }
1039        let (lo, hi) = EXOTIC_LONG_S_RANGE;
1040        let mut trigrams: Vec<Trigram> = self
1041            .base
1042            .trigrams
1043            .list_parts_in(lo, hi)
1044            .into_iter()
1045            .map(|(t, _, _)| t)
1046            .collect();
1047        for segment in [&self.delta, &self.recent].into_iter().flatten() {
1048            trigrams.extend(
1049                segment
1050                    .trigrams
1051                    .list_parts_in(lo, hi)
1052                    .into_iter()
1053                    .map(|(t, _, _)| t),
1054            );
1055        }
1056        trigrams.extend(exotic_singles());
1057        trigrams.sort_unstable();
1058        trigrams.dedup();
1059        let lists: Vec<TrigramList> = trigrams.into_iter().filter_map(|t| self.list(t)).collect();
1060        union_lists(&lists)
1061    }
1062
1063    /// Distinct trigrams in the snapshot's live files.
1064    pub fn trigram_count(&self) -> usize {
1065        match &self.manifest {
1066            Some(m) => m.live_trigrams as usize,
1067            None => self.base.trigrams.trigram_count(),
1068        }
1069    }
1070}
1071
1072/// What the published snapshot is made of, for sizes and integrity checks. Every
1073/// file is counted once (the fixed names are links to the base, not extra bytes).
1074#[derive(Debug, Clone)]
1075pub struct StoreSummary {
1076    /// `content.bin`-format files of the snapshot.
1077    pub content_files: Vec<PathBuf>,
1078    /// `trigrams.bin`-format files of the snapshot.
1079    pub trigram_files: Vec<PathBuf>,
1080    /// Sidecars of the snapshot (planning sizes).
1081    pub other_files: Vec<PathBuf>,
1082    /// Text bytes of the live files, when a manifest records it.
1083    pub live_corpus_bytes: Option<u64>,
1084    /// Distinct trigrams of the live files, when a manifest records it.
1085    pub live_trigrams: Option<u64>,
1086}
1087
1088impl StoreSummary {
1089    /// Every store file of the snapshot.
1090    pub fn files(&self) -> impl Iterator<Item = &PathBuf> {
1091        self.content_files
1092            .iter()
1093            .chain(&self.trigram_files)
1094            .chain(&self.other_files)
1095    }
1096}
1097
1098/// The snapshot of `cache_dir` as its manifest names it, or the fixed names
1099/// without one.
1100pub fn store_summary(cache_dir: &Path) -> StoreSummary {
1101    match read_manifest(cache_dir) {
1102        Ok(Some(m)) => {
1103            let mut content_files = vec![cache_dir.join(&m.base.content)];
1104            let mut trigram_files = vec![cache_dir.join(&m.base.trigrams)];
1105            let mut other_files: Vec<PathBuf> =
1106                m.base.plan.iter().map(|p| cache_dir.join(p)).collect();
1107            for d in [&m.delta, &m.recent].into_iter().flatten() {
1108                content_files.push(cache_dir.join(&d.content));
1109                trigram_files.push(cache_dir.join(&d.trigrams));
1110                other_files.extend(d.plan.iter().map(|p| cache_dir.join(p)));
1111            }
1112            other_files.extend(m.tomb.iter().map(|t| cache_dir.join(t)));
1113            other_files.extend(m.tomb_delta.iter().map(|t| cache_dir.join(t)));
1114            StoreSummary {
1115                content_files,
1116                trigram_files,
1117                other_files,
1118                live_corpus_bytes: Some(m.live_corpus_bytes),
1119                live_trigrams: Some(m.live_trigrams),
1120            }
1121        }
1122        _ => StoreSummary {
1123            content_files: vec![cache_dir.join(FIXED_CONTENT)],
1124            trigram_files: vec![cache_dir.join(FIXED_TRIGRAMS)],
1125            other_files: Vec::new(),
1126            live_corpus_bytes: None,
1127            live_trigrams: None,
1128        },
1129    }
1130}
1131
1132#[cfg(test)]
1133mod tests {
1134    use super::*;
1135
1136    #[test]
1137    fn generation_file_names() {
1138        assert!(is_generation_file("content.12.bin"));
1139        assert!(is_generation_file("trigrams.3.bin"));
1140        assert!(is_generation_file("trigrams.3.plan"));
1141        assert!(is_generation_file("delta.4.content.bin"));
1142        assert!(is_generation_file("delta.4.trigrams.bin"));
1143        assert!(is_generation_file("delta.4.plan"));
1144        assert!(is_generation_file("delta.4.tomb"));
1145        assert!(is_generation_file("delta.4.dtomb"));
1146        assert!(is_generation_file("recent.4.content.bin"));
1147        assert!(is_generation_file("recent.4.trigrams.bin"));
1148        assert!(is_generation_file("recent.4.plan"));
1149        assert!(!is_generation_file("recent.4.tomb"));
1150        assert!(!is_generation_file("content.3.plan"));
1151        assert!(!is_generation_file("content.bin"));
1152        assert!(!is_generation_file("trigrams.bin"));
1153        assert!(!is_generation_file("content.x.bin"));
1154        assert!(!is_generation_file("meta.db"));
1155        assert!(!is_generation_file("manifest.json"));
1156    }
1157
1158    #[test]
1159    fn manifest_round_trip_and_checksum() {
1160        let temp = tempfile::TempDir::new().unwrap();
1161        let (content, trigrams, plan) = base_file_names(7);
1162        let m = Manifest::for_base(
1163            7,
1164            SegmentFiles {
1165                content,
1166                trigrams,
1167                plan: Some(plan),
1168                files: 3,
1169                content_bytes: 100,
1170                trigrams_bytes: 200,
1171            },
1172            42,
1173            90,
1174        );
1175        write_manifest(temp.path(), &m).unwrap();
1176        assert_eq!(read_manifest(temp.path()).unwrap(), Some(m));
1177
1178        // A tampered manifest fails its checksum.
1179        let path = temp.path().join(MANIFEST);
1180        let text = std::fs::read_to_string(&path)
1181            .unwrap()
1182            .replace("\"generation\": 7", "\"generation\": 8");
1183        std::fs::write(&path, text).unwrap();
1184        assert!(read_manifest(temp.path()).is_err());
1185    }
1186
1187    #[test]
1188    fn tomb_file_round_trip() {
1189        let temp = tempfile::TempDir::new().unwrap();
1190        let path = temp.path().join("t.tomb");
1191        write_tomb_file(&path, &[(3, 10), (70_000, 4), (0xE2_84_AA, 7)]).unwrap();
1192        let tomb = TombPlan::load(&path).unwrap();
1193        assert_eq!(tomb.get(3), 10);
1194        assert_eq!(tomb.get(70_000), 4);
1195        assert_eq!(tomb.get(0xE2_84_AA), 7);
1196        assert_eq!(tomb.get(4), 0);
1197        write_tomb_file(&path, &[]).unwrap();
1198        assert_eq!(TombPlan::load(&path).unwrap().get(3), 0);
1199    }
1200
1201    /// A reader that meets a manifest naming a file that is gone (a publish in
1202    /// between) retries and opens the snapshot the next manifest names; with no
1203    /// next manifest it reports the cache corrupted.
1204    #[test]
1205    fn open_retries_while_a_publish_replaces_the_manifest() {
1206        let temp = tempfile::TempDir::new().unwrap();
1207        let root = temp.path();
1208        write(root, "a.rs", "fn alpha() {}\n");
1209        index_unlimited(root);
1210        let cache = root.join(".reflex");
1211        let good = read_manifest(&cache).unwrap().unwrap();
1212        let mut files = good.base.clone();
1213        files.content = "content.999.bin".to_string();
1214        let broken = Manifest::for_base(
1215            good.generation + 1,
1216            files,
1217            good.live_trigrams,
1218            good.live_corpus_bytes,
1219        );
1220
1221        // The first read sees the manifest a publish is about to replace.
1222        let mut reads = 0;
1223        let snapshot = IndexSnapshot::open_reading(&cache, || {
1224            reads += 1;
1225            Ok(Some(if reads == 1 {
1226                broken.clone()
1227            } else {
1228                good.clone()
1229            }))
1230        })
1231        .expect("a retry opens the next manifest");
1232        assert_eq!(reads, 2);
1233        assert_eq!(snapshot.generation(), Some(good.generation));
1234
1235        write_manifest(&cache, &broken).unwrap();
1236        let err = IndexSnapshot::open(&cache).unwrap_err();
1237        assert!(
1238            matches!(
1239                err.downcast_ref::<ReflexError>(),
1240                Some(ReflexError::CacheCorrupted(_))
1241            ),
1242            "{err:#}"
1243        );
1244    }
1245
1246    fn index_unlimited(root: &Path) {
1247        index_with(root, None);
1248    }
1249
1250    fn write(root: &Path, rel: &str, body: &str) {
1251        let p = root.join(rel);
1252        std::fs::create_dir_all(p.parent().unwrap()).unwrap();
1253        std::fs::write(p, body).unwrap();
1254    }
1255
1256    /// Candidate lines of `snapshot` for `pattern`, by path.
1257    fn candidates(snapshot: &IndexSnapshot, pattern: &str) -> Vec<(PathBuf, u32)> {
1258        let mut out: Vec<(PathBuf, u32)> = snapshot
1259            .search_candidates(pattern)
1260            .into_iter()
1261            .map(|l| {
1262                let path = snapshot.get_file_path(l.file_id).expect("a live candidate");
1263                (path.to_path_buf(), l.line_no)
1264            })
1265            .collect();
1266        out.sort();
1267        out
1268    }
1269
1270    /// Copy the files under `from` into `to`, without `.reflex`.
1271    fn copy_tree(from: &Path, to: &Path) {
1272        for entry in std::fs::read_dir(from).unwrap() {
1273            let entry = entry.unwrap();
1274            let name = entry.file_name();
1275            if name == ".reflex" {
1276                continue;
1277            }
1278            let (src, dst) = (entry.path(), to.join(&name));
1279            if entry.file_type().unwrap().is_dir() {
1280                std::fs::create_dir_all(&dst).unwrap();
1281                copy_tree(&src, &dst);
1282            } else {
1283                std::fs::copy(&src, &dst).unwrap();
1284            }
1285        }
1286    }
1287
1288    fn index_with(root: &Path, recent: Option<(usize, u64)>) {
1289        let mut indexer = crate::Indexer::new(
1290            crate::CacheManager::new(root),
1291            crate::models::IndexConfig::default(),
1292        );
1293        indexer.set_merge_limits(usize::MAX, u64::MAX);
1294        if let Some((files, bytes)) = recent {
1295            indexer.set_recent_limits(files, bytes);
1296        }
1297        indexer.index(root, false).unwrap();
1298    }
1299
1300    /// Every trigram of the published snapshot has the planning size and presence a
1301    /// fresh build of the same tree has (so a trigram whose files are all
1302    /// tombstoned has no list), and so do the live counts and some candidate sets.
1303    fn assert_plans_as_fresh(root: &Path) {
1304        let cache = root.join(".reflex");
1305        let updated = IndexSnapshot::open(&cache).unwrap();
1306        assert!(!updated.base_alone());
1307        // The fresh build indexes a copy of the tree: Windows cannot rename a
1308        // directory whose files are mapped, and `updated` maps `.reflex`.
1309        let copy = tempfile::TempDir::new().unwrap();
1310        copy_tree(root, copy.path());
1311        index_unlimited(copy.path());
1312        let fresh = IndexSnapshot::open(&copy.path().join(".reflex")).unwrap();
1313        assert!(fresh.base_alone());
1314
1315        let mut all: Vec<Trigram> = updated.base().trigrams.trigrams().collect();
1316        for segment in [updated.delta(), updated.recent()].into_iter().flatten() {
1317            all.extend(segment.trigrams.trigrams());
1318        }
1319        all.extend(fresh.base().trigrams.trigrams());
1320        all.sort_unstable();
1321        all.dedup();
1322        let mut live = 0;
1323        for t in all {
1324            let u = updated.list(t).map(|l| l.plan);
1325            let f = fresh.list(t).map(|l| l.plan);
1326            assert_eq!(u, f, "planning size of trigram {t:#08x}");
1327            live += u.is_some() as usize;
1328        }
1329        assert_eq!(live, fresh.trigram_count());
1330        assert_eq!(updated.trigram_count(), fresh.trigram_count());
1331        assert_eq!(updated.live_file_count(), fresh.live_file_count());
1332        assert_eq!(
1333            updated.manifest().unwrap().live_corpus_bytes,
1334            fresh.manifest().unwrap().live_corpus_bytes
1335        );
1336
1337        for pattern in [
1338            "shared",
1339            "common_1",
1340            "rare_marker_qx",
1341            "edited",
1342            "brand_new_token",
1343            "vanishing_zqj",
1344            "line 1",
1345        ] {
1346            assert_eq!(
1347                candidates(&updated, pattern),
1348                candidates(&fresh, pattern),
1349                "{pattern}"
1350            );
1351        }
1352        let fold_paths = |s: &IndexSnapshot| {
1353            let mut v: Vec<(PathBuf, u32)> = s
1354                .search_candidates_fold(b"RARE_MARKER")
1355                .into_iter()
1356                .map(|l| (s.get_file_path(l.file_id).unwrap().to_path_buf(), l.line_no))
1357                .collect();
1358            v.sort();
1359            v
1360        };
1361        assert_eq!(fold_paths(&updated), fold_paths(&fresh));
1362    }
1363
1364    /// 200 files, so file-id gaps pass 128 (where planning size and on-disk size
1365    /// part ways).
1366    fn seed_tree(root: &Path) {
1367        for i in 0..200 {
1368            let rare = if i % 50 == 0 { "rare_marker_qx\n" } else { "" };
1369            write(
1370                root,
1371                &format!("src/f{i:03}.rs"),
1372                &format!("pub fn common_{i}() {{ shared(); }}\n{rare}// line {i}\n"),
1373            );
1374        }
1375        write(root, "only/gone.rs", "fn vanishing_zqj() {}\n");
1376    }
1377
1378    #[test]
1379    fn a_delta_snapshot_plans_and_answers_as_a_fresh_build() {
1380        let temp = tempfile::TempDir::new().unwrap();
1381        let root = temp.path();
1382        seed_tree(root);
1383        index_unlimited(root);
1384
1385        write(root, "src/f000.rs", "pub fn common_0() { edited(); }\n");
1386        write(
1387            root,
1388            "src/f150.rs",
1389            "pub fn common_150() {}\nrare_marker_qx\n",
1390        );
1391        write(
1392            root,
1393            "src/new_file.rs",
1394            "fn brand_new_token() { shared(); }\n",
1395        );
1396        std::fs::remove_file(root.join("src/f100.rs")).unwrap();
1397        std::fs::remove_file(root.join("only/gone.rs")).unwrap();
1398        index_unlimited(root);
1399        let updated = IndexSnapshot::open(&root.join(".reflex")).unwrap();
1400        assert!(candidates(&updated, "vanishing_zqj").is_empty());
1401        drop(updated);
1402        assert_plans_as_fresh(root);
1403    }
1404
1405    /// The two tiers: the recent segment folds into a new delta past its limit;
1406    /// delta files are tombstoned while a recent segment is live; a second fold
1407    /// drops the delta's tombstones.
1408    #[test]
1409    fn a_two_tier_snapshot_plans_and_answers_as_a_fresh_build() {
1410        let temp = tempfile::TempDir::new().unwrap();
1411        let root = temp.path();
1412        seed_tree(root);
1413        index_unlimited(root);
1414        let recent = Some((2, u64::MAX));
1415        let manifest = |root: &Path| read_manifest(&root.join(".reflex")).unwrap().unwrap();
1416
1417        write(root, "src/f000.rs", "pub fn common_0() { edited(); }\n");
1418        write(root, "src/f001.rs", "pub fn common_1() { edited(); }\n");
1419        index_with(root, recent);
1420        let m = manifest(root);
1421        assert!(m.delta.is_none() && m.recent.as_ref().map(|r| r.files) == Some(2));
1422        assert_plans_as_fresh(root);
1423
1424        write(root, "src/f002.rs", "pub fn common_2() { edited(); }\n");
1425        index_with(root, recent);
1426        let m = manifest(root);
1427        assert!(
1428            m.recent.is_none() && m.delta.as_ref().map(|d| d.files) == Some(3),
1429            "{m:?}"
1430        );
1431        assert_plans_as_fresh(root);
1432
1433        // f001 now lives in the delta: its new version tombstones a delta id.
1434        write(
1435            root,
1436            "src/f001.rs",
1437            "pub fn common_1() {}\nrare_marker_qx\n",
1438        );
1439        std::fs::remove_file(root.join("only/gone.rs")).unwrap();
1440        index_with(root, recent);
1441        let m = manifest(root);
1442        assert!(
1443            m.delta.is_some() && m.recent.is_some() && m.tomb_delta.is_some(),
1444            "{m:?}"
1445        );
1446        assert_plans_as_fresh(root);
1447
1448        write(root, "src/f150.rs", "pub fn common_150() {}\n");
1449        write(
1450            root,
1451            "src/new_file.rs",
1452            "fn brand_new_token() { shared(); }\n",
1453        );
1454        index_with(root, recent);
1455        let m = manifest(root);
1456        assert!(m.recent.is_none() && m.tomb_delta.is_none(), "{m:?}");
1457        assert_eq!(m.delta.as_ref().map(|d| d.files), Some(5));
1458        assert_plans_as_fresh(root);
1459    }
1460}