1use std::path::{Path, PathBuf};
31
32use anyhow::{Context, Result};
33use serde::{Deserialize, Serialize};
34
35use crate::content_store::ContentReader;
36use crate::errors::ReflexError;
37use crate::trigram::{
38 EXOTIC_LONG_S_RANGE, FileLocation, ListPart, Trigram, TrigramIndex, TrigramList,
39 exotic_singles, fold_windows, pattern_trigrams, plan_fold_intersection, plan_intersection,
40 union_lists,
41};
42
43pub const MANIFEST: &str = "manifest.json";
45pub const FIXED_CONTENT: &str = "content.bin";
47pub const FIXED_TRIGRAMS: &str = "trigrams.bin";
48
49const MANIFEST_FORMAT: u32 = 2;
51
52#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
54pub struct SegmentFiles {
55 pub content: String,
57 pub trigrams: String,
59 #[serde(default, skip_serializing_if = "Option::is_none")]
61 pub plan: Option<String>,
62 pub files: u64,
64 pub content_bytes: u64,
66 pub trigrams_bytes: u64,
67}
68
69impl SegmentFiles {
70 fn names(&self) -> Vec<&str> {
71 let mut out = vec![self.content.as_str(), self.trigrams.as_str()];
72 out.extend(self.plan.as_deref());
73 out
74 }
75}
76
77#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
79pub struct Manifest {
80 pub format: u32,
81 pub generation: u64,
84 pub base: SegmentFiles,
85 #[serde(default, skip_serializing_if = "Option::is_none")]
87 pub delta: Option<SegmentFiles>,
88 #[serde(default, skip_serializing_if = "Option::is_none")]
90 pub recent: Option<SegmentFiles>,
91 #[serde(default, skip_serializing_if = "Vec::is_empty")]
93 pub tombstones: Vec<u32>,
94 #[serde(default, skip_serializing_if = "Option::is_none")]
97 pub tomb: Option<String>,
98 #[serde(default, skip_serializing_if = "Option::is_none")]
100 pub tomb_delta: Option<String>,
101 pub live_trigrams: u64,
103 #[serde(default)]
106 pub publish_id: u64,
107 pub live_corpus_bytes: u64,
110 #[serde(default)]
112 pub checksum: String,
113}
114
115impl Manifest {
116 pub fn for_base(
118 generation: u64,
119 base: SegmentFiles,
120 live_trigrams: u64,
121 live_corpus_bytes: u64,
122 ) -> Self {
123 Self {
124 format: MANIFEST_FORMAT,
125 generation,
126 base,
127 delta: None,
128 recent: None,
129 tombstones: Vec::new(),
130 tomb: None,
131 tomb_delta: None,
132 live_trigrams,
133 publish_id: 0,
134 live_corpus_bytes,
135 checksum: String::new(),
136 }
137 .sealed()
138 }
139
140 pub fn sealed(mut self) -> Self {
143 self.format = MANIFEST_FORMAT;
144 self.publish_id = new_publish_id();
145 self.checksum = self.compute_checksum();
146 self
147 }
148
149 fn compute_checksum(&self) -> String {
150 let mut unsigned = self.clone();
151 unsigned.checksum = String::new();
152 let body = serde_json::to_vec(&unsigned).expect("manifest serializes");
153 blake3::hash(&body).to_hex().to_string()
154 }
155
156 pub fn files(&self) -> Vec<&str> {
158 let mut out = self.base.names();
159 for segment in [&self.delta, &self.recent].into_iter().flatten() {
160 out.extend(segment.names());
161 }
162 out.extend(self.tomb.as_deref());
163 out.extend(self.tomb_delta.as_deref());
164 out
165 }
166
167 pub fn base_only(&self) -> bool {
169 self.delta.is_none() && self.recent.is_none() && self.tombstones.is_empty()
170 }
171
172 pub fn live_files(&self) -> u64 {
174 self.base.files - self.tombstones.len() as u64
175 + self.delta.as_ref().map_or(0, |d| d.files)
176 + self.recent.as_ref().map_or(0, |d| d.files)
177 }
178}
179
180fn new_publish_id() -> u64 {
182 static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
183 let mut h = blake3::Hasher::new();
184 let nanos = std::time::SystemTime::now()
185 .duration_since(std::time::UNIX_EPOCH)
186 .map_or(0, |d| d.as_nanos());
187 h.update(&nanos.to_le_bytes());
188 h.update(&std::process::id().to_le_bytes());
189 h.update(
190 &COUNTER
191 .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
192 .to_le_bytes(),
193 );
194 let bytes = h.finalize();
195 u64::from_le_bytes(bytes.as_bytes()[..8].try_into().expect("8 bytes")).max(1)
196}
197
198pub fn base_file_names(generation: u64) -> (String, String, String) {
201 (
202 format!("content.{generation}.bin"),
203 format!("trigrams.{generation}.bin"),
204 format!("trigrams.{generation}.plan"),
205 )
206}
207
208pub fn delta_file_names(generation: u64) -> (String, String, String, String) {
211 (
212 format!("delta.{generation}.content.bin"),
213 format!("delta.{generation}.trigrams.bin"),
214 format!("delta.{generation}.plan"),
215 format!("delta.{generation}.tomb"),
216 )
217}
218
219pub fn recent_file_names(generation: u64) -> (String, String, String, String) {
222 (
223 format!("recent.{generation}.content.bin"),
224 format!("recent.{generation}.trigrams.bin"),
225 format!("recent.{generation}.plan"),
226 format!("delta.{generation}.dtomb"),
227 )
228}
229
230pub fn is_generation_file(name: &str) -> bool {
233 let parts: Vec<&str> = name.split('.').collect();
234 let digits = |g: &str| !g.is_empty() && g.bytes().all(|b| b.is_ascii_digit());
235 match parts.as_slice() {
236 ["content" | "trigrams", g, "bin"] => digits(g),
237 ["trigrams", g, "plan"] => digits(g),
238 ["delta" | "recent", g, "content" | "trigrams", "bin"] => digits(g),
239 ["delta" | "recent", g, "plan"] => digits(g),
240 ["delta", g, "tomb" | "dtomb"] => digits(g),
241 _ => false,
242 }
243}
244
245pub fn read_manifest(cache_dir: &Path) -> Result<Option<Manifest>> {
247 let path = cache_dir.join(MANIFEST);
248 let bytes = match std::fs::read(&path) {
249 Ok(b) => b,
250 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
251 Err(e) => return Err(e).with_context(|| format!("Failed to read {}", path.display())),
252 };
253 let manifest: Manifest = serde_json::from_slice(&bytes)
254 .with_context(|| format!("{} is not a valid manifest", path.display()))?;
255 if manifest.format != MANIFEST_FORMAT {
256 anyhow::bail!(
257 "{} has format {}, this binary reads {}",
258 path.display(),
259 manifest.format,
260 MANIFEST_FORMAT
261 );
262 }
263 if manifest.checksum != manifest.compute_checksum() {
264 anyhow::bail!("{} failed its checksum", path.display());
265 }
266 Ok(Some(manifest))
267}
268
269pub fn write_manifest(cache_dir: &Path, manifest: &Manifest) -> Result<()> {
272 use std::io::Write;
273 let path = cache_dir.join(MANIFEST);
274 let tmp = crate::atomic_write::tmp_path_for(&path);
275 let body = serde_json::to_vec_pretty(manifest).context("Failed to serialize the manifest")?;
276 {
277 let mut f = std::fs::File::create(&tmp)
278 .with_context(|| format!("Failed to create {}", tmp.display()))?;
279 f.write_all(&body)?;
280 f.sync_all()?;
281 }
282 crate::atomic_write::atomic_replace(&tmp, &path)
283 .with_context(|| format!("Failed to move {} into place", path.display()))?;
284 sync_dir(cache_dir);
285 Ok(())
286}
287
288pub fn sync_dir(dir: &Path) {
290 #[cfg(unix)]
291 if let Ok(d) = std::fs::File::open(dir) {
292 let _ = d.sync_all();
293 }
294 #[cfg(not(unix))]
295 let _ = dir;
296}
297
298pub fn link_fixed_names(cache_dir: &Path, manifest: &Manifest) {
303 if !manifest.base_only() {
304 unlink_fixed_names(cache_dir);
305 return;
306 }
307 for (generation_file, fixed) in [
308 (&manifest.base.content, FIXED_CONTENT),
309 (&manifest.base.trigrams, FIXED_TRIGRAMS),
310 ] {
311 let target = cache_dir.join(fixed);
312 let tmp = crate::atomic_write::tmp_path_for(&target);
313 let _ = std::fs::remove_file(&tmp);
314 let linked = std::fs::hard_link(cache_dir.join(generation_file), &tmp)
315 .and_then(|_| crate::atomic_write::atomic_replace(&tmp, &target));
316 if let Err(e) = linked {
317 log::debug!(
318 "Cannot link {} to {}: {}; removing it",
319 fixed,
320 generation_file,
321 e
322 );
323 let _ = std::fs::remove_file(&tmp);
324 let _ = std::fs::remove_file(&target);
325 }
326 }
327}
328
329pub fn unlink_fixed_names(cache_dir: &Path) {
332 let mut removed = false;
333 for fixed in [FIXED_CONTENT, FIXED_TRIGRAMS] {
334 match std::fs::remove_file(cache_dir.join(fixed)) {
335 Ok(()) => removed = true,
336 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
337 Err(e) => log::warn!("Could not remove {}: {}", fixed, e),
338 }
339 }
340 if removed {
342 sync_dir(cache_dir);
343 }
344}
345
346pub fn remove_unreferenced(cache_dir: &Path, current: &Manifest, previous: Option<&Manifest>) {
350 let mut keep: std::collections::HashSet<&str> = current.files().into_iter().collect();
351 if let Some(prev) = previous {
352 keep.extend(prev.files());
353 }
354 let Ok(entries) = std::fs::read_dir(cache_dir) else {
355 return;
356 };
357 for entry in entries.flatten() {
358 let name = entry.file_name();
359 let Some(name) = name.to_str() else { continue };
360 if is_generation_file(name)
361 && !keep.contains(name)
362 && let Err(e) = std::fs::remove_file(entry.path())
363 {
364 log::debug!("Could not remove {}: {}", name, e);
365 }
366 }
367}
368
369const TOMB_MAGIC: &[u8; 4] = b"RFTB";
370const TOMB_VERSION: u32 = 1;
371const TOMB_HEADER: usize = 16;
372
373pub struct TombPlan {
378 mmap: Option<memmap2::Mmap>,
379 count: usize,
380}
381
382impl TombPlan {
383 fn load(path: &Path) -> Result<Self> {
384 let file = std::fs::File::open(path)
385 .with_context(|| format!("Failed to open {}", path.display()))?;
386 let len = file.metadata()?.len() as usize;
387 if len == TOMB_HEADER {
388 return Ok(Self {
390 mmap: None,
391 count: 0,
392 });
393 }
394 let mmap = unsafe {
395 memmap2::Mmap::map(&file)
396 .with_context(|| format!("Failed to mmap {}", path.display()))?
397 };
398 if mmap.len() < TOMB_HEADER || &mmap[0..4] != TOMB_MAGIC {
399 anyhow::bail!("{} is not a tombstone planning file", path.display());
400 }
401 if u32::from_le_bytes(mmap[4..8].try_into().unwrap()) != TOMB_VERSION {
402 anyhow::bail!("{} has an unsupported version", path.display());
403 }
404 let count = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
405 if mmap.len() != TOMB_HEADER + count * 8 {
406 anyhow::bail!("{} is truncated", path.display());
407 }
408 Ok(Self {
409 mmap: Some(mmap),
410 count,
411 })
412 }
413
414 #[inline]
415 fn entry(&self, i: usize) -> (Trigram, u32) {
416 let mmap = self.mmap.as_ref().expect("non-empty");
417 let off = TOMB_HEADER + i * 8;
418 (
419 u32::from_le_bytes(mmap[off..off + 4].try_into().unwrap()),
420 u32::from_le_bytes(mmap[off + 4..off + 8].try_into().unwrap()),
421 )
422 }
423
424 pub fn get(&self, trigram: Trigram) -> u32 {
426 let (mut lo, mut hi) = (0usize, self.count);
427 while lo < hi {
428 let mid = lo + (hi - lo) / 2;
429 let (t, size) = self.entry(mid);
430 match t.cmp(&trigram) {
431 std::cmp::Ordering::Less => lo = mid + 1,
432 std::cmp::Ordering::Greater => hi = mid,
433 std::cmp::Ordering::Equal => return size,
434 }
435 }
436 0
437 }
438
439 pub fn entries(&self) -> impl Iterator<Item = (Trigram, u32)> + '_ {
441 (0..self.count).map(|i| self.entry(i))
442 }
443}
444
445pub fn write_tomb_file(path: &Path, entries: &[(Trigram, u32)]) -> Result<()> {
447 use std::io::Write;
448 let tmp = crate::atomic_write::tmp_path_for(path);
449 {
450 let mut w = std::io::BufWriter::new(
451 std::fs::File::create(&tmp)
452 .with_context(|| format!("Failed to create {}", tmp.display()))?,
453 );
454 w.write_all(TOMB_MAGIC)?;
455 w.write_all(&TOMB_VERSION.to_le_bytes())?;
456 w.write_all(&(entries.len() as u64).to_le_bytes())?;
457 for (t, size) in entries {
458 w.write_all(&t.to_le_bytes())?;
459 w.write_all(&size.to_le_bytes())?;
460 }
461 w.flush()?;
462 w.get_ref().sync_all()?;
463 }
464 crate::atomic_write::atomic_replace(&tmp, path)
465 .with_context(|| format!("Failed to move {} into place", path.display()))
466}
467
468fn read_header(cache_dir: &Path, logical: &str, name: &str) -> Result<[u8; 32]> {
470 use std::io::Read;
471 let mut header = [0u8; 32];
472 std::fs::File::open(cache_dir.join(name))
473 .and_then(|mut f| f.read_exact(&mut header))
474 .map_err(|e| {
475 if e.kind() == std::io::ErrorKind::NotFound {
476 missing(logical, name)
477 } else {
478 ReflexError::CacheCorrupted(format!("{logical}: {name}: {e}")).into()
479 }
480 })?;
481 Ok(header)
482}
483
484pub fn check_published(cache_dir: &Path) -> Result<Option<Manifest>> {
491 let Some(m) = read_manifest(cache_dir)? else {
492 return Ok(None);
493 };
494 let u32_at = |h: &[u8; 32], at: usize| u32::from_le_bytes(h[at..at + 4].try_into().unwrap());
495 let u64_at = |h: &[u8; 32], at: usize| u64::from_le_bytes(h[at..at + 8].try_into().unwrap());
496 for segment in [Some(&m.base), m.delta.as_ref(), m.recent.as_ref()]
497 .into_iter()
498 .flatten()
499 {
500 check_size(
501 cache_dir,
502 "content.bin",
503 &segment.content,
504 segment.content_bytes,
505 )?;
506 check_size(
507 cache_dir,
508 "trigrams.bin",
509 &segment.trigrams,
510 segment.trigrams_bytes,
511 )?;
512 let content = read_header(cache_dir, "content.bin", &segment.content)?;
513 if &content[0..4] != b"RFCT"
514 || u32_at(&content, 4) != 2
515 || u64_at(&content, 8) != segment.files
516 {
517 return Err(ReflexError::CacheCorrupted(format!(
518 "content.bin: {} does not hold the manifest's {} files",
519 segment.content, segment.files
520 ))
521 .into());
522 }
523 let trigrams = read_header(cache_dir, "trigrams.bin", &segment.trigrams)?;
524 if &trigrams[0..4] != crate::trigram::MAGIC
525 || u32_at(&trigrams, 4) != crate::trigram::VERSION
526 || u64_at(&trigrams, 16) != segment.files
527 {
528 return Err(ReflexError::CacheCorrupted(format!(
529 "trigrams.bin: {} does not hold the manifest's {} files",
530 segment.trigrams, segment.files
531 ))
532 .into());
533 }
534 if let Some(plan) = &segment.plan
535 && !cache_dir.join(plan).exists()
536 {
537 return Err(missing("trigrams.bin", plan));
538 }
539 }
540 for tomb in [&m.tomb, &m.tomb_delta].into_iter().flatten() {
541 if !cache_dir.join(tomb).exists() {
542 return Err(missing("trigrams.bin", tomb));
543 }
544 }
545 let bound = m.base.files + m.delta.as_ref().map_or(0, |d| d.files);
546 if m.tombstones.iter().any(|&id| id as u64 >= bound) {
547 return Err(ReflexError::CacheCorrupted(
548 "content.bin: a tombstone is past the base and delta's files".to_string(),
549 )
550 .into());
551 }
552 Ok(Some(m))
553}
554
555pub struct Segment {
557 pub content: ContentReader,
558 pub trigrams: TrigramIndex,
559}
560
561pub struct IndexSnapshot {
564 base: Segment,
565 delta: Option<Segment>,
566 recent: Option<Segment>,
567 dead: Vec<u64>,
569 dead_count: usize,
570 tomb: Option<TombPlan>,
571 tomb_delta: Option<TombPlan>,
572 base_len: u32,
574 delta_len: u32,
576 manifest: Option<Manifest>,
578}
579
580impl std::fmt::Debug for IndexSnapshot {
581 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
582 f.debug_struct("IndexSnapshot")
583 .field("files", &self.live_file_count())
584 .field("generation", &self.generation())
585 .finish()
586 }
587}
588
589fn is_not_found(err: &anyhow::Error) -> bool {
591 err.chain().any(|c| {
592 c.downcast_ref::<std::io::Error>()
593 .is_some_and(|e| e.kind() == std::io::ErrorKind::NotFound)
594 })
595}
596
597fn missing(logical: &str, name: &str) -> anyhow::Error {
600 anyhow::Error::new(std::io::Error::new(
601 std::io::ErrorKind::NotFound,
602 format!("{} is missing", name),
603 ))
604 .context(ReflexError::CacheCorrupted(format!(
605 "{}: {} is missing",
606 logical, name
607 )))
608}
609
610fn check_size(cache_dir: &Path, logical: &str, name: &str, expected: u64) -> Result<()> {
612 match std::fs::metadata(cache_dir.join(name)) {
613 Ok(md) if md.len() == expected => Ok(()),
614 Ok(md) => Err(ReflexError::CacheCorrupted(format!(
615 "{}: {} is {} bytes, the manifest says {}",
616 logical,
617 name,
618 md.len(),
619 expected
620 ))
621 .into()),
622 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(missing(logical, name)),
623 Err(e) => Err(e).with_context(|| format!("Failed to stat {}", name)),
624 }
625}
626
627fn open_segment(cache_dir: &Path, files: &SegmentFiles) -> Result<Segment> {
629 check_size(
630 cache_dir,
631 "content.bin",
632 &files.content,
633 files.content_bytes,
634 )?;
635 check_size(
636 cache_dir,
637 "trigrams.bin",
638 &files.trigrams,
639 files.trigrams_bytes,
640 )?;
641 let content = ContentReader::open(cache_dir.join(&files.content))
642 .map_err(|e| ReflexError::CacheCorrupted(format!("content.bin: {e:#}")))?;
643 let mut trigrams = TrigramIndex::load(cache_dir.join(&files.trigrams))
644 .map_err(|e| ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}")))?;
645 if let Some(plan) = &files.plan {
648 let plan_path = cache_dir.join(plan);
649 if !plan_path.exists() {
650 return Err(missing("trigrams.bin", plan));
651 }
652 trigrams
653 .attach_plan(&plan_path)
654 .map_err(|e| ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}")))?;
655 }
656 check_counts(&content, &trigrams)?;
657 if content.file_count() as u64 != files.files {
658 return Err(ReflexError::CacheCorrupted(format!(
659 "content.bin holds {} files but the manifest lists {}",
660 content.file_count(),
661 files.files
662 ))
663 .into());
664 }
665 Ok(Segment { content, trigrams })
666}
667
668fn check_counts(content: &ContentReader, trigrams: &TrigramIndex) -> Result<()> {
669 if trigrams.file_count() != content.file_count() {
670 return Err(ReflexError::CacheCorrupted(format!(
671 "trigrams.bin lists {} files but content.bin holds {} (index written by two runs?)",
672 trigrams.file_count(),
673 content.file_count()
674 ))
675 .into());
676 }
677 Ok(())
678}
679
680impl IndexSnapshot {
681 pub fn open(cache_dir: &Path) -> Result<Self> {
685 Self::open_reading(cache_dir, || read_manifest(cache_dir))
686 }
687
688 fn open_reading(
690 cache_dir: &Path,
691 mut read: impl FnMut() -> Result<Option<Manifest>>,
692 ) -> Result<Self> {
693 let mut last: Option<anyhow::Error> = None;
694 for attempt in 0..5u64 {
695 let manifest =
696 read().map_err(|e| ReflexError::CacheCorrupted(format!("content.bin: {e:#}")))?;
697 let result = match manifest {
698 None => return Self::open_legacy(cache_dir),
699 Some(m) => Self::open_manifest(cache_dir, m),
700 };
701 match result {
702 Ok(s) => return Ok(s),
703 Err(e) if is_not_found(&e) => {
704 last = Some(e);
705 std::thread::sleep(std::time::Duration::from_millis(10 * (attempt + 1)));
706 }
707 Err(e) => return Err(e),
708 }
709 }
710 Err(last.expect("at least one attempt"))
711 }
712
713 fn open_manifest(cache_dir: &Path, manifest: Manifest) -> Result<Self> {
714 let base = open_segment(cache_dir, &manifest.base)?;
715 let base_len = base.content.file_count() as u32;
716 let delta = match &manifest.delta {
717 Some(d) => Some(open_segment(cache_dir, d)?),
718 None => None,
719 };
720 let delta_len = delta.as_ref().map_or(0, |d| d.content.file_count() as u32);
721 let recent = match &manifest.recent {
722 Some(d) => Some(open_segment(cache_dir, d)?),
723 None => None,
724 };
725 let load_tomb = |name: &Option<String>| -> Result<Option<TombPlan>> {
726 let Some(name) = name else { return Ok(None) };
727 let tomb_path = cache_dir.join(name);
728 if !tomb_path.exists() {
729 return Err(missing("trigrams.bin", name));
730 }
731 Ok(Some(TombPlan::load(&tomb_path).map_err(|e| {
732 ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}"))
733 })?))
734 };
735 let tomb = load_tomb(&manifest.tomb)?;
736 let tomb_delta = load_tomb(&manifest.tomb_delta)?;
737 let dead_bound = base_len + delta_len;
738 let mut dead = vec![0u64; (dead_bound as usize).div_ceil(64)];
739 for &id in &manifest.tombstones {
740 if id >= dead_bound {
741 return Err(ReflexError::CacheCorrupted(format!(
742 "content.bin: tombstone {} is past the base and delta's {} files",
743 id, dead_bound
744 ))
745 .into());
746 }
747 dead[(id / 64) as usize] |= 1u64 << (id % 64);
748 }
749 Ok(Self {
750 base,
751 delta,
752 recent,
753 dead,
754 dead_count: manifest.tombstones.len(),
755 tomb,
756 tomb_delta,
757 base_len,
758 delta_len,
759 manifest: Some(manifest),
760 })
761 }
762
763 fn open_legacy(cache_dir: &Path) -> Result<Self> {
764 let content = ContentReader::open(cache_dir.join(FIXED_CONTENT))
765 .map_err(|e| ReflexError::CacheCorrupted(format!("content.bin: {e:#}")))?;
766 let trigrams_path = cache_dir.join(FIXED_TRIGRAMS);
767 let trigrams = if trigrams_path.exists() {
768 match TrigramIndex::load(&trigrams_path) {
769 Ok(index) => index,
770 Err(e) if e.to_string().contains("Unsupported trigrams.bin version") => {
775 log::warn!("{}; rebuilding trigram index in memory for this process", e);
776 crate::query::result::rebuild_trigram_index(&content)?
777 }
778 Err(e) => {
779 return Err(ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}")).into());
780 }
781 }
782 } else {
783 log::debug!("trigrams.bin not found, rebuilding from content store");
784 crate::query::result::rebuild_trigram_index(&content)?
785 };
786 check_counts(&content, &trigrams)?;
787 let base_len = content.file_count() as u32;
788 Ok(Self {
789 base: Segment { content, trigrams },
790 delta: None,
791 recent: None,
792 dead: Vec::new(),
793 dead_count: 0,
794 tomb: None,
795 tomb_delta: None,
796 base_len,
797 delta_len: 0,
798 manifest: None,
799 })
800 }
801
802 pub fn generation(&self) -> Option<u64> {
804 self.manifest.as_ref().map(|m| m.generation)
805 }
806
807 pub fn manifest(&self) -> Option<&Manifest> {
809 self.manifest.as_ref()
810 }
811
812 pub fn base(&self) -> &Segment {
814 &self.base
815 }
816
817 pub fn delta(&self) -> Option<&Segment> {
819 self.delta.as_ref()
820 }
821
822 pub fn recent(&self) -> Option<&Segment> {
824 self.recent.as_ref()
825 }
826
827 pub fn base_len(&self) -> u32 {
829 self.base_len
830 }
831
832 pub fn delta_len(&self) -> u32 {
834 self.delta_len
835 }
836
837 fn recent_len(&self) -> u32 {
838 self.recent
839 .as_ref()
840 .map_or(0, |d| d.content.file_count() as u32)
841 }
842
843 pub fn tomb(&self) -> Option<&TombPlan> {
845 self.tomb.as_ref()
846 }
847
848 pub fn tomb_delta(&self) -> Option<&TombPlan> {
850 self.tomb_delta.as_ref()
851 }
852
853 #[inline]
855 pub fn is_dead(&self, id: u32) -> bool {
856 id < self.base_len + self.delta_len
857 && self
858 .dead
859 .get((id / 64) as usize)
860 .is_some_and(|w| w & (1u64 << (id % 64)) != 0)
861 }
862
863 pub fn id_bound(&self) -> u32 {
865 self.base_len + self.delta_len + self.recent_len()
866 }
867
868 pub fn live_file_count(&self) -> usize {
870 self.id_bound() as usize - self.dead_count
871 }
872
873 pub fn is_live(&self, id: u32) -> bool {
875 id < self.id_bound() && !self.is_dead(id)
876 }
877
878 pub fn live_ids(&self) -> impl Iterator<Item = u32> + '_ {
880 (0..self.id_bound()).filter(|&id| !self.is_dead(id))
881 }
882
883 #[inline]
885 fn route(&self, id: u32) -> (&Segment, u32) {
886 if id < self.base_len {
887 return (&self.base, id);
888 }
889 match (&self.delta, &self.recent) {
890 (Some(delta), _) if id < self.base_len + self.delta_len => (delta, id - self.base_len),
891 (_, Some(recent)) => (recent, id - self.base_len - self.delta_len),
892 _ => (&self.base, id),
893 }
894 }
895
896 pub fn get_file_content(&self, file_id: u32) -> Result<&str> {
898 let (segment, local) = self.route(file_id);
899 segment.content.get_file_content(local)
900 }
901
902 pub fn release_pages(&self) {
905 for segment in [Some(&self.base), self.delta.as_ref(), self.recent.as_ref()]
906 .into_iter()
907 .flatten()
908 {
909 segment.content.release_pages();
910 }
911 }
912
913 pub fn file_len(&self, file_id: u32) -> Option<u64> {
915 let (segment, local) = self.route(file_id);
916 segment.content.file_len(local)
917 }
918
919 pub fn get_file_path(&self, file_id: u32) -> Option<&Path> {
921 if !self.is_live(file_id) {
922 return None;
923 }
924 let (segment, local) = self.route(file_id);
925 segment.content.get_file_path(local)
926 }
927
928 pub fn get_context_by_line(
930 &self,
931 file_id: u32,
932 line_number: usize,
933 context_lines: usize,
934 ) -> Result<(Vec<String>, Vec<String>)> {
935 let (segment, local) = self.route(file_id);
936 segment
937 .content
938 .get_context_by_line(local, line_number, context_lines)
939 }
940
941 pub fn get_file_id_by_path(&self, path: &str) -> Option<u32> {
944 let normalized = path.strip_prefix("./").unwrap_or(path);
945 self.live_ids().find(|&id| {
946 self.get_file_path(id)
947 .and_then(|p| p.to_str())
948 .is_some_and(|p| p.strip_prefix("./").unwrap_or(p) == normalized)
949 })
950 }
951
952 fn base_alone(&self) -> bool {
954 self.delta.is_none() && self.recent.is_none() && self.dead_count == 0
955 }
956
957 pub fn live_plan(&self, trigram: Trigram) -> u64 {
959 self.list(trigram).map_or(0, |l| l.plan)
960 }
961
962 fn list(&self, trigram: Trigram) -> Option<TrigramList<'_>> {
966 let mut parts: Vec<ListPart<'_>> = Vec::with_capacity(3);
967 let mut plan: i64 = 0;
968 let dead: Option<&[u64]> = (self.dead_count > 0).then_some(self.dead.as_slice());
969 if let Some((mut part, base_plan)) = self.base.trigrams.list_part(trigram) {
970 part.dead = dead;
971 parts.push(part);
972 plan += base_plan as i64;
973 }
974 for tomb in [&self.tomb, &self.tomb_delta].into_iter().flatten() {
975 plan -= tomb.get(trigram) as i64;
976 }
977 if let Some(delta) = &self.delta
978 && let Some((mut part, delta_plan)) = delta.trigrams.list_part(trigram)
979 {
980 part.id_offset = self.base_len;
981 part.dead = dead;
982 parts.push(part);
983 plan += delta_plan as i64;
984 }
985 if let Some(recent) = &self.recent
986 && let Some((mut part, recent_plan)) = recent.trigrams.list_part(trigram)
987 {
988 part.id_offset = self.base_len + self.delta_len;
989 parts.push(part);
990 plan += recent_plan as i64;
991 }
992 if parts.is_empty() || plan <= 0 {
995 return None;
996 }
997 Some(TrigramList {
998 trigram,
999 plan: plan as u64,
1000 parts,
1001 })
1002 }
1003
1004 pub fn search_candidates(&self, pattern: &str) -> Vec<FileLocation> {
1006 if self.base_alone() {
1007 return self.base.trigrams.search_candidates(pattern);
1008 }
1009 let lists = pattern_trigrams(pattern)
1010 .into_iter()
1011 .map(|t| self.list(t))
1012 .collect::<Vec<_>>();
1013 if lists.is_empty() {
1014 return vec![];
1015 }
1016 plan_intersection(lists, true)
1017 }
1018
1019 pub fn search_candidates_fold(&self, literal: &[u8]) -> Vec<FileLocation> {
1021 if self.base_alone() {
1022 return self.base.trigrams.search_candidates_fold(literal);
1023 }
1024 let Some(windows) = fold_windows(literal) else {
1025 return vec![];
1026 };
1027 let lists = windows
1028 .iter()
1029 .map(|variants| variants.iter().filter_map(|&t| self.list(t)).collect())
1030 .collect();
1031 plan_fold_intersection(lists)
1032 }
1033
1034 pub fn exotic_fold_lines(&self) -> Vec<FileLocation> {
1036 if self.base_alone() {
1037 return self.base.trigrams.exotic_fold_lines();
1038 }
1039 let (lo, hi) = EXOTIC_LONG_S_RANGE;
1040 let mut trigrams: Vec<Trigram> = self
1041 .base
1042 .trigrams
1043 .list_parts_in(lo, hi)
1044 .into_iter()
1045 .map(|(t, _, _)| t)
1046 .collect();
1047 for segment in [&self.delta, &self.recent].into_iter().flatten() {
1048 trigrams.extend(
1049 segment
1050 .trigrams
1051 .list_parts_in(lo, hi)
1052 .into_iter()
1053 .map(|(t, _, _)| t),
1054 );
1055 }
1056 trigrams.extend(exotic_singles());
1057 trigrams.sort_unstable();
1058 trigrams.dedup();
1059 let lists: Vec<TrigramList> = trigrams.into_iter().filter_map(|t| self.list(t)).collect();
1060 union_lists(&lists)
1061 }
1062
1063 pub fn trigram_count(&self) -> usize {
1065 match &self.manifest {
1066 Some(m) => m.live_trigrams as usize,
1067 None => self.base.trigrams.trigram_count(),
1068 }
1069 }
1070}
1071
1072#[derive(Debug, Clone)]
1075pub struct StoreSummary {
1076 pub content_files: Vec<PathBuf>,
1078 pub trigram_files: Vec<PathBuf>,
1080 pub other_files: Vec<PathBuf>,
1082 pub live_corpus_bytes: Option<u64>,
1084 pub live_trigrams: Option<u64>,
1086}
1087
1088impl StoreSummary {
1089 pub fn files(&self) -> impl Iterator<Item = &PathBuf> {
1091 self.content_files
1092 .iter()
1093 .chain(&self.trigram_files)
1094 .chain(&self.other_files)
1095 }
1096}
1097
1098pub fn store_summary(cache_dir: &Path) -> StoreSummary {
1101 match read_manifest(cache_dir) {
1102 Ok(Some(m)) => {
1103 let mut content_files = vec![cache_dir.join(&m.base.content)];
1104 let mut trigram_files = vec![cache_dir.join(&m.base.trigrams)];
1105 let mut other_files: Vec<PathBuf> =
1106 m.base.plan.iter().map(|p| cache_dir.join(p)).collect();
1107 for d in [&m.delta, &m.recent].into_iter().flatten() {
1108 content_files.push(cache_dir.join(&d.content));
1109 trigram_files.push(cache_dir.join(&d.trigrams));
1110 other_files.extend(d.plan.iter().map(|p| cache_dir.join(p)));
1111 }
1112 other_files.extend(m.tomb.iter().map(|t| cache_dir.join(t)));
1113 other_files.extend(m.tomb_delta.iter().map(|t| cache_dir.join(t)));
1114 StoreSummary {
1115 content_files,
1116 trigram_files,
1117 other_files,
1118 live_corpus_bytes: Some(m.live_corpus_bytes),
1119 live_trigrams: Some(m.live_trigrams),
1120 }
1121 }
1122 _ => StoreSummary {
1123 content_files: vec![cache_dir.join(FIXED_CONTENT)],
1124 trigram_files: vec![cache_dir.join(FIXED_TRIGRAMS)],
1125 other_files: Vec::new(),
1126 live_corpus_bytes: None,
1127 live_trigrams: None,
1128 },
1129 }
1130}
1131
1132#[cfg(test)]
1133mod tests {
1134 use super::*;
1135
1136 #[test]
1137 fn generation_file_names() {
1138 assert!(is_generation_file("content.12.bin"));
1139 assert!(is_generation_file("trigrams.3.bin"));
1140 assert!(is_generation_file("trigrams.3.plan"));
1141 assert!(is_generation_file("delta.4.content.bin"));
1142 assert!(is_generation_file("delta.4.trigrams.bin"));
1143 assert!(is_generation_file("delta.4.plan"));
1144 assert!(is_generation_file("delta.4.tomb"));
1145 assert!(is_generation_file("delta.4.dtomb"));
1146 assert!(is_generation_file("recent.4.content.bin"));
1147 assert!(is_generation_file("recent.4.trigrams.bin"));
1148 assert!(is_generation_file("recent.4.plan"));
1149 assert!(!is_generation_file("recent.4.tomb"));
1150 assert!(!is_generation_file("content.3.plan"));
1151 assert!(!is_generation_file("content.bin"));
1152 assert!(!is_generation_file("trigrams.bin"));
1153 assert!(!is_generation_file("content.x.bin"));
1154 assert!(!is_generation_file("meta.db"));
1155 assert!(!is_generation_file("manifest.json"));
1156 }
1157
1158 #[test]
1159 fn manifest_round_trip_and_checksum() {
1160 let temp = tempfile::TempDir::new().unwrap();
1161 let (content, trigrams, plan) = base_file_names(7);
1162 let m = Manifest::for_base(
1163 7,
1164 SegmentFiles {
1165 content,
1166 trigrams,
1167 plan: Some(plan),
1168 files: 3,
1169 content_bytes: 100,
1170 trigrams_bytes: 200,
1171 },
1172 42,
1173 90,
1174 );
1175 write_manifest(temp.path(), &m).unwrap();
1176 assert_eq!(read_manifest(temp.path()).unwrap(), Some(m));
1177
1178 let path = temp.path().join(MANIFEST);
1180 let text = std::fs::read_to_string(&path)
1181 .unwrap()
1182 .replace("\"generation\": 7", "\"generation\": 8");
1183 std::fs::write(&path, text).unwrap();
1184 assert!(read_manifest(temp.path()).is_err());
1185 }
1186
1187 #[test]
1188 fn tomb_file_round_trip() {
1189 let temp = tempfile::TempDir::new().unwrap();
1190 let path = temp.path().join("t.tomb");
1191 write_tomb_file(&path, &[(3, 10), (70_000, 4), (0xE2_84_AA, 7)]).unwrap();
1192 let tomb = TombPlan::load(&path).unwrap();
1193 assert_eq!(tomb.get(3), 10);
1194 assert_eq!(tomb.get(70_000), 4);
1195 assert_eq!(tomb.get(0xE2_84_AA), 7);
1196 assert_eq!(tomb.get(4), 0);
1197 write_tomb_file(&path, &[]).unwrap();
1198 assert_eq!(TombPlan::load(&path).unwrap().get(3), 0);
1199 }
1200
1201 #[test]
1205 fn open_retries_while_a_publish_replaces_the_manifest() {
1206 let temp = tempfile::TempDir::new().unwrap();
1207 let root = temp.path();
1208 write(root, "a.rs", "fn alpha() {}\n");
1209 index_unlimited(root);
1210 let cache = root.join(".reflex");
1211 let good = read_manifest(&cache).unwrap().unwrap();
1212 let mut files = good.base.clone();
1213 files.content = "content.999.bin".to_string();
1214 let broken = Manifest::for_base(
1215 good.generation + 1,
1216 files,
1217 good.live_trigrams,
1218 good.live_corpus_bytes,
1219 );
1220
1221 let mut reads = 0;
1223 let snapshot = IndexSnapshot::open_reading(&cache, || {
1224 reads += 1;
1225 Ok(Some(if reads == 1 {
1226 broken.clone()
1227 } else {
1228 good.clone()
1229 }))
1230 })
1231 .expect("a retry opens the next manifest");
1232 assert_eq!(reads, 2);
1233 assert_eq!(snapshot.generation(), Some(good.generation));
1234
1235 write_manifest(&cache, &broken).unwrap();
1236 let err = IndexSnapshot::open(&cache).unwrap_err();
1237 assert!(
1238 matches!(
1239 err.downcast_ref::<ReflexError>(),
1240 Some(ReflexError::CacheCorrupted(_))
1241 ),
1242 "{err:#}"
1243 );
1244 }
1245
1246 fn index_unlimited(root: &Path) {
1247 index_with(root, None);
1248 }
1249
1250 fn write(root: &Path, rel: &str, body: &str) {
1251 let p = root.join(rel);
1252 std::fs::create_dir_all(p.parent().unwrap()).unwrap();
1253 std::fs::write(p, body).unwrap();
1254 }
1255
1256 fn candidates(snapshot: &IndexSnapshot, pattern: &str) -> Vec<(PathBuf, u32)> {
1258 let mut out: Vec<(PathBuf, u32)> = snapshot
1259 .search_candidates(pattern)
1260 .into_iter()
1261 .map(|l| {
1262 let path = snapshot.get_file_path(l.file_id).expect("a live candidate");
1263 (path.to_path_buf(), l.line_no)
1264 })
1265 .collect();
1266 out.sort();
1267 out
1268 }
1269
1270 fn copy_tree(from: &Path, to: &Path) {
1272 for entry in std::fs::read_dir(from).unwrap() {
1273 let entry = entry.unwrap();
1274 let name = entry.file_name();
1275 if name == ".reflex" {
1276 continue;
1277 }
1278 let (src, dst) = (entry.path(), to.join(&name));
1279 if entry.file_type().unwrap().is_dir() {
1280 std::fs::create_dir_all(&dst).unwrap();
1281 copy_tree(&src, &dst);
1282 } else {
1283 std::fs::copy(&src, &dst).unwrap();
1284 }
1285 }
1286 }
1287
1288 fn index_with(root: &Path, recent: Option<(usize, u64)>) {
1289 let mut indexer = crate::Indexer::new(
1290 crate::CacheManager::new(root),
1291 crate::models::IndexConfig::default(),
1292 );
1293 indexer.set_merge_limits(usize::MAX, u64::MAX);
1294 if let Some((files, bytes)) = recent {
1295 indexer.set_recent_limits(files, bytes);
1296 }
1297 indexer.index(root, false).unwrap();
1298 }
1299
1300 fn assert_plans_as_fresh(root: &Path) {
1304 let cache = root.join(".reflex");
1305 let updated = IndexSnapshot::open(&cache).unwrap();
1306 assert!(!updated.base_alone());
1307 let copy = tempfile::TempDir::new().unwrap();
1310 copy_tree(root, copy.path());
1311 index_unlimited(copy.path());
1312 let fresh = IndexSnapshot::open(©.path().join(".reflex")).unwrap();
1313 assert!(fresh.base_alone());
1314
1315 let mut all: Vec<Trigram> = updated.base().trigrams.trigrams().collect();
1316 for segment in [updated.delta(), updated.recent()].into_iter().flatten() {
1317 all.extend(segment.trigrams.trigrams());
1318 }
1319 all.extend(fresh.base().trigrams.trigrams());
1320 all.sort_unstable();
1321 all.dedup();
1322 let mut live = 0;
1323 for t in all {
1324 let u = updated.list(t).map(|l| l.plan);
1325 let f = fresh.list(t).map(|l| l.plan);
1326 assert_eq!(u, f, "planning size of trigram {t:#08x}");
1327 live += u.is_some() as usize;
1328 }
1329 assert_eq!(live, fresh.trigram_count());
1330 assert_eq!(updated.trigram_count(), fresh.trigram_count());
1331 assert_eq!(updated.live_file_count(), fresh.live_file_count());
1332 assert_eq!(
1333 updated.manifest().unwrap().live_corpus_bytes,
1334 fresh.manifest().unwrap().live_corpus_bytes
1335 );
1336
1337 for pattern in [
1338 "shared",
1339 "common_1",
1340 "rare_marker_qx",
1341 "edited",
1342 "brand_new_token",
1343 "vanishing_zqj",
1344 "line 1",
1345 ] {
1346 assert_eq!(
1347 candidates(&updated, pattern),
1348 candidates(&fresh, pattern),
1349 "{pattern}"
1350 );
1351 }
1352 let fold_paths = |s: &IndexSnapshot| {
1353 let mut v: Vec<(PathBuf, u32)> = s
1354 .search_candidates_fold(b"RARE_MARKER")
1355 .into_iter()
1356 .map(|l| (s.get_file_path(l.file_id).unwrap().to_path_buf(), l.line_no))
1357 .collect();
1358 v.sort();
1359 v
1360 };
1361 assert_eq!(fold_paths(&updated), fold_paths(&fresh));
1362 }
1363
1364 fn seed_tree(root: &Path) {
1367 for i in 0..200 {
1368 let rare = if i % 50 == 0 { "rare_marker_qx\n" } else { "" };
1369 write(
1370 root,
1371 &format!("src/f{i:03}.rs"),
1372 &format!("pub fn common_{i}() {{ shared(); }}\n{rare}// line {i}\n"),
1373 );
1374 }
1375 write(root, "only/gone.rs", "fn vanishing_zqj() {}\n");
1376 }
1377
1378 #[test]
1379 fn a_delta_snapshot_plans_and_answers_as_a_fresh_build() {
1380 let temp = tempfile::TempDir::new().unwrap();
1381 let root = temp.path();
1382 seed_tree(root);
1383 index_unlimited(root);
1384
1385 write(root, "src/f000.rs", "pub fn common_0() { edited(); }\n");
1386 write(
1387 root,
1388 "src/f150.rs",
1389 "pub fn common_150() {}\nrare_marker_qx\n",
1390 );
1391 write(
1392 root,
1393 "src/new_file.rs",
1394 "fn brand_new_token() { shared(); }\n",
1395 );
1396 std::fs::remove_file(root.join("src/f100.rs")).unwrap();
1397 std::fs::remove_file(root.join("only/gone.rs")).unwrap();
1398 index_unlimited(root);
1399 let updated = IndexSnapshot::open(&root.join(".reflex")).unwrap();
1400 assert!(candidates(&updated, "vanishing_zqj").is_empty());
1401 drop(updated);
1402 assert_plans_as_fresh(root);
1403 }
1404
1405 #[test]
1409 fn a_two_tier_snapshot_plans_and_answers_as_a_fresh_build() {
1410 let temp = tempfile::TempDir::new().unwrap();
1411 let root = temp.path();
1412 seed_tree(root);
1413 index_unlimited(root);
1414 let recent = Some((2, u64::MAX));
1415 let manifest = |root: &Path| read_manifest(&root.join(".reflex")).unwrap().unwrap();
1416
1417 write(root, "src/f000.rs", "pub fn common_0() { edited(); }\n");
1418 write(root, "src/f001.rs", "pub fn common_1() { edited(); }\n");
1419 index_with(root, recent);
1420 let m = manifest(root);
1421 assert!(m.delta.is_none() && m.recent.as_ref().map(|r| r.files) == Some(2));
1422 assert_plans_as_fresh(root);
1423
1424 write(root, "src/f002.rs", "pub fn common_2() { edited(); }\n");
1425 index_with(root, recent);
1426 let m = manifest(root);
1427 assert!(
1428 m.recent.is_none() && m.delta.as_ref().map(|d| d.files) == Some(3),
1429 "{m:?}"
1430 );
1431 assert_plans_as_fresh(root);
1432
1433 write(
1435 root,
1436 "src/f001.rs",
1437 "pub fn common_1() {}\nrare_marker_qx\n",
1438 );
1439 std::fs::remove_file(root.join("only/gone.rs")).unwrap();
1440 index_with(root, recent);
1441 let m = manifest(root);
1442 assert!(
1443 m.delta.is_some() && m.recent.is_some() && m.tomb_delta.is_some(),
1444 "{m:?}"
1445 );
1446 assert_plans_as_fresh(root);
1447
1448 write(root, "src/f150.rs", "pub fn common_150() {}\n");
1449 write(
1450 root,
1451 "src/new_file.rs",
1452 "fn brand_new_token() { shared(); }\n",
1453 );
1454 index_with(root, recent);
1455 let m = manifest(root);
1456 assert!(m.recent.is_none() && m.tomb_delta.is_none(), "{m:?}");
1457 assert_eq!(m.delta.as_ref().map(|d| d.files), Some(5));
1458 assert_plans_as_fresh(root);
1459 }
1460}