use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use crate::content_store::ContentReader;
use crate::errors::ReflexError;
use crate::trigram::{
EXOTIC_LONG_S_RANGE, FileLocation, ListPart, Trigram, TrigramIndex, TrigramList,
exotic_singles, fold_windows, pattern_trigrams, plan_fold_intersection, plan_intersection,
union_lists,
};
pub const MANIFEST: &str = "manifest.json";
pub const FIXED_CONTENT: &str = "content.bin";
pub const FIXED_TRIGRAMS: &str = "trigrams.bin";
const MANIFEST_FORMAT: u32 = 2;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SegmentFiles {
pub content: String,
pub trigrams: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub plan: Option<String>,
pub files: u64,
pub content_bytes: u64,
pub trigrams_bytes: u64,
}
impl SegmentFiles {
fn names(&self) -> Vec<&str> {
let mut out = vec![self.content.as_str(), self.trigrams.as_str()];
out.extend(self.plan.as_deref());
out
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Manifest {
pub format: u32,
pub generation: u64,
pub base: SegmentFiles,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub delta: Option<SegmentFiles>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub recent: Option<SegmentFiles>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tombstones: Vec<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tomb: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tomb_delta: Option<String>,
pub live_trigrams: u64,
#[serde(default)]
pub publish_id: u64,
pub live_corpus_bytes: u64,
#[serde(default)]
pub checksum: String,
}
impl Manifest {
pub fn for_base(
generation: u64,
base: SegmentFiles,
live_trigrams: u64,
live_corpus_bytes: u64,
) -> Self {
Self {
format: MANIFEST_FORMAT,
generation,
base,
delta: None,
recent: None,
tombstones: Vec::new(),
tomb: None,
tomb_delta: None,
live_trigrams,
publish_id: 0,
live_corpus_bytes,
checksum: String::new(),
}
.sealed()
}
pub fn sealed(mut self) -> Self {
self.format = MANIFEST_FORMAT;
self.publish_id = new_publish_id();
self.checksum = self.compute_checksum();
self
}
fn compute_checksum(&self) -> String {
let mut unsigned = self.clone();
unsigned.checksum = String::new();
let body = serde_json::to_vec(&unsigned).expect("manifest serializes");
blake3::hash(&body).to_hex().to_string()
}
pub fn files(&self) -> Vec<&str> {
let mut out = self.base.names();
for segment in [&self.delta, &self.recent].into_iter().flatten() {
out.extend(segment.names());
}
out.extend(self.tomb.as_deref());
out.extend(self.tomb_delta.as_deref());
out
}
pub fn base_only(&self) -> bool {
self.delta.is_none() && self.recent.is_none() && self.tombstones.is_empty()
}
pub fn live_files(&self) -> u64 {
self.base.files - self.tombstones.len() as u64
+ self.delta.as_ref().map_or(0, |d| d.files)
+ self.recent.as_ref().map_or(0, |d| d.files)
}
}
fn new_publish_id() -> u64 {
static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let mut h = blake3::Hasher::new();
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos());
h.update(&nanos.to_le_bytes());
h.update(&std::process::id().to_le_bytes());
h.update(
&COUNTER
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
.to_le_bytes(),
);
let bytes = h.finalize();
u64::from_le_bytes(bytes.as_bytes()[..8].try_into().expect("8 bytes")).max(1)
}
pub fn base_file_names(generation: u64) -> (String, String, String) {
(
format!("content.{generation}.bin"),
format!("trigrams.{generation}.bin"),
format!("trigrams.{generation}.plan"),
)
}
pub fn delta_file_names(generation: u64) -> (String, String, String, String) {
(
format!("delta.{generation}.content.bin"),
format!("delta.{generation}.trigrams.bin"),
format!("delta.{generation}.plan"),
format!("delta.{generation}.tomb"),
)
}
pub fn recent_file_names(generation: u64) -> (String, String, String, String) {
(
format!("recent.{generation}.content.bin"),
format!("recent.{generation}.trigrams.bin"),
format!("recent.{generation}.plan"),
format!("delta.{generation}.dtomb"),
)
}
pub fn is_generation_file(name: &str) -> bool {
let parts: Vec<&str> = name.split('.').collect();
let digits = |g: &str| !g.is_empty() && g.bytes().all(|b| b.is_ascii_digit());
match parts.as_slice() {
["content" | "trigrams", g, "bin"] => digits(g),
["trigrams", g, "plan"] => digits(g),
["delta" | "recent", g, "content" | "trigrams", "bin"] => digits(g),
["delta" | "recent", g, "plan"] => digits(g),
["delta", g, "tomb" | "dtomb"] => digits(g),
_ => false,
}
}
pub fn read_manifest(cache_dir: &Path) -> Result<Option<Manifest>> {
let path = cache_dir.join(MANIFEST);
let bytes = match std::fs::read(&path) {
Ok(b) => b,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e).with_context(|| format!("Failed to read {}", path.display())),
};
let manifest: Manifest = serde_json::from_slice(&bytes)
.with_context(|| format!("{} is not a valid manifest", path.display()))?;
if manifest.format != MANIFEST_FORMAT {
anyhow::bail!(
"{} has format {}, this binary reads {}",
path.display(),
manifest.format,
MANIFEST_FORMAT
);
}
if manifest.checksum != manifest.compute_checksum() {
anyhow::bail!("{} failed its checksum", path.display());
}
Ok(Some(manifest))
}
pub fn write_manifest(cache_dir: &Path, manifest: &Manifest) -> Result<()> {
use std::io::Write;
let path = cache_dir.join(MANIFEST);
let tmp = crate::atomic_write::tmp_path_for(&path);
let body = serde_json::to_vec_pretty(manifest).context("Failed to serialize the manifest")?;
{
let mut f = std::fs::File::create(&tmp)
.with_context(|| format!("Failed to create {}", tmp.display()))?;
f.write_all(&body)?;
f.sync_all()?;
}
crate::atomic_write::atomic_replace(&tmp, &path)
.with_context(|| format!("Failed to move {} into place", path.display()))?;
sync_dir(cache_dir);
Ok(())
}
pub fn sync_dir(dir: &Path) {
#[cfg(unix)]
if let Ok(d) = std::fs::File::open(dir) {
let _ = d.sync_all();
}
#[cfg(not(unix))]
let _ = dir;
}
pub fn link_fixed_names(cache_dir: &Path, manifest: &Manifest) {
if !manifest.base_only() {
unlink_fixed_names(cache_dir);
return;
}
for (generation_file, fixed) in [
(&manifest.base.content, FIXED_CONTENT),
(&manifest.base.trigrams, FIXED_TRIGRAMS),
] {
let target = cache_dir.join(fixed);
let tmp = crate::atomic_write::tmp_path_for(&target);
let _ = std::fs::remove_file(&tmp);
let linked = std::fs::hard_link(cache_dir.join(generation_file), &tmp)
.and_then(|_| crate::atomic_write::atomic_replace(&tmp, &target));
if let Err(e) = linked {
log::debug!(
"Cannot link {} to {}: {}; removing it",
fixed,
generation_file,
e
);
let _ = std::fs::remove_file(&tmp);
let _ = std::fs::remove_file(&target);
}
}
}
pub fn unlink_fixed_names(cache_dir: &Path) {
let mut removed = false;
for fixed in [FIXED_CONTENT, FIXED_TRIGRAMS] {
match std::fs::remove_file(cache_dir.join(fixed)) {
Ok(()) => removed = true,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => log::warn!("Could not remove {}: {}", fixed, e),
}
}
if removed {
sync_dir(cache_dir);
}
}
pub fn remove_unreferenced(cache_dir: &Path, current: &Manifest, previous: Option<&Manifest>) {
let mut keep: std::collections::HashSet<&str> = current.files().into_iter().collect();
if let Some(prev) = previous {
keep.extend(prev.files());
}
let Ok(entries) = std::fs::read_dir(cache_dir) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name();
let Some(name) = name.to_str() else { continue };
if is_generation_file(name)
&& !keep.contains(name)
&& let Err(e) = std::fs::remove_file(entry.path())
{
log::debug!("Could not remove {}: {}", name, e);
}
}
}
const TOMB_MAGIC: &[u8; 4] = b"RFTB";
const TOMB_VERSION: u32 = 1;
const TOMB_HEADER: usize = 16;
pub struct TombPlan {
mmap: Option<memmap2::Mmap>,
count: usize,
}
impl TombPlan {
fn load(path: &Path) -> Result<Self> {
let file = std::fs::File::open(path)
.with_context(|| format!("Failed to open {}", path.display()))?;
let len = file.metadata()?.len() as usize;
if len == TOMB_HEADER {
return Ok(Self {
mmap: None,
count: 0,
});
}
let mmap = unsafe {
memmap2::Mmap::map(&file)
.with_context(|| format!("Failed to mmap {}", path.display()))?
};
if mmap.len() < TOMB_HEADER || &mmap[0..4] != TOMB_MAGIC {
anyhow::bail!("{} is not a tombstone planning file", path.display());
}
if u32::from_le_bytes(mmap[4..8].try_into().unwrap()) != TOMB_VERSION {
anyhow::bail!("{} has an unsupported version", path.display());
}
let count = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
if mmap.len() != TOMB_HEADER + count * 8 {
anyhow::bail!("{} is truncated", path.display());
}
Ok(Self {
mmap: Some(mmap),
count,
})
}
#[inline]
fn entry(&self, i: usize) -> (Trigram, u32) {
let mmap = self.mmap.as_ref().expect("non-empty");
let off = TOMB_HEADER + i * 8;
(
u32::from_le_bytes(mmap[off..off + 4].try_into().unwrap()),
u32::from_le_bytes(mmap[off + 4..off + 8].try_into().unwrap()),
)
}
pub fn get(&self, trigram: Trigram) -> u32 {
let (mut lo, mut hi) = (0usize, self.count);
while lo < hi {
let mid = lo + (hi - lo) / 2;
let (t, size) = self.entry(mid);
match t.cmp(&trigram) {
std::cmp::Ordering::Less => lo = mid + 1,
std::cmp::Ordering::Greater => hi = mid,
std::cmp::Ordering::Equal => return size,
}
}
0
}
pub fn entries(&self) -> impl Iterator<Item = (Trigram, u32)> + '_ {
(0..self.count).map(|i| self.entry(i))
}
}
pub fn write_tomb_file(path: &Path, entries: &[(Trigram, u32)]) -> Result<()> {
use std::io::Write;
let tmp = crate::atomic_write::tmp_path_for(path);
{
let mut w = std::io::BufWriter::new(
std::fs::File::create(&tmp)
.with_context(|| format!("Failed to create {}", tmp.display()))?,
);
w.write_all(TOMB_MAGIC)?;
w.write_all(&TOMB_VERSION.to_le_bytes())?;
w.write_all(&(entries.len() as u64).to_le_bytes())?;
for (t, size) in entries {
w.write_all(&t.to_le_bytes())?;
w.write_all(&size.to_le_bytes())?;
}
w.flush()?;
w.get_ref().sync_all()?;
}
crate::atomic_write::atomic_replace(&tmp, path)
.with_context(|| format!("Failed to move {} into place", path.display()))
}
fn read_header(cache_dir: &Path, logical: &str, name: &str) -> Result<[u8; 32]> {
use std::io::Read;
let mut header = [0u8; 32];
std::fs::File::open(cache_dir.join(name))
.and_then(|mut f| f.read_exact(&mut header))
.map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
missing(logical, name)
} else {
ReflexError::CacheCorrupted(format!("{logical}: {name}: {e}")).into()
}
})?;
Ok(header)
}
pub fn check_published(cache_dir: &Path) -> Result<Option<Manifest>> {
let Some(m) = read_manifest(cache_dir)? else {
return Ok(None);
};
let u32_at = |h: &[u8; 32], at: usize| u32::from_le_bytes(h[at..at + 4].try_into().unwrap());
let u64_at = |h: &[u8; 32], at: usize| u64::from_le_bytes(h[at..at + 8].try_into().unwrap());
for segment in [Some(&m.base), m.delta.as_ref(), m.recent.as_ref()]
.into_iter()
.flatten()
{
check_size(
cache_dir,
"content.bin",
&segment.content,
segment.content_bytes,
)?;
check_size(
cache_dir,
"trigrams.bin",
&segment.trigrams,
segment.trigrams_bytes,
)?;
let content = read_header(cache_dir, "content.bin", &segment.content)?;
if &content[0..4] != b"RFCT"
|| u32_at(&content, 4) != 2
|| u64_at(&content, 8) != segment.files
{
return Err(ReflexError::CacheCorrupted(format!(
"content.bin: {} does not hold the manifest's {} files",
segment.content, segment.files
))
.into());
}
let trigrams = read_header(cache_dir, "trigrams.bin", &segment.trigrams)?;
if &trigrams[0..4] != crate::trigram::MAGIC
|| u32_at(&trigrams, 4) != crate::trigram::VERSION
|| u64_at(&trigrams, 16) != segment.files
{
return Err(ReflexError::CacheCorrupted(format!(
"trigrams.bin: {} does not hold the manifest's {} files",
segment.trigrams, segment.files
))
.into());
}
if let Some(plan) = &segment.plan
&& !cache_dir.join(plan).exists()
{
return Err(missing("trigrams.bin", plan));
}
}
for tomb in [&m.tomb, &m.tomb_delta].into_iter().flatten() {
if !cache_dir.join(tomb).exists() {
return Err(missing("trigrams.bin", tomb));
}
}
let bound = m.base.files + m.delta.as_ref().map_or(0, |d| d.files);
if m.tombstones.iter().any(|&id| id as u64 >= bound) {
return Err(ReflexError::CacheCorrupted(
"content.bin: a tombstone is past the base and delta's files".to_string(),
)
.into());
}
Ok(Some(m))
}
pub struct Segment {
pub content: ContentReader,
pub trigrams: TrigramIndex,
}
pub struct IndexSnapshot {
base: Segment,
delta: Option<Segment>,
recent: Option<Segment>,
dead: Vec<u64>,
dead_count: usize,
tomb: Option<TombPlan>,
tomb_delta: Option<TombPlan>,
base_len: u32,
delta_len: u32,
manifest: Option<Manifest>,
}
impl std::fmt::Debug for IndexSnapshot {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("IndexSnapshot")
.field("files", &self.live_file_count())
.field("generation", &self.generation())
.finish()
}
}
fn is_not_found(err: &anyhow::Error) -> bool {
err.chain().any(|c| {
c.downcast_ref::<std::io::Error>()
.is_some_and(|e| e.kind() == std::io::ErrorKind::NotFound)
})
}
fn missing(logical: &str, name: &str) -> anyhow::Error {
anyhow::Error::new(std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("{} is missing", name),
))
.context(ReflexError::CacheCorrupted(format!(
"{}: {} is missing",
logical, name
)))
}
fn check_size(cache_dir: &Path, logical: &str, name: &str, expected: u64) -> Result<()> {
match std::fs::metadata(cache_dir.join(name)) {
Ok(md) if md.len() == expected => Ok(()),
Ok(md) => Err(ReflexError::CacheCorrupted(format!(
"{}: {} is {} bytes, the manifest says {}",
logical,
name,
md.len(),
expected
))
.into()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(missing(logical, name)),
Err(e) => Err(e).with_context(|| format!("Failed to stat {}", name)),
}
}
fn open_segment(cache_dir: &Path, files: &SegmentFiles) -> Result<Segment> {
check_size(
cache_dir,
"content.bin",
&files.content,
files.content_bytes,
)?;
check_size(
cache_dir,
"trigrams.bin",
&files.trigrams,
files.trigrams_bytes,
)?;
let content = ContentReader::open(cache_dir.join(&files.content))
.map_err(|e| ReflexError::CacheCorrupted(format!("content.bin: {e:#}")))?;
let mut trigrams = TrigramIndex::load(cache_dir.join(&files.trigrams))
.map_err(|e| ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}")))?;
if let Some(plan) = &files.plan {
let plan_path = cache_dir.join(plan);
if !plan_path.exists() {
return Err(missing("trigrams.bin", plan));
}
trigrams
.attach_plan(&plan_path)
.map_err(|e| ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}")))?;
}
check_counts(&content, &trigrams)?;
if content.file_count() as u64 != files.files {
return Err(ReflexError::CacheCorrupted(format!(
"content.bin holds {} files but the manifest lists {}",
content.file_count(),
files.files
))
.into());
}
Ok(Segment { content, trigrams })
}
fn check_counts(content: &ContentReader, trigrams: &TrigramIndex) -> Result<()> {
if trigrams.file_count() != content.file_count() {
return Err(ReflexError::CacheCorrupted(format!(
"trigrams.bin lists {} files but content.bin holds {} (index written by two runs?)",
trigrams.file_count(),
content.file_count()
))
.into());
}
Ok(())
}
impl IndexSnapshot {
pub fn open(cache_dir: &Path) -> Result<Self> {
Self::open_reading(cache_dir, || read_manifest(cache_dir))
}
fn open_reading(
cache_dir: &Path,
mut read: impl FnMut() -> Result<Option<Manifest>>,
) -> Result<Self> {
let mut last: Option<anyhow::Error> = None;
for attempt in 0..5u64 {
let manifest =
read().map_err(|e| ReflexError::CacheCorrupted(format!("content.bin: {e:#}")))?;
let result = match manifest {
None => return Self::open_legacy(cache_dir),
Some(m) => Self::open_manifest(cache_dir, m),
};
match result {
Ok(s) => return Ok(s),
Err(e) if is_not_found(&e) => {
last = Some(e);
std::thread::sleep(std::time::Duration::from_millis(10 * (attempt + 1)));
}
Err(e) => return Err(e),
}
}
Err(last.expect("at least one attempt"))
}
fn open_manifest(cache_dir: &Path, manifest: Manifest) -> Result<Self> {
let base = open_segment(cache_dir, &manifest.base)?;
let base_len = base.content.file_count() as u32;
let delta = match &manifest.delta {
Some(d) => Some(open_segment(cache_dir, d)?),
None => None,
};
let delta_len = delta.as_ref().map_or(0, |d| d.content.file_count() as u32);
let recent = match &manifest.recent {
Some(d) => Some(open_segment(cache_dir, d)?),
None => None,
};
let load_tomb = |name: &Option<String>| -> Result<Option<TombPlan>> {
let Some(name) = name else { return Ok(None) };
let tomb_path = cache_dir.join(name);
if !tomb_path.exists() {
return Err(missing("trigrams.bin", name));
}
Ok(Some(TombPlan::load(&tomb_path).map_err(|e| {
ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}"))
})?))
};
let tomb = load_tomb(&manifest.tomb)?;
let tomb_delta = load_tomb(&manifest.tomb_delta)?;
let dead_bound = base_len + delta_len;
let mut dead = vec![0u64; (dead_bound as usize).div_ceil(64)];
for &id in &manifest.tombstones {
if id >= dead_bound {
return Err(ReflexError::CacheCorrupted(format!(
"content.bin: tombstone {} is past the base and delta's {} files",
id, dead_bound
))
.into());
}
dead[(id / 64) as usize] |= 1u64 << (id % 64);
}
Ok(Self {
base,
delta,
recent,
dead,
dead_count: manifest.tombstones.len(),
tomb,
tomb_delta,
base_len,
delta_len,
manifest: Some(manifest),
})
}
fn open_legacy(cache_dir: &Path) -> Result<Self> {
let content = ContentReader::open(cache_dir.join(FIXED_CONTENT))
.map_err(|e| ReflexError::CacheCorrupted(format!("content.bin: {e:#}")))?;
let trigrams_path = cache_dir.join(FIXED_TRIGRAMS);
let trigrams = if trigrams_path.exists() {
match TrigramIndex::load(&trigrams_path) {
Ok(index) => index,
Err(e) if e.to_string().contains("Unsupported trigrams.bin version") => {
log::warn!("{}; rebuilding trigram index in memory for this process", e);
crate::query::result::rebuild_trigram_index(&content)?
}
Err(e) => {
return Err(ReflexError::CacheCorrupted(format!("trigrams.bin: {e:#}")).into());
}
}
} else {
log::debug!("trigrams.bin not found, rebuilding from content store");
crate::query::result::rebuild_trigram_index(&content)?
};
check_counts(&content, &trigrams)?;
let base_len = content.file_count() as u32;
Ok(Self {
base: Segment { content, trigrams },
delta: None,
recent: None,
dead: Vec::new(),
dead_count: 0,
tomb: None,
tomb_delta: None,
base_len,
delta_len: 0,
manifest: None,
})
}
pub fn generation(&self) -> Option<u64> {
self.manifest.as_ref().map(|m| m.generation)
}
pub fn manifest(&self) -> Option<&Manifest> {
self.manifest.as_ref()
}
pub fn base(&self) -> &Segment {
&self.base
}
pub fn delta(&self) -> Option<&Segment> {
self.delta.as_ref()
}
pub fn recent(&self) -> Option<&Segment> {
self.recent.as_ref()
}
pub fn base_len(&self) -> u32 {
self.base_len
}
pub fn delta_len(&self) -> u32 {
self.delta_len
}
fn recent_len(&self) -> u32 {
self.recent
.as_ref()
.map_or(0, |d| d.content.file_count() as u32)
}
pub fn tomb(&self) -> Option<&TombPlan> {
self.tomb.as_ref()
}
pub fn tomb_delta(&self) -> Option<&TombPlan> {
self.tomb_delta.as_ref()
}
#[inline]
pub fn is_dead(&self, id: u32) -> bool {
id < self.base_len + self.delta_len
&& self
.dead
.get((id / 64) as usize)
.is_some_and(|w| w & (1u64 << (id % 64)) != 0)
}
pub fn id_bound(&self) -> u32 {
self.base_len + self.delta_len + self.recent_len()
}
pub fn live_file_count(&self) -> usize {
self.id_bound() as usize - self.dead_count
}
pub fn is_live(&self, id: u32) -> bool {
id < self.id_bound() && !self.is_dead(id)
}
pub fn live_ids(&self) -> impl Iterator<Item = u32> + '_ {
(0..self.id_bound()).filter(|&id| !self.is_dead(id))
}
#[inline]
fn route(&self, id: u32) -> (&Segment, u32) {
if id < self.base_len {
return (&self.base, id);
}
match (&self.delta, &self.recent) {
(Some(delta), _) if id < self.base_len + self.delta_len => (delta, id - self.base_len),
(_, Some(recent)) => (recent, id - self.base_len - self.delta_len),
_ => (&self.base, id),
}
}
pub fn get_file_content(&self, file_id: u32) -> Result<&str> {
let (segment, local) = self.route(file_id);
segment.content.get_file_content(local)
}
pub fn release_pages(&self) {
for segment in [Some(&self.base), self.delta.as_ref(), self.recent.as_ref()]
.into_iter()
.flatten()
{
segment.content.release_pages();
}
}
pub fn file_len(&self, file_id: u32) -> Option<u64> {
let (segment, local) = self.route(file_id);
segment.content.file_len(local)
}
pub fn get_file_path(&self, file_id: u32) -> Option<&Path> {
if !self.is_live(file_id) {
return None;
}
let (segment, local) = self.route(file_id);
segment.content.get_file_path(local)
}
pub fn get_context_by_line(
&self,
file_id: u32,
line_number: usize,
context_lines: usize,
) -> Result<(Vec<String>, Vec<String>)> {
let (segment, local) = self.route(file_id);
segment
.content
.get_context_by_line(local, line_number, context_lines)
}
pub fn get_file_id_by_path(&self, path: &str) -> Option<u32> {
let normalized = path.strip_prefix("./").unwrap_or(path);
self.live_ids().find(|&id| {
self.get_file_path(id)
.and_then(|p| p.to_str())
.is_some_and(|p| p.strip_prefix("./").unwrap_or(p) == normalized)
})
}
fn base_alone(&self) -> bool {
self.delta.is_none() && self.recent.is_none() && self.dead_count == 0
}
pub fn live_plan(&self, trigram: Trigram) -> u64 {
self.list(trigram).map_or(0, |l| l.plan)
}
fn list(&self, trigram: Trigram) -> Option<TrigramList<'_>> {
let mut parts: Vec<ListPart<'_>> = Vec::with_capacity(3);
let mut plan: i64 = 0;
let dead: Option<&[u64]> = (self.dead_count > 0).then_some(self.dead.as_slice());
if let Some((mut part, base_plan)) = self.base.trigrams.list_part(trigram) {
part.dead = dead;
parts.push(part);
plan += base_plan as i64;
}
for tomb in [&self.tomb, &self.tomb_delta].into_iter().flatten() {
plan -= tomb.get(trigram) as i64;
}
if let Some(delta) = &self.delta
&& let Some((mut part, delta_plan)) = delta.trigrams.list_part(trigram)
{
part.id_offset = self.base_len;
part.dead = dead;
parts.push(part);
plan += delta_plan as i64;
}
if let Some(recent) = &self.recent
&& let Some((mut part, recent_plan)) = recent.trigrams.list_part(trigram)
{
part.id_offset = self.base_len + self.delta_len;
parts.push(part);
plan += recent_plan as i64;
}
if parts.is_empty() || plan <= 0 {
return None;
}
Some(TrigramList {
trigram,
plan: plan as u64,
parts,
})
}
pub fn search_candidates(&self, pattern: &str) -> Vec<FileLocation> {
if self.base_alone() {
return self.base.trigrams.search_candidates(pattern);
}
let lists = pattern_trigrams(pattern)
.into_iter()
.map(|t| self.list(t))
.collect::<Vec<_>>();
if lists.is_empty() {
return vec![];
}
plan_intersection(lists, true)
}
pub fn search_candidates_fold(&self, literal: &[u8]) -> Vec<FileLocation> {
if self.base_alone() {
return self.base.trigrams.search_candidates_fold(literal);
}
let Some(windows) = fold_windows(literal) else {
return vec![];
};
let lists = windows
.iter()
.map(|variants| variants.iter().filter_map(|&t| self.list(t)).collect())
.collect();
plan_fold_intersection(lists)
}
pub fn exotic_fold_lines(&self) -> Vec<FileLocation> {
if self.base_alone() {
return self.base.trigrams.exotic_fold_lines();
}
let (lo, hi) = EXOTIC_LONG_S_RANGE;
let mut trigrams: Vec<Trigram> = self
.base
.trigrams
.list_parts_in(lo, hi)
.into_iter()
.map(|(t, _, _)| t)
.collect();
for segment in [&self.delta, &self.recent].into_iter().flatten() {
trigrams.extend(
segment
.trigrams
.list_parts_in(lo, hi)
.into_iter()
.map(|(t, _, _)| t),
);
}
trigrams.extend(exotic_singles());
trigrams.sort_unstable();
trigrams.dedup();
let lists: Vec<TrigramList> = trigrams.into_iter().filter_map(|t| self.list(t)).collect();
union_lists(&lists)
}
pub fn trigram_count(&self) -> usize {
match &self.manifest {
Some(m) => m.live_trigrams as usize,
None => self.base.trigrams.trigram_count(),
}
}
}
#[derive(Debug, Clone)]
pub struct StoreSummary {
pub content_files: Vec<PathBuf>,
pub trigram_files: Vec<PathBuf>,
pub other_files: Vec<PathBuf>,
pub live_corpus_bytes: Option<u64>,
pub live_trigrams: Option<u64>,
}
impl StoreSummary {
pub fn files(&self) -> impl Iterator<Item = &PathBuf> {
self.content_files
.iter()
.chain(&self.trigram_files)
.chain(&self.other_files)
}
}
pub fn store_summary(cache_dir: &Path) -> StoreSummary {
match read_manifest(cache_dir) {
Ok(Some(m)) => {
let mut content_files = vec![cache_dir.join(&m.base.content)];
let mut trigram_files = vec![cache_dir.join(&m.base.trigrams)];
let mut other_files: Vec<PathBuf> =
m.base.plan.iter().map(|p| cache_dir.join(p)).collect();
for d in [&m.delta, &m.recent].into_iter().flatten() {
content_files.push(cache_dir.join(&d.content));
trigram_files.push(cache_dir.join(&d.trigrams));
other_files.extend(d.plan.iter().map(|p| cache_dir.join(p)));
}
other_files.extend(m.tomb.iter().map(|t| cache_dir.join(t)));
other_files.extend(m.tomb_delta.iter().map(|t| cache_dir.join(t)));
StoreSummary {
content_files,
trigram_files,
other_files,
live_corpus_bytes: Some(m.live_corpus_bytes),
live_trigrams: Some(m.live_trigrams),
}
}
_ => StoreSummary {
content_files: vec![cache_dir.join(FIXED_CONTENT)],
trigram_files: vec![cache_dir.join(FIXED_TRIGRAMS)],
other_files: Vec::new(),
live_corpus_bytes: None,
live_trigrams: None,
},
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn generation_file_names() {
assert!(is_generation_file("content.12.bin"));
assert!(is_generation_file("trigrams.3.bin"));
assert!(is_generation_file("trigrams.3.plan"));
assert!(is_generation_file("delta.4.content.bin"));
assert!(is_generation_file("delta.4.trigrams.bin"));
assert!(is_generation_file("delta.4.plan"));
assert!(is_generation_file("delta.4.tomb"));
assert!(is_generation_file("delta.4.dtomb"));
assert!(is_generation_file("recent.4.content.bin"));
assert!(is_generation_file("recent.4.trigrams.bin"));
assert!(is_generation_file("recent.4.plan"));
assert!(!is_generation_file("recent.4.tomb"));
assert!(!is_generation_file("content.3.plan"));
assert!(!is_generation_file("content.bin"));
assert!(!is_generation_file("trigrams.bin"));
assert!(!is_generation_file("content.x.bin"));
assert!(!is_generation_file("meta.db"));
assert!(!is_generation_file("manifest.json"));
}
#[test]
fn manifest_round_trip_and_checksum() {
let temp = tempfile::TempDir::new().unwrap();
let (content, trigrams, plan) = base_file_names(7);
let m = Manifest::for_base(
7,
SegmentFiles {
content,
trigrams,
plan: Some(plan),
files: 3,
content_bytes: 100,
trigrams_bytes: 200,
},
42,
90,
);
write_manifest(temp.path(), &m).unwrap();
assert_eq!(read_manifest(temp.path()).unwrap(), Some(m));
let path = temp.path().join(MANIFEST);
let text = std::fs::read_to_string(&path)
.unwrap()
.replace("\"generation\": 7", "\"generation\": 8");
std::fs::write(&path, text).unwrap();
assert!(read_manifest(temp.path()).is_err());
}
#[test]
fn tomb_file_round_trip() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("t.tomb");
write_tomb_file(&path, &[(3, 10), (70_000, 4), (0xE2_84_AA, 7)]).unwrap();
let tomb = TombPlan::load(&path).unwrap();
assert_eq!(tomb.get(3), 10);
assert_eq!(tomb.get(70_000), 4);
assert_eq!(tomb.get(0xE2_84_AA), 7);
assert_eq!(tomb.get(4), 0);
write_tomb_file(&path, &[]).unwrap();
assert_eq!(TombPlan::load(&path).unwrap().get(3), 0);
}
#[test]
fn open_retries_while_a_publish_replaces_the_manifest() {
let temp = tempfile::TempDir::new().unwrap();
let root = temp.path();
write(root, "a.rs", "fn alpha() {}\n");
index_unlimited(root);
let cache = root.join(".reflex");
let good = read_manifest(&cache).unwrap().unwrap();
let mut files = good.base.clone();
files.content = "content.999.bin".to_string();
let broken = Manifest::for_base(
good.generation + 1,
files,
good.live_trigrams,
good.live_corpus_bytes,
);
let mut reads = 0;
let snapshot = IndexSnapshot::open_reading(&cache, || {
reads += 1;
Ok(Some(if reads == 1 {
broken.clone()
} else {
good.clone()
}))
})
.expect("a retry opens the next manifest");
assert_eq!(reads, 2);
assert_eq!(snapshot.generation(), Some(good.generation));
write_manifest(&cache, &broken).unwrap();
let err = IndexSnapshot::open(&cache).unwrap_err();
assert!(
matches!(
err.downcast_ref::<ReflexError>(),
Some(ReflexError::CacheCorrupted(_))
),
"{err:#}"
);
}
fn index_unlimited(root: &Path) {
index_with(root, None);
}
fn write(root: &Path, rel: &str, body: &str) {
let p = root.join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, body).unwrap();
}
fn candidates(snapshot: &IndexSnapshot, pattern: &str) -> Vec<(PathBuf, u32)> {
let mut out: Vec<(PathBuf, u32)> = snapshot
.search_candidates(pattern)
.into_iter()
.map(|l| {
let path = snapshot.get_file_path(l.file_id).expect("a live candidate");
(path.to_path_buf(), l.line_no)
})
.collect();
out.sort();
out
}
fn copy_tree(from: &Path, to: &Path) {
for entry in std::fs::read_dir(from).unwrap() {
let entry = entry.unwrap();
let name = entry.file_name();
if name == ".reflex" {
continue;
}
let (src, dst) = (entry.path(), to.join(&name));
if entry.file_type().unwrap().is_dir() {
std::fs::create_dir_all(&dst).unwrap();
copy_tree(&src, &dst);
} else {
std::fs::copy(&src, &dst).unwrap();
}
}
}
fn index_with(root: &Path, recent: Option<(usize, u64)>) {
let mut indexer = crate::Indexer::new(
crate::CacheManager::new(root),
crate::models::IndexConfig::default(),
);
indexer.set_merge_limits(usize::MAX, u64::MAX);
if let Some((files, bytes)) = recent {
indexer.set_recent_limits(files, bytes);
}
indexer.index(root, false).unwrap();
}
fn assert_plans_as_fresh(root: &Path) {
let cache = root.join(".reflex");
let updated = IndexSnapshot::open(&cache).unwrap();
assert!(!updated.base_alone());
let copy = tempfile::TempDir::new().unwrap();
copy_tree(root, copy.path());
index_unlimited(copy.path());
let fresh = IndexSnapshot::open(©.path().join(".reflex")).unwrap();
assert!(fresh.base_alone());
let mut all: Vec<Trigram> = updated.base().trigrams.trigrams().collect();
for segment in [updated.delta(), updated.recent()].into_iter().flatten() {
all.extend(segment.trigrams.trigrams());
}
all.extend(fresh.base().trigrams.trigrams());
all.sort_unstable();
all.dedup();
let mut live = 0;
for t in all {
let u = updated.list(t).map(|l| l.plan);
let f = fresh.list(t).map(|l| l.plan);
assert_eq!(u, f, "planning size of trigram {t:#08x}");
live += u.is_some() as usize;
}
assert_eq!(live, fresh.trigram_count());
assert_eq!(updated.trigram_count(), fresh.trigram_count());
assert_eq!(updated.live_file_count(), fresh.live_file_count());
assert_eq!(
updated.manifest().unwrap().live_corpus_bytes,
fresh.manifest().unwrap().live_corpus_bytes
);
for pattern in [
"shared",
"common_1",
"rare_marker_qx",
"edited",
"brand_new_token",
"vanishing_zqj",
"line 1",
] {
assert_eq!(
candidates(&updated, pattern),
candidates(&fresh, pattern),
"{pattern}"
);
}
let fold_paths = |s: &IndexSnapshot| {
let mut v: Vec<(PathBuf, u32)> = s
.search_candidates_fold(b"RARE_MARKER")
.into_iter()
.map(|l| (s.get_file_path(l.file_id).unwrap().to_path_buf(), l.line_no))
.collect();
v.sort();
v
};
assert_eq!(fold_paths(&updated), fold_paths(&fresh));
}
fn seed_tree(root: &Path) {
for i in 0..200 {
let rare = if i % 50 == 0 { "rare_marker_qx\n" } else { "" };
write(
root,
&format!("src/f{i:03}.rs"),
&format!("pub fn common_{i}() {{ shared(); }}\n{rare}// line {i}\n"),
);
}
write(root, "only/gone.rs", "fn vanishing_zqj() {}\n");
}
#[test]
fn a_delta_snapshot_plans_and_answers_as_a_fresh_build() {
let temp = tempfile::TempDir::new().unwrap();
let root = temp.path();
seed_tree(root);
index_unlimited(root);
write(root, "src/f000.rs", "pub fn common_0() { edited(); }\n");
write(
root,
"src/f150.rs",
"pub fn common_150() {}\nrare_marker_qx\n",
);
write(
root,
"src/new_file.rs",
"fn brand_new_token() { shared(); }\n",
);
std::fs::remove_file(root.join("src/f100.rs")).unwrap();
std::fs::remove_file(root.join("only/gone.rs")).unwrap();
index_unlimited(root);
let updated = IndexSnapshot::open(&root.join(".reflex")).unwrap();
assert!(candidates(&updated, "vanishing_zqj").is_empty());
drop(updated);
assert_plans_as_fresh(root);
}
#[test]
fn a_two_tier_snapshot_plans_and_answers_as_a_fresh_build() {
let temp = tempfile::TempDir::new().unwrap();
let root = temp.path();
seed_tree(root);
index_unlimited(root);
let recent = Some((2, u64::MAX));
let manifest = |root: &Path| read_manifest(&root.join(".reflex")).unwrap().unwrap();
write(root, "src/f000.rs", "pub fn common_0() { edited(); }\n");
write(root, "src/f001.rs", "pub fn common_1() { edited(); }\n");
index_with(root, recent);
let m = manifest(root);
assert!(m.delta.is_none() && m.recent.as_ref().map(|r| r.files) == Some(2));
assert_plans_as_fresh(root);
write(root, "src/f002.rs", "pub fn common_2() { edited(); }\n");
index_with(root, recent);
let m = manifest(root);
assert!(
m.recent.is_none() && m.delta.as_ref().map(|d| d.files) == Some(3),
"{m:?}"
);
assert_plans_as_fresh(root);
write(
root,
"src/f001.rs",
"pub fn common_1() {}\nrare_marker_qx\n",
);
std::fs::remove_file(root.join("only/gone.rs")).unwrap();
index_with(root, recent);
let m = manifest(root);
assert!(
m.delta.is_some() && m.recent.is_some() && m.tomb_delta.is_some(),
"{m:?}"
);
assert_plans_as_fresh(root);
write(root, "src/f150.rs", "pub fn common_150() {}\n");
write(
root,
"src/new_file.rs",
"fn brand_new_token() { shared(); }\n",
);
index_with(root, recent);
let m = manifest(root);
assert!(m.recent.is_none() && m.tomb_delta.is_none(), "{m:?}");
assert_eq!(m.delta.as_ref().map(|d| d.files), Some(5));
assert_plans_as_fresh(root);
}
}