use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, RwLock};
use crate::error::Result;
use crate::fs::Fs;
use crate::index::{DocId, DocMeta, IndexReader, IndexStats, SegmentReader};
use super::reader::{open, OpenOptions};
use super::segment::nanos_to_time;
use super::{index_dir, Segment};
pub struct LiveSegment {
inner: Segment,
dir: PathBuf,
fs: Arc<dyn Fs>,
retire: AtomicBool,
}
impl std::ops::Deref for LiveSegment {
type Target = Segment;
fn deref(&self) -> &Segment {
&self.inner
}
}
impl SegmentReader for LiveSegment {
type List<'a>
= <Segment as SegmentReader>::List<'a>
where
Self: 'a;
fn base(&self) -> DocId {
self.inner.base()
}
fn num_docs(&self) -> u32 {
self.inner.num_docs()
}
fn postings(&self, term: &str) -> Option<Self::List<'_>> {
self.inner.postings(term)
}
fn doc_len(&self, local: DocId) -> Option<u32> {
self.inner.doc_len(local)
}
}
impl Drop for LiveSegment {
fn drop(&mut self) {
if self.retire.load(Ordering::Acquire) {
for p in self.inner.files(&self.dir) {
if let Err(e) = self.fs.remove_file(&p) {
log::debug!("retire {}: {e} (will be cleaned up on next open)", p.display());
}
}
}
}
}
type OverlayPatch = (DocId, u64, i64, u64, String);
struct Snapshot0 {
generation: u64,
segments: Vec<Arc<LiveSegment>>,
overlay: Vec<OverlayPatch>,
n_indexed: u32,
avg_len: f32,
stats: IndexStats,
root: PathBuf,
}
pub struct LiveIndex {
fs: Arc<dyn Fs>,
root: PathBuf,
dir: PathBuf,
current: RwLock<Arc<Snapshot0>>,
}
fn wrap(index: super::Index, fs: &Arc<dyn Fs>, dir: &Path) -> Snapshot0 {
let stats = index.stats().clone();
let n_indexed = index.indexed_count();
let avg_len = index.avg_len();
let (manifest, segments, overlay, root) = index.into_parts();
let segments = segments
.into_iter()
.map(|inner| Arc::new(LiveSegment { inner, dir: dir.to_path_buf(), fs: fs.clone(), retire: AtomicBool::new(false) }))
.collect();
Snapshot0 { generation: manifest.generation, segments, overlay, n_indexed, avg_len, stats, root }
}
impl LiveIndex {
pub fn open(fs: Arc<dyn Fs>, root: &Path) -> Result<Option<LiveIndex>> {
let Some(index) = open(&*fs, root, &OpenOptions::default())? else { return Ok(None) };
let dir = index_dir(root);
let snapshot0 = wrap(index, &fs, &dir);
Ok(Some(LiveIndex { fs, root: root.to_path_buf(), dir, current: RwLock::new(Arc::new(snapshot0)) }))
}
pub fn root(&self) -> &Path {
&self.root
}
pub fn generation(&self) -> u64 {
self.current.read().unwrap().generation
}
pub fn snapshot(&self) -> IndexSnapshot {
IndexSnapshot(self.current.read().unwrap().clone())
}
pub fn refresh(&self) -> Result<bool> {
let Some(fresh) = open(&*self.fs, &self.root, &OpenOptions::default())? else {
return Ok(false); };
let old = self.current.read().unwrap().clone();
if fresh.manifest().generation == old.generation {
return Ok(false);
}
let new_ids: HashSet<u32> = fresh.manifest().segments.iter().map(|s| s.segment_id).collect();
for seg in &old.segments {
if !new_ids.contains(&seg.inner.entry().segment_id) {
seg.retire.store(true, Ordering::Release);
}
}
let snapshot0 = wrap(fresh, &self.fs, &self.dir);
*self.current.write().unwrap() = Arc::new(snapshot0);
Ok(true)
}
}
#[derive(Clone)]
pub struct IndexSnapshot(Arc<Snapshot0>);
impl IndexSnapshot {
pub fn generation(&self) -> u64 {
self.0.generation
}
}
impl IndexReader for IndexSnapshot {
type Segment = LiveSegment;
fn segments(&self) -> impl Iterator<Item = &LiveSegment> + '_ {
self.0.segments.iter().map(|a| a.as_ref())
}
fn indexed_count(&self) -> u32 {
self.0.n_indexed
}
fn avg_len(&self) -> f32 {
self.0.avg_len
}
fn doc(&self, id: DocId) -> Option<DocMeta> {
let i = self.0.segments.partition_point(|s| s.inner.entry().end_doc() <= id as u64);
let seg = self.0.segments.get(i)?;
if id < seg.inner.base() {
return None;
}
let mut meta = seg.inner.doc_meta(id - seg.inner.base(), &self.0.root)?;
if let Ok(pos) = self.0.overlay.binary_search_by_key(&id, |p| p.0) {
let (_, inode, mtime_nanos, size, path) = &self.0.overlay[pos];
meta.inode = *inode;
meta.mtime = nanos_to_time(*mtime_nanos);
meta.size = *size;
meta.path = self.0.root.clone();
meta.path.extend(path.split('/').filter(|c| !c.is_empty()));
}
Some(meta)
}
fn stats(&self) -> &IndexStats {
&self.0.stats
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::crawler::CrawlConfig;
use crate::fs::RealFs;
use crate::index::builder::BuildConfig;
use crate::query::{search, Query, SearchOptions};
use crate::store::{merge_index, update_index, MergePolicy, NoProgress};
use std::fs;
use std::sync::Barrier;
use std::thread;
fn cfg() -> CrawlConfig {
CrawlConfig { respect_global_gitignore: false, ..CrawlConfig::default() }
}
fn small_build() -> BuildConfig {
BuildConfig { shard_size: 1, docs_per_segment: 2, ..BuildConfig::default() }
}
fn hits(snap: &IndexSnapshot, q: &str) -> usize {
search(snap, &Query::parse(q).unwrap(), &SearchOptions { limit: 10_000, ..Default::default() }).total_matches
}
#[test]
fn refresh_picks_up_new_generations_and_old_snapshot_keeps_working() {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("a.txt"), "alpha beta").unwrap();
update_index(&RealFs, dir.path(), &cfg(), &small_build(), &NoProgress).unwrap();
let live = LiveIndex::open(Arc::new(RealFs), dir.path()).unwrap().unwrap();
let before = live.snapshot();
assert_eq!(hits(&before, "alpha"), 1);
fs::write(dir.path().join("b.txt"), "gamma delta").unwrap();
update_index(&RealFs, dir.path(), &cfg(), &small_build(), &NoProgress).unwrap();
assert_eq!(hits(&before, "alpha"), 1);
assert_eq!(hits(&before, "gamma"), 0);
assert!(live.refresh().unwrap());
let after = live.snapshot();
assert_eq!(hits(&after, "alpha"), 1);
assert_eq!(hits(&after, "gamma"), 1);
assert!(!live.refresh().unwrap(), "no further change");
}
#[test]
fn merge_and_update_never_invalidate_an_in_flight_snapshot() {
let dir = tempfile::tempdir().unwrap();
for i in 0..12 {
fs::write(dir.path().join(format!("f{i}.txt")), format!("alpha beta{i} gamma")).unwrap();
}
update_index(&RealFs, dir.path(), &cfg(), &small_build(), &NoProgress).unwrap();
let live = Arc::new(LiveIndex::open(Arc::new(RealFs), dir.path()).unwrap().unwrap());
let policy = MergePolicy { tier_size: 2, max_merge: 4, ..Default::default() };
let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
let start = Arc::new(Barrier::new(2));
let writer = {
let root = dir.path().to_path_buf();
let stop = stop.clone();
let start = start.clone();
thread::spawn(move || {
start.wait();
for i in 0..15 {
fs::write(root.join(format!("f{}.txt", 100 + i)), format!("alpha extra{i} gamma")).unwrap();
update_index(&RealFs, &root, &cfg(), &small_build(), &NoProgress).unwrap();
let _ = merge_index(&RealFs, &root, &policy);
}
stop.store(true, std::sync::atomic::Ordering::Release);
})
};
let reader = {
let live = live.clone();
let stop = stop.clone();
let start = start.clone();
thread::spawn(move || {
start.wait();
let mut held = Vec::new();
while !stop.load(std::sync::atomic::Ordering::Acquire) {
let snap = live.snapshot();
assert!(hits(&snap, "alpha") >= 12, "every generation must still contain the original docs");
held.push(snap);
if held.len() > 5 {
held.remove(0);
}
let _ = live.refresh();
}
for snap in &held {
assert!(hits(snap, "alpha") >= 12);
}
})
};
writer.join().unwrap();
reader.join().unwrap();
live.refresh().unwrap();
let final_snap = live.snapshot();
assert_eq!(hits(&final_snap, "alpha"), 27);
}
}