pub mod builder;
pub mod doc_table;
pub mod memory;
pub mod postings;
use std::time::Duration;
pub use builder::{BuildConfig, IndexBuilder};
pub use doc_table::{DocId, DocMeta, DocStatus, DocTable};
pub use memory::MemTermStore;
pub use postings::{Position, PostingCursor, PostingList, TermStore};
pub use crate::store::{build_from_dir, Index};
use crate::crawler::CrawlStats;
pub trait SegmentReader {
type List<'a>: PostingList
where
Self: 'a;
fn base(&self) -> DocId;
fn num_docs(&self) -> u32;
fn postings(&self, term: &str) -> Option<Self::List<'_>>;
fn doc_len(&self, local: DocId) -> Option<u32>;
}
pub trait IndexReader {
type Segment: SegmentReader;
fn segments(&self) -> impl Iterator<Item = &Self::Segment> + '_;
fn indexed_count(&self) -> u32;
fn avg_len(&self) -> f32;
fn doc(&self, id: DocId) -> Option<DocMeta>;
fn stats(&self) -> &IndexStats;
}
pub struct MemIndex<S: TermStore = MemTermStore> {
docs: DocTable,
terms: S,
stats: IndexStats,
}
impl<S: TermStore> MemIndex<S> {
pub(crate) fn new(docs: DocTable, terms: S, stats: IndexStats) -> Self {
Self { docs, terms, stats }
}
pub fn docs(&self) -> &DocTable {
&self.docs
}
pub fn terms(&self) -> &S {
&self.terms
}
pub fn into_parts(self) -> (DocTable, S, IndexStats) {
(self.docs, self.terms, self.stats)
}
}
impl<S: TermStore> SegmentReader for MemIndex<S> {
type List<'a>
= S::List<'a>
where
Self: 'a;
fn base(&self) -> DocId {
0
}
fn num_docs(&self) -> u32 {
self.docs.id_bound()
}
fn postings(&self, term: &str) -> Option<S::List<'_>> {
self.terms.get(term)
}
fn doc_len(&self, local: DocId) -> Option<u32> {
self.docs.get(local).filter(|m| m.status == DocStatus::Indexed).map(|m| m.len)
}
}
impl<S: TermStore> IndexReader for MemIndex<S> {
type Segment = Self;
fn segments(&self) -> impl Iterator<Item = &Self> + '_ {
std::iter::once(self)
}
fn indexed_count(&self) -> u32 {
self.docs.indexed_count()
}
fn avg_len(&self) -> f32 {
self.docs.avg_len()
}
fn doc(&self, id: DocId) -> Option<DocMeta> {
self.docs.get(id).cloned()
}
fn stats(&self) -> &IndexStats {
&self.stats
}
}
#[derive(Debug, Default, Clone)]
pub struct IndexStats {
pub crawl: CrawlStats,
pub docs_total: u32,
pub docs_indexed: u32,
pub docs_skipped: u32,
pub total_tokens: u64,
pub total_postings: u64,
pub unique_terms: usize,
pub segments: u32,
pub crawl_time: Duration,
pub build_time: Duration,
pub open_time: Duration,
pub memory_bytes: usize,
pub on_disk_bytes: u64,
pub rss_bytes: Option<usize>,
pub rss_delta_bytes: Option<i64>,
}
pub fn rss_bytes() -> Option<usize> {
memory_stats::memory_stats().map(|m| m.physical_mem)
}