urna_runtime/bm25/mod.rs
1//! tiny BM25 inverted index for the hybrid search path.
2//!
3//! Tokenization is intentionally simple - lowercase, split on
4//! Unicode-aware whitespace + punctuation, drop tokens with `len < 2`.
5//! Multilingual corpora (PT-BR, EN, ES) get reasonable behavior without
6//! a stemmer or stop list. A future version can pluggable-ize the
7//! tokenizer if needed.
8//!
9//! The index is small enough that we keep it fully decoded in memory at
10//! open time. On-disk layout (section `0x08`, payload version 1):
11//!
12//! ```text
13//! u32 LE payload_version = 1
14//! f32 LE k1 (BM25 hyperparameter)
15//! f32 LE b (BM25 hyperparameter)
16//! f32 LE avgdl (average doc length)
17//! u32 LE n_docs
18//! u32 LE n_terms
19//! for d in 0..n_docs:
20//! u32 LE dl_d (document length in tokens)
21//! for t in 0..n_terms:
22//! u32 LE len_t
23//! bytes token_t (UTF-8)
24//! u32 LE df_t (document frequency)
25//! for posting in postings_t:
26//! u32 LE doc_id
27//! u32 LE tf
28//! ```
29
30mod codec;
31mod fusion;
32mod index;
33mod tokenize;
34
35#[cfg(test)]
36mod tests;
37
38pub use fusion::rrf_union;
39pub use index::{BM25_PAYLOAD_VERSION, Bm25Index, DEFAULT_B, DEFAULT_K1};
40#[cfg(test)]
41pub(crate) use tokenize::tokenize;