Skip to main content

mnemo_engine/
text.rs

1//! FMT-9d (Phase 2): the keyword side of hybrid recall.
2//!
3//! * [`tokenize`]: UAX #29 words, lowercased, plus **compound tokens**. A
4//!   compound is a whitespace-separated run with its leading and trailing
5//!   punctuation stripped, kept whole when it still has punctuation inside,
6//!   so identifiers like `BP.1`, `v0.9.0` or `PR-10b.2` are searchable as
7//!   themselves (UAX #29 splits them). No stemming in v1.
8//! * TOK-1 (v0.15), [`Tokenizer::V2`]: tokenizer 1's tokens plus the parts
9//!   of each, split on `.`, `/`, `_`, `-` and `::`, and a possessive's stem,
10//!   so `nosync` finds `mnemo.nosync` and `icloud` finds `iCloud's`. A
11//!   segment written with it carries [`V2_MARK`] (see [`Tokenizer`]).
12//! * Segments: the on-disk unit of the BM25 index. A segment is
13//!   `[u32 dict length][dict][postings]`. The dict is an rmp list of
14//!   `(term, offset, length)`, sorted by term; postings are, per term,
15//!   `(doc delta varint, tf u8)` pairs in ascending doc order.
16//! * [`bm25`]: the per-term score, with k1 1.2 and b 0.75 by default.
17
18use serde::{Deserialize, Serialize};
19use unicode_segmentation::UnicodeSegmentation;
20
21use crate::error::{MnemoError, Result};
22
23/// The tokenizer above. Recorded in the BM25 manifest, so a later
24/// tokenizer (a CJK bigram one, say) can be added without a format change.
25///
26/// TOK-1: the manifest byte stays 1 whatever the segments hold, because
27/// every v9 build before v0.15 refuses to open a file whose byte isn't 1.
28/// Tokenizer 2 is recorded per segment instead ([`V2_MARK`]).
29pub(crate) const TOKENIZER_V1: u8 = 1;
30
31/// TOK-1: the reserved term a segment written with tokenizer 2 carries.
32/// Its one posting is document 0, which is never live, so no query
33/// matches it and every merge drops it. No tokenizer makes it (it starts
34/// with a NUL), and it sorts first in the dict.
35pub(crate) const V2_MARK: &str = "\u{0}mnemo.tokenizer.2";
36
37/// TOK-1: the reserved term a segment carries when some of its documents
38/// were written with tokenizer 2 and some with tokenizer 1 (a v0.15 merge
39/// of a mixed index). Like [`V2_MARK`], it posts document 0 only.
40pub(crate) const MIXED_MARK: &str = "\u{0}mnemo.tokenizer.mixed";
41
42/// TOK-1: the tokenizer a document or a query is split with.
43#[derive(Clone, Copy, Debug, PartialEq, Eq)]
44pub(crate) enum Tokenizer {
45    /// UAX #29 words and compounds (FMT-9d).
46    V1,
47    /// Tokenizer 1 plus the parts (TOK-1, v0.15).
48    V2,
49}
50
51/// TOK-1: which tokenizer a segment's documents were written with, from
52/// its first dict term.
53#[derive(Clone, Copy, Debug, PartialEq, Eq)]
54pub(crate) enum SegmentTokens {
55    /// No mark: tokenizer 1 (any build), or an older build's merge.
56    V1,
57    /// [`V2_MARK`]: every document is tokenizer 2.
58    V2,
59    /// [`MIXED_MARK`]: some of each.
60    Mixed,
61}
62
63impl SegmentTokens {
64    /// The mark of a segment whose dict starts with `first`.
65    pub(crate) fn from_first_term(first: Option<&str>) -> Self {
66        match first {
67            Some(V2_MARK) => SegmentTokens::V2,
68            Some(MIXED_MARK) => SegmentTokens::Mixed,
69            _ => SegmentTokens::V1,
70        }
71    }
72
73    /// The mark a segment built from these parts carries: tokenizer 2 only
74    /// when every part is, tokenizer 1 only when every part is.
75    pub(crate) fn combine(parts: impl IntoIterator<Item = SegmentTokens>) -> Option<Self> {
76        let mut out: Option<SegmentTokens> = None;
77        for p in parts {
78            out = Some(match (out, p) {
79                (None, p) => p,
80                (Some(a), b) if a == b => a,
81                _ => SegmentTokens::Mixed,
82            });
83        }
84        out
85    }
86}
87
88/// TOK-1: the state of a file's keyword index, as `info` and `verify` show
89/// it.
90#[derive(Clone, Copy, Debug, PartialEq, Eq)]
91pub enum KeywordTokenizer {
92    /// Tokenizer 1: whole words as written. A compaction or `reembed`
93    /// rebuilds the index with tokenizer 2.
94    V1,
95    /// Tokenizer 2: whole words and their parts.
96    V2,
97    /// Some documents have tokenizer 2 and some tokenizer 1, because a
98    /// build before v0.15 wrote to or merged the index. Parts match only
99    /// the tokenizer 2 documents; whole words match every document. A
100    /// compaction or `reembed` rebuilds it with tokenizer 2.
101    Mixed,
102}
103
104impl KeywordTokenizer {
105    /// A file's state from its segments' marks: no segment is tokenizer 2
106    /// (an empty index takes tokenizer 2 from its first segment).
107    pub(crate) fn of(segments: impl IntoIterator<Item = SegmentTokens>) -> Self {
108        match SegmentTokens::combine(segments) {
109            None | Some(SegmentTokens::V2) => KeywordTokenizer::V2,
110            Some(SegmentTokens::V1) => KeywordTokenizer::V1,
111            Some(SegmentTokens::Mixed) => KeywordTokenizer::Mixed,
112        }
113    }
114
115    /// The tokenizer new documents and queries use: tokenizer 1 only for a
116    /// pure tokenizer 1 index, so it stays pure until it's rebuilt. In a
117    /// mixed index a query's whole words match every document and its parts
118    /// match the tokenizer 2 ones.
119    pub(crate) fn tokenizer(self) -> Tokenizer {
120        match self {
121            KeywordTokenizer::V1 => Tokenizer::V1,
122            KeywordTokenizer::V2 | KeywordTokenizer::Mixed => Tokenizer::V2,
123        }
124    }
125
126    /// A short name: `"1"`, `"2"` or `"mixed"`.
127    pub fn as_str(self) -> &'static str {
128        match self {
129            KeywordTokenizer::V1 => "1",
130            KeywordTokenizer::V2 => "2",
131            KeywordTokenizer::Mixed => "mixed",
132        }
133    }
134}
135
136/// Longest token indexed, in bytes (longer ones are dropped).
137const MAX_TOKEN_BYTES: usize = 64;
138
139/// The tokens of `text` under tokenizer `tok`, with repeats.
140///
141/// Tokenizer 2 (R-008.2) works per whitespace run: the run's UAX #29 words,
142/// its compound, and the parts of each, with **each distinct term counted
143/// once per run**. A term in two runs counts twice. (Tokenizer 1 can emit a
144/// dotted name twice, once as a UAX #29 word and once as the compound; it
145/// stays as released, since existing indexes depend on it.)
146pub(crate) fn tokenize_with(tok: Tokenizer, text: &str) -> Vec<String> {
147    match tok {
148        Tokenizer::V1 => tokenize(text),
149        Tokenizer::V2 => {
150            let mut out = Vec::new();
151            for run in text.split_whitespace() {
152                let mut terms: Vec<String> = run
153                    .unicode_words()
154                    .map(|w| w.to_lowercase())
155                    .filter(|w| w.len() <= MAX_TOKEN_BYTES)
156                    .collect();
157                terms.extend(compound(run));
158                let whole = terms.len();
159                for i in 0..whole {
160                    let parts = token_parts(&terms[i]);
161                    terms.extend(parts);
162                }
163                let mut seen = std::collections::BTreeSet::new();
164                out.extend(terms.into_iter().filter(|t| seen.insert(t.clone())));
165            }
166            out
167        }
168    }
169}
170
171/// FMT-9d: a whitespace run's compound token: the run with its leading and
172/// trailing punctuation stripped, when it still has punctuation inside.
173fn compound(run: &str) -> Option<String> {
174    let t = run.trim_matches(|c: char| !c.is_alphanumeric());
175    if t.len() < 2 || t.len() > MAX_TOKEN_BYTES {
176        return None;
177    }
178    (t.chars().any(|c| !c.is_alphanumeric()) && t.chars().any(|c| c.is_alphanumeric()))
179        .then(|| t.to_lowercase())
180}
181
182/// TOK-1: the parts of one tokenizer 1 token: a possessive's stem, and the
183/// pieces between `.`, `/`, `_`, `-` and `::`. Only parts that differ from
184/// the token itself, and no part that is only digits (the `9` and `0` of
185/// `v0.9.0` would match nearly every versioned text; the whole token
186/// still matches the version exactly).
187fn token_parts(token: &str) -> Vec<String> {
188    let mut out: Vec<String> = Vec::new();
189    let stem = token
190        .strip_suffix("'s")
191        .or_else(|| token.strip_suffix("\u{2019}s"))
192        .filter(|s| !s.is_empty());
193    if let Some(s) = stem {
194        out.push(s.to_string());
195    }
196    let base = stem.unwrap_or(token);
197    let pieces: Vec<&str> = base
198        .split("::")
199        .flat_map(|p| p.split(['.', '/', '_', '-']))
200        .filter(|p| !p.is_empty())
201        .collect();
202    let pieces: Vec<&str> = if pieces.len() > 1 {
203        pieces
204            .into_iter()
205            .filter(|p| !p.chars().all(|c| c.is_ascii_digit()))
206            .collect()
207    } else {
208        Vec::new()
209    };
210    for p in pieces {
211        if p != token && !out.iter().any(|o| o == p) {
212            out.push(p.to_string());
213        }
214    }
215    out
216}
217
218/// The tokenizer 1 tokens of `text`, in order, with repeats.
219pub(crate) fn tokenize(text: &str) -> Vec<String> {
220    let mut out: Vec<String> = text
221        .unicode_words()
222        .map(|w| w.to_lowercase())
223        .filter(|w| w.len() <= MAX_TOKEN_BYTES)
224        .collect();
225    out.extend(text.split_whitespace().filter_map(compound));
226    out
227}
228
229/// Term frequencies of one document: `(term, tf)`, sorted by term, tf
230/// saturating at 255. Also returns the document length (all tokens).
231pub(crate) fn term_freqs(tok: Tokenizer, text: &str) -> (Vec<(String, u8)>, u16) {
232    let toks = tokenize_with(tok, text);
233    let len = toks.len().min(u16::MAX as usize) as u16;
234    let mut map: std::collections::BTreeMap<String, u32> = std::collections::BTreeMap::new();
235    for t in toks {
236        *map.entry(t).or_default() += 1;
237    }
238    (
239        map.into_iter()
240            .map(|(t, n)| (t, n.min(255) as u8))
241            .collect(),
242        len,
243    )
244}
245
246/// BM25 score of one term in one document.
247pub(crate) fn bm25(tf: u8, idf: f32, doc_len: u16, avgdl: f32, k1: f32, b: f32) -> f32 {
248    let tf = tf as f32;
249    let norm = 1.0 - b + b * (doc_len as f32 / avgdl.max(1e-6));
250    idf * tf * (k1 + 1.0) / (tf + k1 * norm)
251}
252
253/// BM25 idf for a term in `df` of `n` live documents.
254pub(crate) fn idf(n: usize, df: usize) -> f32 {
255    let (n, df) = (n as f32, df as f32);
256    (1.0 + (n - df + 0.5) / (df + 0.5)).ln()
257}
258
259/// An in-memory keyword index over a set of texts, with the tokenizer and
260/// BM25 scoring (k1 1.2, b 0.75) of v9's keyword recall, for tools that
261/// search what they've already read, without the file open (`mnemo
262/// inspect`). It needs no embedder and works on texts from any format.
263/// It uses tokenizer 2 (TOK-1), so a part of a dotted name matches.
264///
265/// ```
266/// use mnemo_engine::KeywordIndex;
267/// let ix = KeywordIndex::new(["the deploy runs on v0.9.0", "tea, not coffee"]);
268/// assert_eq!(ix.search("v0.9.0", 5)[0].0, 0);
269/// assert!(ix.search("nothing", 5).is_empty());
270/// let ix = KeywordIndex::new(["the repo is mnemo.nosync", "iCloud's copy"]);
271/// assert_eq!(ix.search("nosync", 5)[0].0, 0);
272/// assert_eq!(ix.search("icloud", 5)[0].0, 1);
273/// ```
274#[derive(Clone, Debug, Default)]
275pub struct KeywordIndex {
276    postings: std::collections::BTreeMap<String, Vec<(usize, u8)>>,
277    lens: Vec<u16>,
278    avgdl: f32,
279}
280
281impl KeywordIndex {
282    /// Index `texts`; a hit's index is the text's position here.
283    pub fn new<'a>(texts: impl IntoIterator<Item = &'a str>) -> Self {
284        let mut ix = Self::default();
285        for (doc, text) in texts.into_iter().enumerate() {
286            let (tfs, len) = term_freqs(Tokenizer::V2, text);
287            for (term, tf) in tfs {
288                ix.postings.entry(term).or_default().push((doc, tf));
289            }
290            ix.lens.push(len);
291        }
292        let total: u64 = ix.lens.iter().map(|&l| l as u64).sum();
293        ix.avgdl = if ix.lens.is_empty() {
294            0.0
295        } else {
296            total as f32 / ix.lens.len() as f32
297        };
298        ix
299    }
300
301    /// The texts that match any of `query`'s terms, best first, at most
302    /// `top_k`, as `(index, score)`. Ties keep the texts' order.
303    pub fn search(&self, query: &str, top_k: usize) -> Vec<(usize, f32)> {
304        let terms: std::collections::BTreeSet<String> =
305            tokenize_with(Tokenizer::V2, query).into_iter().collect();
306        let n = self.lens.len();
307        let mut scores: std::collections::BTreeMap<usize, f32> = std::collections::BTreeMap::new();
308        for term in &terms {
309            let Some(posts) = self.postings.get(term) else {
310                continue;
311            };
312            let idf = idf(n, posts.len());
313            for &(doc, tf) in posts {
314                *scores.entry(doc).or_default() +=
315                    bm25(tf, idf, self.lens[doc], self.avgdl, 1.2, 0.75);
316            }
317        }
318        let mut hits: Vec<(usize, f32)> = scores.into_iter().collect();
319        hits.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
320        hits.truncate(top_k);
321        hits
322    }
323
324    /// How many texts it holds.
325    pub fn len(&self) -> usize {
326        self.lens.len()
327    }
328
329    /// Whether it holds no text.
330    pub fn is_empty(&self) -> bool {
331        self.lens.is_empty()
332    }
333}
334
335/// One dict entry: a term and its postings' byte range.
336#[derive(Serialize, Deserialize, Clone, Debug)]
337pub(crate) struct DictEntry {
338    pub term: String,
339    pub off: u32,
340    pub len: u32,
341}
342
343fn put_varint(out: &mut Vec<u8>, mut v: u32) {
344    while v >= 0x80 {
345        out.push((v as u8) | 0x80);
346        v >>= 7;
347    }
348    out.push(v as u8);
349}
350
351/// Decode a postings slice into `(doc, tf)` pairs.
352pub(crate) fn decode_postings(b: &[u8]) -> Result<Vec<(u32, u8)>> {
353    let bad = || MnemoError::Invalid("malformed BM25 postings".into());
354    let mut out = Vec::new();
355    let mut i = 0usize;
356    let mut doc = 0u32;
357    while i < b.len() {
358        let mut delta = 0u32;
359        let mut shift = 0u32;
360        loop {
361            let byte = *b.get(i).ok_or_else(bad)?;
362            i += 1;
363            if shift > 28 {
364                return Err(bad());
365            }
366            delta |= ((byte & 0x7f) as u32) << shift;
367            if byte & 0x80 == 0 {
368                break;
369            }
370            shift += 7;
371        }
372        doc = doc.checked_add(delta).ok_or_else(bad)?;
373        let tf = *b.get(i).ok_or_else(bad)?;
374        i += 1;
375        out.push((doc, tf));
376    }
377    Ok(out)
378}
379
380/// Build a segment from documents' term frequencies: `(doc, terms)`.
381/// TOK-1: a segment whose documents are tokenizer 2, or a mix, carries its
382/// mark ([`V2_MARK`], [`MIXED_MARK`]) as a term posting document 0.
383pub(crate) fn build_segment(
384    docs: &[(u32, Vec<(String, u8)>)],
385    tokens: SegmentTokens,
386) -> Result<Vec<u8>> {
387    let mut by_term: std::collections::BTreeMap<&str, Vec<(u32, u8)>> =
388        std::collections::BTreeMap::new();
389    match tokens {
390        SegmentTokens::V1 => {}
391        SegmentTokens::V2 => {
392            by_term.insert(V2_MARK, vec![(0, 1)]);
393        }
394        SegmentTokens::Mixed => {
395            by_term.insert(MIXED_MARK, vec![(0, 1)]);
396        }
397    }
398    for (doc, terms) in docs {
399        for (t, tf) in terms {
400            by_term.entry(t.as_str()).or_default().push((*doc, *tf));
401        }
402    }
403    let mut dict: Vec<DictEntry> = Vec::with_capacity(by_term.len());
404    let mut postings: Vec<u8> = Vec::new();
405    for (term, mut list) in by_term {
406        list.sort_unstable_by_key(|&(d, _)| d);
407        let off = postings.len() as u32;
408        let mut prev = 0u32;
409        for (d, tf) in list {
410            put_varint(&mut postings, d - prev);
411            postings.push(tf);
412            prev = d;
413        }
414        dict.push(DictEntry {
415            term: term.to_string(),
416            off,
417            len: postings.len() as u32 - off,
418        });
419    }
420    let dict_bytes = rmp_serde::to_vec(&dict).map_err(|e| MnemoError::Serialize(e.to_string()))?;
421    let mut out = Vec::with_capacity(4 + dict_bytes.len() + postings.len());
422    out.extend_from_slice(&(dict_bytes.len() as u32).to_le_bytes());
423    out.extend_from_slice(&dict_bytes);
424    out.extend_from_slice(&postings);
425    Ok(out)
426}
427
428#[cfg(test)]
429mod tests {
430    use super::*;
431
432    #[test]
433    fn tokenizer_keeps_identifiers_whole() {
434        let t = tokenize("Ship BP.1 in v0.9.0 (see PR-10b.2), then flush().");
435        for want in ["ship", "bp", "1", "bp.1", "v0.9.0", "pr-10b.2", "flush"] {
436            assert!(t.contains(&want.to_string()), "{want} missing from {t:?}");
437        }
438        assert!(
439            !t.contains(&"flush()".to_string()),
440            "trailing punctuation stripped"
441        );
442        assert!(tokenize("Ünïcödé Wörds").contains(&"ünïcödé".to_string()));
443    }
444
445    #[test]
446    fn segment_round_trips() {
447        let docs = vec![
448            (3u32, term_freqs(Tokenizer::V1, "alpha beta beta").0),
449            (7u32, term_freqs(Tokenizer::V1, "beta gamma").0),
450            (300u32, term_freqs(Tokenizer::V1, "alpha").0),
451        ];
452        let seg = build_segment(&docs, SegmentTokens::V1).unwrap();
453        let dlen = u32::from_le_bytes(seg[0..4].try_into().unwrap()) as usize;
454        let dict: Vec<DictEntry> = rmp_serde::from_slice(&seg[4..4 + dlen]).unwrap();
455        let post = &seg[4 + dlen..];
456        let find = |t: &str| {
457            let e = dict.iter().find(|e| e.term == t).unwrap();
458            decode_postings(&post[e.off as usize..(e.off + e.len) as usize]).unwrap()
459        };
460        assert_eq!(find("alpha"), vec![(3, 1), (300, 1)]);
461        assert_eq!(find("beta"), vec![(3, 2), (7, 1)]);
462        assert_eq!(find("gamma"), vec![(7, 1)]);
463    }
464
465    #[test]
466    fn tokenizer_2_adds_the_parts() {
467        let t = tokenize_with(
468            Tokenizer::V2,
469            "Moved to ~/Documents/GitHub/mnemo.nosync, out of iCloud's reach; \
470             see std::fs::rename, max_bytes and PR-10b.2 in v0.9.0. It\u{2019}s done.",
471        );
472        for want in [
473            "mnemo.nosync",
474            "mnemo",
475            "nosync",
476            "icloud's",
477            "icloud",
478            "std::fs::rename",
479            "std",
480            "fs",
481            "rename",
482            "max_bytes",
483            "max",
484            "bytes",
485            "pr-10b.2",
486            "pr",
487            "10b",
488            "v0",
489            "documents",
490            "github",
491            "it",
492        ] {
493            assert!(t.contains(&want.to_string()), "{want} missing from {t:?}");
494        }
495        // No digit-only parts.
496        let v = tokenize_with(Tokenizer::V2, "v0.9.0");
497        assert!(v.contains(&"v0.9.0".to_string()) && v.contains(&"v0".to_string()));
498        assert!(!v.contains(&"9".to_string()), "{v:?}");
499        // Tokenizer 1 is unchanged: whole words only.
500        let v1 = tokenize_with(Tokenizer::V1, "mnemo.nosync iCloud's");
501        assert!(!v1.contains(&"nosync".to_string()));
502        assert!(!v1.contains(&"icloud".to_string()));
503        // A token with nothing to split adds nothing.
504        assert_eq!(
505            tokenize_with(Tokenizer::V2, "plain words"),
506            tokenize_with(Tokenizer::V1, "plain words")
507        );
508    }
509
510    /// R-008.2 (REVIEWER-A's case): tokenizer 2 counts each term once per
511    /// whitespace run; a term in two runs counts twice.
512    #[test]
513    fn tokenizer_2_counts_a_term_once_per_run() {
514        let (tfs, len) = term_freqs(
515            Tokenizer::V2,
516            "mnemo.nosync key-file open_shared Mnemo::open iCloud's",
517        );
518        let tf = |t: &str| tfs.iter().find(|(x, _)| x == t).map(|(_, n)| *n);
519        for (term, n) in &tfs {
520            let want = if term == "mnemo" || term == "open" {
521                2
522            } else {
523                1
524            };
525            assert_eq!(*n, want, "{term}: {tfs:?}");
526        }
527        for t in [
528            "mnemo.nosync",
529            "nosync",
530            "key-file",
531            "key",
532            "file",
533            "open_shared",
534            "shared",
535            "mnemo::open",
536            "icloud's",
537            "icloud",
538        ] {
539            assert_eq!(tf(t), Some(1), "{t}: {tfs:?}");
540        }
541        assert_eq!(tf("mnemo"), Some(2));
542        assert_eq!(tf("open"), Some(2));
543        // The length counts the distinct terms of each run.
544        assert_eq!(
545            len as usize,
546            tfs.iter().map(|(_, n)| *n as usize).sum::<usize>()
547        );
548        // Tokenizer 1 is unchanged: the dotted name twice, as in v0.14.0.
549        let (v1, _) = term_freqs(Tokenizer::V1, "mnemo.nosync");
550        assert_eq!(v1, vec![("mnemo.nosync".to_string(), 2)]);
551    }
552
553    #[test]
554    fn a_marked_segment_starts_with_its_mark_and_posts_document_0() {
555        let docs = vec![(5u32, term_freqs(Tokenizer::V2, "mnemo.nosync").0)];
556        for (tokens, mark) in [
557            (SegmentTokens::V2, Some(V2_MARK)),
558            (SegmentTokens::Mixed, Some(MIXED_MARK)),
559            (SegmentTokens::V1, None),
560        ] {
561            let seg = build_segment(&docs, tokens).unwrap();
562            let dlen = u32::from_le_bytes(seg[0..4].try_into().unwrap()) as usize;
563            let dict: Vec<DictEntry> = rmp_serde::from_slice(&seg[4..4 + dlen]).unwrap();
564            let first = dict.first().map(|e| e.term.as_str());
565            assert_eq!(SegmentTokens::from_first_term(first), tokens);
566            if let Some(m) = mark {
567                let e = &dict[0];
568                assert_eq!(e.term, m);
569                let post = &seg[4 + dlen..];
570                let p = decode_postings(&post[e.off as usize..(e.off + e.len) as usize]).unwrap();
571                assert_eq!(p, vec![(0, 1)]);
572            }
573        }
574    }
575
576    #[test]
577    fn a_files_tokenizer_comes_from_its_segments() {
578        use SegmentTokens as S;
579        assert_eq!(KeywordTokenizer::of([]), KeywordTokenizer::V2);
580        assert_eq!(KeywordTokenizer::of([S::V2, S::V2]), KeywordTokenizer::V2);
581        assert_eq!(KeywordTokenizer::of([S::V1, S::V1]), KeywordTokenizer::V1);
582        assert_eq!(
583            KeywordTokenizer::of([S::V2, S::V1]),
584            KeywordTokenizer::Mixed
585        );
586        assert_eq!(KeywordTokenizer::of([S::Mixed]), KeywordTokenizer::Mixed);
587        assert_eq!(KeywordTokenizer::V1.tokenizer(), Tokenizer::V1);
588        assert_eq!(KeywordTokenizer::Mixed.tokenizer(), Tokenizer::V2);
589    }
590
591    #[test]
592    fn bm25_prefers_rarer_terms_and_shorter_docs() {
593        assert!(idf(100, 1) > idf(100, 50));
594        assert!(bm25(1, 1.0, 5, 10.0, 1.2, 0.75) > bm25(1, 1.0, 20, 10.0, 1.2, 0.75));
595        assert!(bm25(3, 1.0, 10, 10.0, 1.2, 0.75) > bm25(1, 1.0, 10, 10.0, 1.2, 0.75));
596    }
597}