urna-runtime 0.5.0

mmap runtime of the urna vector database: simd exact search, hnsw, bm25, graph and hybrid search with mandatory exact rerank
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

//! unicode-aware token splitter used by both the index builder and
//! search path. Lowercase, split on non-alphanumeric, drop tokens of
//! length < 2.

/// Lowercase, split on non-alphanumerics, drop tokens of length < 2.
/// Unicode-aware: handles PT-BR accents (ã, ç, õ) by virtue of `char`
/// iteration. Simple but effective enough for fake-news / news corpora.
pub(crate) fn tokenize(s: &str) -> Vec<String> {
    let mut out: Vec<String> = Vec::new();
    let mut cur = String::new();
    for c in s.chars() {
        if c.is_alphanumeric() {
            for low in c.to_lowercase() {
                cur.push(low);
            }
        } else if !cur.is_empty() {
            if cur.chars().count() >= 2 {
                out.push(std::mem::take(&mut cur));
            } else {
                cur.clear();
            }
        }
    }
    if cur.chars().count() >= 2 {
        out.push(cur);
    }
    out
}