post_cortex_embeddings/
text.rs

1// Copyright (c) 2026 Julius ML
2// Licensed under the MIT License. See LICENSE at the workspace root.
3
4//! Tokenizing text for lexical (BM25) search, shared by every backend: the
5//! SurrealDB full-text query and the in-memory index of
6//! [`VectorDB`](crate::VectorDB). Language-independent: no stopword lists.
7//! Words too common to help are found from the data instead, by document
8//! frequency ([`VectorDB::query_terms`](crate::VectorDB::query_terms)).
9
10/// The words of `text`, in order and with repeats: lowercased, split on
11/// anything that is not a letter or digit (any script).
12pub fn tokens(text: &str) -> Vec<String> {
13    text.to_lowercase()
14        .split(|c: char| !c.is_alphanumeric())
15        .filter(|t| !t.is_empty())
16        .map(String::from)
17        .collect()
18}
19
20#[cfg(test)]
21mod tests {
22    use super::tokens;
23
24    #[test]
25    fn splits_any_script_on_punctuation() {
26        assert_eq!(
27            tokens("What did Caroline say?").join(" "),
28            "what did caroline say"
29        );
30        assert_eq!(
31            tokens("Какво решихме за базата?").join(" "),
32            "какво решихме за базата"
33        );
34        assert_eq!(
35            tokens("Přišel jsem, viděl jsem").join(" "),
36            "přišel jsem viděl jsem"
37        );
38        assert!(tokens("?!").is_empty());
39    }
40}
post_cortex_embeddings/text.rs

post_cortex_embeddings/
text.rs
