searchez 1.0.0

A searchable-model layer for Rust: make a type searchable, keep the index in sync, and search with real relevance ranking — over a pluggable backend, with a batteries-included in-memory engine that needs no external service.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

// searchez/src/token.rs
//
// Turning text into terms. A deliberately simple, dependency-free analyzer:
// lowercase, split on non-alphanumeric, keep the pieces. Good enough for the
// in-memory backend; a server backend brings its own analysis.

/// Split text into lowercase alphanumeric terms.
pub fn tokenize(text: &str) -> Vec<String> {
    text.split(|c: char| !c.is_alphanumeric())
        .filter(|t| !t.is_empty())
        .map(|t| t.to_lowercase())
        .collect()
}

/// Pull every searchable term out of a JSON value: strings are tokenized,
/// numbers and booleans are stringified so `year:2018` is findable, arrays and
/// objects are walked. This is what makes a whole document full-text searchable
/// without the caller listing fields.
pub fn tokenize_value(value: &serde_json::Value, out: &mut Vec<String>) {
    use serde_json::Value;
    match value {
        Value::String(s) => out.extend(tokenize(s)),
        Value::Number(n) => out.extend(tokenize(&n.to_string())),
        Value::Bool(b) => out.push(b.to_string()),
        Value::Array(a) => a.iter().for_each(|v| tokenize_value(v, out)),
        Value::Object(m) => m.values().for_each(|v| tokenize_value(v, out)),
        Value::Null => {}
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn lowercases_and_splits_on_punctuation() {
        assert_eq!(tokenize("Rust-lang, 2018!"), vec!["rust", "lang", "2018"]);
    }

    #[test]
    fn numbers_and_arrays_are_searchable() {
        let mut out = Vec::new();
        tokenize_value(&serde_json::json!({"tags": ["Fast", "Safe"], "year": 2018}), &mut out);
        assert!(out.contains(&"fast".to_string()));
        assert!(out.contains(&"safe".to_string()));
        assert!(out.contains(&"2018".to_string()));
    }
}