hayson-engine 0.27.0

use serde_json::Value;

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SearchTerm {
    pub token: String,
    pub prefix: bool,
}

// NOTE: hand-folded diacritics, not full Unicode NFD -- see report for the coverage tradeoff.
fn fold_char(lowered: char) -> &'static str {
    match lowered {
        '\u{00e0}' | '\u{00e1}' | '\u{00e2}' | '\u{00e3}' | '\u{00e4}' | '\u{00e5}' | '\u{0101}' | '\u{0103}'
        | '\u{0105}' => "a",
        '\u{00e7}' | '\u{0107}' | '\u{0109}' | '\u{010b}' | '\u{010d}' => "c",
        '\u{010f}' | '\u{0111}' => "d",
        '\u{00e8}' | '\u{00e9}' | '\u{00ea}' | '\u{00eb}' | '\u{0113}' | '\u{0115}' | '\u{0117}' | '\u{0119}'
        | '\u{011b}' => "e",
        '\u{011d}' | '\u{011f}' | '\u{0121}' | '\u{0123}' => "g",
        '\u{0125}' | '\u{0127}' => "h",
        '\u{00ec}' | '\u{00ed}' | '\u{00ee}' | '\u{00ef}' | '\u{0129}' | '\u{012b}' | '\u{012d}' | '\u{012f}'
        | '\u{0131}' => "i",
        '\u{0135}' => "j",
        '\u{0137}' => "k",
        '\u{013a}' | '\u{013c}' | '\u{013e}' | '\u{0140}' | '\u{0142}' => "l",
        '\u{00f1}' | '\u{0144}' | '\u{0146}' | '\u{0148}' => "n",
        '\u{00f2}' | '\u{00f3}' | '\u{00f4}' | '\u{00f5}' | '\u{00f6}' | '\u{00f8}' | '\u{014d}' | '\u{014f}'
        | '\u{0151}' => "o",
        '\u{0155}' | '\u{0157}' | '\u{0159}' => "r",
        '\u{015b}' | '\u{015d}' | '\u{015f}' | '\u{0161}' => "s",
        '\u{0163}' | '\u{0165}' | '\u{0167}' => "t",
        '\u{00f9}' | '\u{00fa}' | '\u{00fb}' | '\u{00fc}' | '\u{0169}' | '\u{016b}' | '\u{016d}' | '\u{016f}'
        | '\u{0171}' | '\u{0173}' => "u",
        '\u{0175}' => "w",
        '\u{00fd}' | '\u{0177}' | '\u{00ff}' => "y",
        '\u{017a}' | '\u{017c}' | '\u{017e}' => "z",
        '\u{00df}' => "ss",
        '\u{00e6}' => "ae",
        '\u{0153}' => "oe",
        _ => "",
    }
}

pub fn normalize_token(raw: &str) -> String {
    let mut folded = String::with_capacity(raw.len());
    for ch in raw.to_lowercase().chars() {
        let replacement = fold_char(ch);
        if replacement.is_empty() {
            folded.push(ch);
        } else {
            folded.push_str(replacement);
        }
    }
    folded
}

fn push_token(tokens: &mut Vec<String>, fragment: &str) {
    let token = normalize_token(fragment);
    if !token.is_empty() {
        tokens.push(token);
    }
}

pub fn tokenize(text: &str) -> Vec<String> {
    let mut tokens = Vec::new();
    let mut current = String::new();
    for ch in text.chars() {
        if ch.is_alphanumeric() {
            current.push(ch);
        } else if !current.is_empty() {
            push_token(&mut tokens, &current);
            current.clear();
        }
    }
    if !current.is_empty() {
        push_token(&mut tokens, &current);
    }
    tokens
}

pub fn tokenize_value(value: &Value) -> Vec<String> {
    match value {
        Value::String(s) => tokenize(s),
        Value::Array(items) => {
            let mut tokens = Vec::new();
            for item in items {
                if let Value::String(s) = item {
                    tokens.extend(tokenize(s));
                }
            }
            tokens
        }
        _ => Vec::new(),
    }
}

pub fn parse_search_terms(text: &str) -> Vec<SearchTerm> {
    let chars: Vec<char> = text.chars().collect();
    let mut terms = Vec::new();
    let mut at = 0;
    while at < chars.len() {
        if !chars[at].is_alphanumeric() {
            at += 1;
            continue;
        }
        let start = at;
        while at < chars.len() && chars[at].is_alphanumeric() {
            at += 1;
        }
        let prefix = at < chars.len() && chars[at] == '*';
        let raw: String = chars[start..at].iter().collect();
        if prefix {
            at += 1;
        }
        let token = normalize_token(&raw);
        if !token.is_empty() {
            terms.push(SearchTerm { token, prefix });
        }
    }
    terms
}

#[cfg(test)]
mod tests {
    use super::*;

    fn term(text: &str, prefix: bool) -> SearchTerm {
        SearchTerm {
            token: text.to_string(),
            prefix,
        }
    }

    #[test]
    fn folds_case_and_diacritics_so_lodz_meets_lodz() {
        let text = "\u{141}\u{f3}d\u{17a}, Krak\u{f3}w!";
        assert_eq!(tokenize(text), vec!["lodz", "krakow"]);
    }

    #[test]
    fn reads_a_trailing_asterisk_as_a_prefix_marker() {
        assert_eq!(
            parse_search_terms("silnik skladowania"),
            vec![term("silnik", false), term("skladowania", false)]
        );
        let prefixed = "\u{141}\u{d3}DZ* migracj*";
        assert_eq!(
            parse_search_terms(prefixed),
            vec![term("lodz", true), term("migracj", true)]
        );
        assert_eq!(parse_search_terms("*"), vec![]);
        assert_eq!(parse_search_terms("a*b"), vec![term("a", true), term("b", false)]);
    }
}