minni 0.1.0

/// Code-aware tokenizer.
use std::collections::HashSet;

lazy_static::lazy_static! {
    static ref STOP_WORDS: HashSet<&'static str> = {
        let mut set = HashSet::new();
        // Keep code terms; only drop common English words.
        for word in &["the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for"] {
            set.insert(*word);
        }
        set
    };
}

pub fn tokenize_code(text: &str) -> Vec<String> {
    let mut tokens = Vec::new();
    let mut current = String::new();

    for ch in text.chars() {
        if ch.is_alphanumeric() || ch == '_' {
            current.push(ch);
        } else if !current.is_empty() {
            // Split camelCase and snake_case.
            add_split_tokens(&current, &mut tokens);
            current.clear();
        }
    }

    if !current.is_empty() {
        add_split_tokens(&current, &mut tokens);
    }

    // Filter stop words.
    tokens.retain(|t| !STOP_WORDS.contains(t.as_str()) && t.len() > 1);

    tokens
}

fn add_split_tokens(word: &str, tokens: &mut Vec<String>) {
    // Split on case changes and underscores.
    let mut current = String::new();
    let chars: Vec<char> = word.chars().collect();

    for i in 0..chars.len() {
        let ch = chars[i];

        if ch == '_' {
            if !current.is_empty() {
                tokens.push(current.clone());
                current.clear();
            }
        } else if i > 0 {
            let prev = chars[i - 1];
            let is_case_boundary =
                // lowercase to uppercase: camelCase
                (prev.is_lowercase() && ch.is_uppercase()) ||
                // uppercase to uppercase followed by lowercase: HTTPRequest -> HTTP Request
                (i + 1 < chars.len() && prev.is_uppercase() && ch.is_uppercase() && chars[i + 1].is_lowercase());

            if is_case_boundary {
                if !current.is_empty() {
                    tokens.push(current.clone());
                    current.clear();
                }
            }
            current.push(ch.to_ascii_lowercase());
        } else {
            current.push(ch.to_ascii_lowercase());
        }
    }

    if !current.is_empty() {
        tokens.push(current);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_camel_case() {
        let tokens = tokenize_code("getUserName");
        assert_eq!(tokens, vec!["get", "user", "name"]);
    }

    #[test]
    fn test_snake_case() {
        let tokens = tokenize_code("get_user_name");
        assert_eq!(tokens, vec!["get", "user", "name"]);
    }

    #[test]
    fn test_mixed() {
        let tokens = tokenize_code("handleHTTPRequest");
        assert_eq!(tokens, vec!["handle", "http", "request"]);
    }
}