use rust_stemmers::{Algorithm, Stemmer};
use std::sync::Arc;
use tantivy::tokenizer::{Token, TokenStream, Tokenizer};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenType {
Code,
Prose,
Mixed,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TypedToken {
pub text: String,
pub token_type: TokenType,
pub offset_from: usize,
pub offset_to: usize,
}
#[derive(Clone)]
pub struct CodeTokenizer {
stemmer: Arc<Stemmer>,
}
impl Default for CodeTokenizer {
fn default() -> Self {
Self::new()
}
}
impl CodeTokenizer {
pub fn new() -> Self {
Self {
stemmer: Arc::new(Stemmer::create(Algorithm::English)),
}
}
pub fn tokenize_typed(&self, text: &str) -> Vec<TypedToken> {
let mut tokens = Vec::new();
let mut word_start: Option<usize> = None;
for (idx, ch) in text.char_indices() {
let is_delimiter = ch.is_whitespace() || ch == '_' || is_separator(ch);
if is_delimiter {
if let Some(start) = word_start.take() {
self.push_word_tokens(&mut tokens, &text[start..idx], start);
}
} else if word_start.is_none() {
word_start = Some(idx);
}
}
if let Some(start) = word_start {
self.push_word_tokens(&mut tokens, &text[start..], start);
}
tokens
}
pub fn tokenize(&self, text: &str) -> Vec<String> {
self.tokenize_typed(text)
.into_iter()
.map(|t| t.text)
.collect()
}
fn push_word_tokens(&self, tokens: &mut Vec<TypedToken>, word: &str, word_start: usize) {
let subtokens = split_camel_case(word);
let mut subtoken_pos = word_start;
for subtoken in subtokens {
if subtoken.is_empty() {
continue;
}
let subtoken_end = subtoken_pos + subtoken.len();
let token_type = detect_token_type(&subtoken);
let normalized = match token_type {
TokenType::Code => {
if is_acronym(&subtoken) {
subtoken.clone()
} else {
subtoken.to_lowercase()
}
}
TokenType::Prose => {
let lower = subtoken.to_lowercase();
self.stemmer.stem(&lower).to_string()
}
TokenType::Mixed => subtoken.to_lowercase(),
};
tokens.push(TypedToken {
text: normalized,
token_type,
offset_from: subtoken_pos,
offset_to: subtoken_end,
});
subtoken_pos = subtoken_end;
}
}
}
fn split_camel_case(word: &str) -> Vec<String> {
let mut result = Vec::new();
let mut current = String::new();
let chars: Vec<char> = word.chars().collect();
for i in 0..chars.len() {
let c = chars[i];
if current.is_empty() {
current.push(c);
continue;
}
let prev_upper = chars
.get(i.saturating_sub(1))
.is_some_and(|c| c.is_uppercase());
let curr_upper = c.is_uppercase();
let next_lower = chars.get(i + 1).is_some_and(|c| c.is_lowercase());
let should_split = if !prev_upper && curr_upper {
true
} else if prev_upper && curr_upper && next_lower {
true
} else {
false
};
if should_split {
if !current.is_empty() {
result.push(current);
}
current = String::new();
}
current.push(c);
}
if !current.is_empty() {
result.push(current);
}
result
}
fn is_acronym(s: &str) -> bool {
let chars: Vec<char> = s.chars().collect();
chars.len() >= 2 && chars.iter().all(|c| c.is_uppercase())
}
fn is_separator(c: char) -> bool {
matches!(
c,
'.' | ','
| ';'
| ':'
| '('
| ')'
| '['
| ']'
| '{'
| '}'
| '<'
| '>'
| '"'
| '\''
| '&'
| '|'
| '+'
| '-'
| '*'
| '/'
| '%'
| '='
| '!'
| '?'
)
}
fn detect_token_type(token: &str) -> TokenType {
let has_upper = token.chars().any(|c| c.is_uppercase());
let has_lower = token.chars().any(|c| c.is_lowercase());
let all_alpha = token.chars().all(|c| c.is_alphabetic());
if !all_alpha {
return TokenType::Code;
}
if is_acronym(token) {
return TokenType::Code;
}
if has_upper && has_lower {
TokenType::Code
} else if has_lower && !has_upper {
TokenType::Prose
} else {
TokenType::Mixed
}
}
impl Tokenizer for CodeTokenizer {
type TokenStream<'a> = CodeTokenStream;
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
let tokens = self.tokenize_typed(text);
CodeTokenStream::new(tokens)
}
}
pub struct CodeTokenStream {
tokens: Vec<TypedToken>,
index: usize,
token: Token,
}
impl CodeTokenStream {
fn new(tokens: Vec<TypedToken>) -> Self {
Self {
tokens,
index: 0,
token: Token::default(),
}
}
}
impl TokenStream for CodeTokenStream {
fn advance(&mut self) -> bool {
if self.index >= self.tokens.len() {
return false;
}
let typed = &self.tokens[self.index];
self.token.text.clear();
self.token.text.push_str(&typed.text);
self.token.offset_from = typed.offset_from;
self.token.offset_to = typed.offset_to;
self.token.position = self.index;
self.token.position_length = 1;
self.index += 1;
true
}
fn token(&self) -> &Token {
&self.token
}
fn token_mut(&mut self) -> &mut Token {
&mut self.token
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_camel_case_splitting() {
let tokenizer = CodeTokenizer::new();
let tokens = tokenizer.tokenize("getUserById");
assert_eq!(tokens, vec!["get", "user", "by", "id"]);
let tokens = tokenizer.tokenize("parseUserInput");
assert_eq!(tokens, vec!["pars", "user", "input"]); }
#[test]
fn test_snake_case_splitting() {
let tokenizer = CodeTokenizer::new();
let tokens = tokenizer.tokenize("get_user_by_id");
assert_eq!(tokens, vec!["get", "user", "by", "id"]);
let tokens = tokenizer.tokenize("parse_user_input");
assert_eq!(tokens, vec!["pars", "user", "input"]);
}
#[test]
fn test_acronym_preservation() {
let tokenizer = CodeTokenizer::new();
let tokens = tokenizer.tokenize("HTTPResponse");
assert_eq!(tokens, vec!["HTTP", "response"]);
let tokens = tokenizer.tokenize("parseJSONData");
assert_eq!(tokens, vec!["pars", "JSON", "data"]);
let tokens = tokenizer.tokenize("getAPIKey");
assert_eq!(tokens, vec!["get", "API", "key"]);
}
#[test]
fn test_prose_normalization() {
let tokenizer = CodeTokenizer::new();
let tokens = tokenizer.tokenize("running quickly");
assert_eq!(tokens, vec!["run", "quick"]);
let tokens = tokenizer.tokenize("the users are connecting");
assert_eq!(tokens, vec!["the", "user", "are", "connect"]);
}
#[test]
fn test_operator_preservation() {
let tokenizer = CodeTokenizer::new();
let tokens = tokenizer.tokenize("x && y");
assert_eq!(tokens, vec!["x", "y"]); }
#[test]
fn test_mixed_identifiers() {
let tokenizer = CodeTokenizer::new();
let tokens = tokenizer.tokenize("parseJSONData");
assert_eq!(tokens, vec!["pars", "JSON", "data"]);
let tokens = tokenizer.tokenize("XMLHttpRequest");
assert_eq!(tokens, vec!["XML", "http", "request"]);
}
#[test]
fn test_typed_tokens() {
let tokenizer = CodeTokenizer::new();
let typed = tokenizer.tokenize_typed("HTTPResponse");
assert_eq!(typed.len(), 2);
assert_eq!(typed[0].text, "HTTP");
assert_eq!(typed[0].token_type, TokenType::Code);
assert_eq!(typed[1].text, "response");
assert_eq!(typed[1].token_type, TokenType::Code);
}
#[test]
fn test_tantivy_tokenizer_trait() {
use tantivy::tokenizer::Tokenizer;
let mut tokenizer = CodeTokenizer::new();
let mut stream = tokenizer.token_stream("getUserById");
let mut tokens = Vec::new();
while stream.advance() {
tokens.push(stream.token().text.clone());
}
assert_eq!(tokens, vec!["get", "user", "by", "id"]);
}
#[test]
fn test_unicode_whitespace_does_not_panic() {
let tokenizer = CodeTokenizer::new();
let text = "result\u{200a}value with\u{2003}unicode separators";
let tokens = tokenizer.tokenize(text);
assert!(!tokens.is_empty());
assert!(tokens.contains(&"result".to_string()));
assert!(tokens.contains(&"valu".to_string()));
}
}