use std::collections::HashSet;
pub(crate) fn tokenize_ascii_alphanumeric(text: &str) -> Vec<&str> {
let mut out: Vec<&str> = Vec::with_capacity(text.len() / 6 + 1);
let mut start: Option<usize> = None;
for (i, c) in text.char_indices() {
if c.is_alphanumeric() {
if start.is_none() {
start = Some(i);
}
} else if let Some(s) = start.take() {
out.push(&text[s..i]);
}
}
if let Some(s) = start {
out.push(&text[s..]);
}
out
}
pub(crate) fn tokenize_whitespace(text: &str) -> Vec<&str> {
text.split_whitespace().collect()
}
#[cfg(test)]
pub(crate) fn filter_token_length(tokens: Vec<&str>, min: u16, max: u16) -> Vec<&str> {
tokens
.into_iter()
.filter(|token| {
let len = token.chars().count();
let too_short = min > 0 && len < min as usize;
let too_long = max > 0 && len > max as usize;
!too_short && !too_long
})
.collect()
}
#[allow(dead_code)]
pub(crate) fn filter_stopwords<'a>(
tokens: Vec<&'a str>,
stopwords: &'_ HashSet<Vec<u8>>,
) -> Vec<&'a str> {
tokens
.into_iter()
.filter(|token| !stopwords.contains(token.as_bytes()))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn whitespace_splits_on_any_whitespace() {
let words = tokenize_whitespace(" one\ttwo\nthree ");
assert_eq!(words, vec!["one", "two", "three"]);
}
#[test]
fn whitespace_preserves_punctuation_in_tokens() {
let words = tokenize_whitespace("hello, world!");
assert_eq!(words, vec!["hello,", "world!"]);
}
#[test]
fn length_filter_min_only() {
let tokens = vec!["a", "ab", "abc", "abcd"];
assert_eq!(filter_token_length(tokens, 2, 0), vec!["ab", "abc", "abcd"]);
}
#[test]
fn length_filter_max_only() {
let tokens = vec!["a", "ab", "abc", "abcd"];
assert_eq!(filter_token_length(tokens, 0, 3), vec!["a", "ab", "abc"]);
}
#[test]
fn length_filter_both_bounds() {
let tokens = vec!["a", "ab", "abc", "abcd", "abcde"];
assert_eq!(filter_token_length(tokens, 2, 4), vec!["ab", "abc", "abcd"]);
}
#[test]
fn length_filter_zero_zero_passes_all() {
let tokens = vec!["a", "ab", "abc"];
assert_eq!(filter_token_length(tokens, 0, 0), vec!["a", "ab", "abc"]);
}
#[test]
fn length_filter_uses_char_count_not_byte_count() {
let tokens = vec!["café"];
assert_eq!(filter_token_length(tokens.clone(), 0, 4), tokens);
assert_eq!(filter_token_length(tokens, 5, 0), Vec::<&str>::new());
}
fn stopword_set(words: &[&str]) -> HashSet<Vec<u8>> {
words.iter().map(|w| w.as_bytes().to_vec()).collect()
}
#[test]
fn stopword_filter_drops_matches() {
let stopwords = stopword_set(&["the", "a", "of"]);
let tokens = vec!["the", "quick", "fox", "of", "logic"];
assert_eq!(
filter_stopwords(tokens, &stopwords),
vec!["quick", "fox", "logic"]
);
}
#[test]
fn stopword_filter_empty_set_passes_all() {
let stopwords = HashSet::new();
let tokens = vec!["the", "quick", "fox"];
assert_eq!(
filter_stopwords(tokens.clone(), &stopwords),
vec!["the", "quick", "fox"]
);
}
#[test]
fn stopword_filter_is_case_sensitive() {
let stopwords = stopword_set(&["the"]);
let tokens = vec!["The", "the"];
assert_eq!(filter_stopwords(tokens, &stopwords), vec!["The"]);
}
}