pub struct Tokenizer;
impl Tokenizer {
pub fn tokenize(input: &str) -> Vec<String> {
input
.split_whitespace()
.map(|s| s.to_string())
.collect()
}
pub fn tokenize_by(input: &str, delimiter: char) -> Vec<String> {
input
.split(delimiter)
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect()
}
}