1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

pub mod map;
pub mod object;

pub type LineId = usize;

pub type LcsToken = String;
pub type LcsSeq = Vec<LcsToken>;
pub type LcsDelimiters = Vec<char>;

/// # Examples
///
/// ```
/// let tokens : Vec<_> = spell::tokenize("a,b c", &[' ', ',']).collect();
///
/// assert_eq!(vec!["a", "b", "c"], tokens);
/// ```
pub fn tokenize<'a>(input: &'a str, delimiters: &'a [char]) -> impl Iterator<Item = &'a str> {
    input
        .trim()
        .split(&delimiters[..])
        .filter(|s| !s.is_empty())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn tokenization() {
        let input = "Command Failed on: node-127,node-234";
        let tokenized: Vec<_> = tokenize(input, &[' ']).collect();
        let expected = vec!["Command", "Failed", "on:", "node-127,node-234"];
        assert_eq!(tokenized, expected);
    }

    #[test]
    fn tokenization_with_multiple_delimiters() {
        let input = "Command Failed on: node-127,node-234";
        let tokenized: Vec<_> = tokenize(input, &[' ', ',', ':']).collect();
        let expected = vec!["Command", "Failed", "on", "node-127", "node-234"];
        assert_eq!(tokenized, expected);
    }

    #[test]
    fn tokenization_with_no_delimiters() {
        let input = "Command Failed on: node-127,node-234";
        let tokenized: Vec<_> = tokenize(input, &[]).collect();
        assert_eq!(tokenized, [input]);
    }
}