1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

pub mod map;
pub mod object;

pub type LineId = usize;

pub type LcsToken = String;
pub type LcsSeq = Vec<LcsToken>;

pub fn tokenize(input: &str) -> impl Iterator<Item = &str> {
    input.trim().split_whitespace().filter(|s| !s.is_empty())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn tokenization() {
        let input = "Command Failed on: node-127";
        let tokenized: Vec<_> = tokenize(input).collect();
        let expected = vec!["Command", "Failed", "on:", "node-127"];
        assert_eq!(tokenized, expected);
    }
}