natural 0.5.0

Pure rust library for natural language processing.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18


pub fn tokenize(text: &str) -> Vec<&str> {
  text.split(Splitter::is_match)
      .filter(|s| !s.is_empty())
      .collect()
}

struct Splitter;

impl Splitter {
    fn is_match(c: char) -> bool {
        match c {
            ' ' | ',' | '.' | '!' | '?' | ';' | '\'' |  '"'
            | ':' | '\t' | '\n' | '(' | ')' | '-' => true,
            _ => false
        }
    }
}