tokenizer 0.1.2

Thai text tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11

//! tokenizer implementation, currently target for Thai language
//! 
//! It re-export two main module in root module.
//! - `en` - A space based tokenizer.
//! - `th` - A dictionary based tokenizer.
mod dict;
mod tokenizer;

pub use self::tokenizer::Tokenizer;
pub use self::tokenizer::en;
pub use self::tokenizer::th;