1//! # Tools for splitting strings into words 2//! 3//! Tokenizers just provide handy methods to split a string into words, 4//! following certain rules. 5 6mod main; 7 8pub use self::main::split_whitespace_remove_punc;