1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

//! A regex-based tokenizer.
//!
//! ```
//! use regex_tokenizer::tokenizer;
//!
//! tokenizer! {
//!     Test
//!
//!     r"[a-zA-Z]\w*" => Identifier
//!     r"\d+" => Number
//!     r"\s+" => _
//! }
//!
//! let tokenizer = Test::new();
//! let query = "Identifier  11";
//!
//! let mut tokens = tokenizer.tokenize(query);
//! let token = tokens.next().unwrap();
//!
//! assert_eq!(token.position, 0);
//! assert_eq!(token.value, "Identifier");
//! assert_eq!(token.type_, Test_types::Identifier);
//!
//! let token = tokens.next().unwrap();
//!
//! assert_eq!(token.position, 12);
//! assert_eq!(token.value, "11");
//! assert_eq!(token.type_, Test_types::Number);
//! ```

mod tokenization;

pub use regex_tokenizer_impl::*;

pub use tokenization::matcher::BuildableMatcher;
pub use tokenization::matcher::Matcher;
pub use tokenization::Tokenizer;