regex_tokenizer/
lib.rs

1//! A regex-based tokenizer.
2//!
3//! ```
4//! use regex_tokenizer::tokenizer;
5//!
6//! tokenizer! {
7//!     Test
8//!
9//!     r"[a-zA-Z]\w*" => Identifier
10//!     r"\d+" => Number
11//!     r"\s+" => _
12//! }
13//!
14//! let tokenizer = Test::new();
15//! let query = "Identifier  11";
16//!
17//! let mut tokens = tokenizer.tokenize(query);
18//! let token = tokens.next().unwrap();
19//!
20//! assert_eq!(token.position, 0);
21//! assert_eq!(token.value, "Identifier");
22//! assert_eq!(token.type_, Test_types::Identifier);
23//!
24//! let token = tokens.next().unwrap();
25//!
26//! assert_eq!(token.position, 12);
27//! assert_eq!(token.value, "11");
28//! assert_eq!(token.type_, Test_types::Number);
29//! ```
30
31mod tokenization;
32
33pub use regex_tokenizer_impl::*;
34
35pub use tokenization::matcher::BuildableMatcher;
36pub use tokenization::matcher::Matcher;
37pub use tokenization::Tokenizer;