tokenate/
lib.rs

1//! An InnerTokenizer is intended to be a child to your actual tokenizer, providing the string
2//! and indexing utility, so you can worry about consuming characters.
3//! The main methods you will need are "start_token" "next" "peek" and "unpeek", then finally "token_res"
4//! which will build the resulting Token with Line Number and Value
5//! The other methods like "follow","follow_fn" and "take_while" are helpers to make getting
6//! strings and numbers more easily
7//! The Token::value is generic allowing you to choose define how tokens for your language look.
8//!
9//! ```rust
10//! use tokenate::*;
11//!
12//! #[derive(Clone, Debug, PartialEq)]
13//! pub enum TKind {
14//!     GreaterEqual,
15//!     Ident(String),
16//!     Num(String),
17//! }
18//!
19//! struct TestTok<'a> {
20//!     tk: InnerTokenizer<'a>,
21//! }
22//!
23//! fn num_digit(c: char) -> bool {
24//!     c >= '0' && c <= '9'
25//! }
26//!
27//! impl<'a> TestTok<'a> {
28//!     pub fn new(s: &'a str) -> Self {
29//!         TestTok {
30//!             tk: InnerTokenizer::new(s),
31//!         }
32//!     }
33//!     pub fn next(&mut self) -> TokenRes<'a, TKind> {
34//!         self.tk.skip(char::is_whitespace);
35//!         self.tk.start_token();
36//!         match self.tk.peek_char() {
37//!             Some(c) if num_digit(c) => self.tk.take_while(num_digit, |s| Ok(TKind::Num(s.to_string()))),
38//!             Some('>') => self.tk.follow('=', TKind::GreaterEqual),
39//!             Some(c) if char::is_alphabetic(c) => self
40//!                 .tk
41//!                 .take_while(char::is_alphabetic, |s| Ok(TKind::Ident(s.to_string()))),
42//!
43//!             _ => self.tk.expected("A legal token".to_string()),
44//!         }
45//!     }
46//! }
47//!
48//!
49//! fn main() {
50//!     let s = "a >= 54";
51//!     let mut tt = TestTok::new(s);
52//!     assert_eq!(
53//!         tt.next().unwrap().unwrap().value,
54//!         TKind::Ident("a".to_string())
55//!     );
56//!     assert_eq!(tt.next().unwrap().unwrap().value, TKind::GreaterEqual,);
57//!     assert_eq!(
58//!         tt.next().unwrap().unwrap().value,
59//!         TKind::Num("54".to_string())
60//!     );
61//! }
62//! ```
63
64mod charbool;
65mod err;
66mod inner_token;
67#[cfg(test)]
68mod test;
69
70pub use charbool::CharBool;
71pub use err::TErr;
72pub use inner_token::*;
tokenate/lib.rs

tokenate/
lib.rs
