1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
use plex::lexer; #[derive(Debug)] pub enum Token { Whitespace, Str(String), GroupStart, GroupEnd, And, Or, Not, Equal, EqualCI, Greater, Less, Wildcard, Regex, } lexer! { fn take_token(tok: 'a) -> Token; r#"[ \t\r\n]"# => Token::Whitespace, r#"`[^`]*`"# => Token::Str(tok[1..tok.len() - 1].into()), r#"\("# => Token::GroupStart, r#"\)"# => Token::GroupEnd, r#"\&"# => Token::And, r#"\|"# => Token::Or, r#"\!"# => Token::Not, r#"="# => Token::Equal, r#"\~"# => Token::EqualCI, r#">"# => Token::Greater, r#"<"# => Token::Less, r#"\*"# => Token::Wildcard, r#"\$"# => Token::Regex, "." => Token::Whitespace } pub struct Lexer<'a> { original: &'a str, remaining: &'a str, } impl<'a> Lexer<'a> { pub fn new(s: &'a str) -> Lexer<'a> { Lexer { original: s, remaining: s, } } } #[derive(Debug, Clone, Copy)] pub struct Span { pub lo: usize, pub hi: usize, } impl<'a> Iterator for Lexer<'a> { type Item = (Token, Span); fn next(&mut self) -> Option<(Token, Span)> { loop { let (tok, span) = if let Some((tok, new_remaining)) = take_token(self.remaining) { let lo = self.original.len() - self.remaining.len(); let hi = self.original.len() - new_remaining.len(); self.remaining = new_remaining; (tok, Span { lo, hi }) } else { return None; }; match tok { Token::Whitespace => { continue; } tok => { return Some((tok, span)); } } } } }