compiler_tools/
tokenizer.rs

1use std::marker::PhantomData;
2
3use crate::span::Spanned;
4
5pub trait TokenExt: Clone + Copy + PartialEq {
6    fn matches_class(&self, other: &Self) -> bool;
7}
8
9pub trait TokenParse<'a> {
10    type Token: TokenExt + 'a;
11
12    fn next(&mut self) -> Option<Spanned<Self::Token>>;
13}
14
15pub struct TokenizerWrap<'a, T: TokenParse<'a>> {
16    inner: T,
17    peeked: Option<Spanned<T::Token>>,
18    tokens_to_ignore: Vec<T::Token>,
19    _lifetime: PhantomData<&'a ()>,
20}
21
22impl<'a, T: TokenParse<'a>> TokenizerWrap<'a, T> {
23    pub fn new(inner: T, tokens_to_ignore: impl IntoIterator<Item = T::Token>) -> Self {
24        Self {
25            inner,
26            tokens_to_ignore: tokens_to_ignore.into_iter().collect(),
27            peeked: None,
28            _lifetime: PhantomData,
29        }
30    }
31
32    pub fn next(&mut self) -> Option<Spanned<T::Token>> {
33        if let Some(peeked) = self.peeked.take() {
34            Some(peeked)
35        } else {
36            loop {
37                let next = self.inner.next()?;
38                if self.tokens_to_ignore.iter().all(|x| !x.matches_class(&*next)) {
39                    break Some(next);
40                }
41            }
42        }
43    }
44
45    pub fn peek(&mut self) -> Option<&Spanned<T::Token>> {
46        if self.peeked.is_none() {
47            self.peeked = self.next();
48        }
49        self.peeked.as_ref()
50    }
51
52    pub fn eat(&mut self, token: T::Token) -> Option<Spanned<T::Token>> {
53        let next = self.next()?;
54        if next.matches_class(&token) {
55            Some(next)
56        } else {
57            self.peeked = Some(next);
58            None
59        }
60    }
61
62    pub fn eat_any(&mut self, tokens: &[T::Token]) -> Option<Spanned<T::Token>> {
63        let next = self.next()?;
64        for token in tokens {
65            if next.matches_class(token) {
66                return Some(next);
67            }
68        }
69        self.peeked = Some(next);
70        None
71    }
72}
compiler_tools/tokenizer.rs

compiler_tools/
tokenizer.rs
