1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
use std::marker::PhantomData;
use crate::span::Spanned;
pub trait TokenExt: Clone + Copy + PartialEq {
fn matches_class(&self, other: &Self) -> bool;
}
pub trait TokenParse<'a> {
type Token: TokenExt + 'a;
fn next(&mut self) -> Option<Spanned<Self::Token>>;
}
pub struct TokenizerWrap<'a, T: TokenParse<'a>> {
inner: T,
peeked: Option<Spanned<T::Token>>,
tokens_to_ignore: Vec<T::Token>,
_lifetime: PhantomData<&'a ()>,
}
impl<'a, T: TokenParse<'a>> TokenizerWrap<'a, T> {
pub fn new(inner: T, tokens_to_ignore: impl IntoIterator<Item = T::Token>) -> Self {
Self {
inner,
tokens_to_ignore: tokens_to_ignore.into_iter().collect(),
peeked: None,
_lifetime: PhantomData,
}
}
pub fn next(&mut self) -> Option<Spanned<T::Token>> {
if let Some(peeked) = self.peeked.take() {
Some(peeked)
} else {
loop {
let next = self.inner.next()?;
if self.tokens_to_ignore.iter().all(|x| !x.matches_class(&*next)) {
break Some(next);
}
}
}
}
pub fn peek(&mut self) -> Option<&Spanned<T::Token>> {
if self.peeked.is_none() {
self.peeked = self.next();
}
self.peeked.as_ref()
}
pub fn eat(&mut self, token: T::Token) -> Option<Spanned<T::Token>> {
let next = self.next()?;
if next.matches_class(&token) {
Some(next)
} else {
self.peeked = Some(next);
None
}
}
pub fn eat_any(&mut self, tokens: &[T::Token]) -> Option<Spanned<T::Token>> {
let next = self.next()?;
for token in tokens {
if next.matches_class(token) {
return Some(next);
}
}
self.peeked = Some(next);
None
}
}