udled/
tokenizer.rs

1use alloc::{format, string::ToString};
2
3use crate::{
4    buffer::Buffer, error::Error, item::Item, reader::Reader, span::Span, AsBytes, AsChar, Either,
5};
6
7pub trait Tokenizer<'input, B: Buffer<'input>> {
8    type Token;
9
10    fn to_token(&self, reader: &mut Reader<'_, 'input, B>) -> Result<Self::Token, Error>;
11
12    fn eat(&self, reader: &mut Reader<'_, 'input, B>) -> Result<(), Error> {
13        let _ = self.to_token(reader)?;
14        Ok(())
15    }
16
17    fn peek(&self, reader: &mut Reader<'_, 'input, B>) -> bool {
18        self.eat(reader).is_ok()
19    }
20}
21
22impl<'a, 'input, B, T> Tokenizer<'input, B> for &'a T
23where
24    B: Buffer<'input>,
25    T: Tokenizer<'input, B>,
26{
27    type Token = T::Token;
28
29    fn eat(&self, reader: &mut Reader<'_, 'input, B>) -> Result<(), Error> {
30        (**self).eat(reader)
31    }
32
33    fn peek(&self, reader: &mut Reader<'_, 'input, B>) -> bool {
34        (**self).peek(reader)
35    }
36
37    fn to_token(&self, reader: &mut Reader<'_, 'input, B>) -> Result<Self::Token, Error> {
38        (**self).to_token(reader)
39    }
40}
41
42/// Match a literal char
43impl<'input, S> Tokenizer<'input, S> for char
44where
45    S: Buffer<'input>,
46    S::Item: AsChar,
47{
48    type Token = Item<char>;
49    fn to_token(&self, reader: &mut Reader<'_, 'input, S>) -> Result<Self::Token, Error> {
50        let next = reader.parse(Char)?;
51        if &next.value == self {
52            Ok(next)
53        } else {
54            Err(reader.error(format!("{}", self)))
55        }
56    }
57}
58
59/// Match a literal string
60impl<'lit, 'input, B> Tokenizer<'lit, B> for &'input str
61where
62    B: Buffer<'lit>,
63    B::Item: AsChar,
64    B::Source: AsBytes<'lit>,
65{
66    type Token = Item<&'lit str>;
67    fn to_token(&self, reader: &mut Reader<'_, 'lit, B>) -> Result<Self::Token, Error> {
68        let tokens = self.chars();
69
70        let start = reader.position();
71
72        for token in tokens {
73            let Some(next) = reader.read()?.as_char() else {
74                return Err(reader.error(self.to_string()));
75            };
76            if token != next {
77                return Err(reader.error(self.to_string()));
78            }
79        }
80
81        if start == reader.position() {
82            return Err(reader.error(self.to_string()));
83        }
84
85        let span = Span {
86            start,
87            end: reader.position(),
88        };
89
90        let string = reader.buffer().source().as_bytes();
91        let string = unsafe { core::str::from_utf8_unchecked(string) };
92
93        Ok(Item {
94            value: span.slice(string).unwrap(),
95            span,
96        })
97    }
98
99    fn peek(&self, reader: &mut Reader<'_, 'lit, B>) -> bool {
100        let tokens = self.chars();
101        for (idx, next) in tokens.enumerate() {
102            if Some(next) == reader.peek_chn(idx).and_then(|m| m.as_char()) {
103                continue;
104            }
105            return false;
106        }
107
108        true
109    }
110}
111
112/// Match a char
113#[derive(Debug, Clone, Copy, Default)]
114pub struct Char;
115
116impl<'input, S> Tokenizer<'input, S> for Char
117where
118    S: Buffer<'input>,
119    S::Item: AsChar,
120{
121    type Token = Item<char>;
122
123    fn to_token(&self, reader: &mut Reader<'_, 'input, S>) -> Result<Self::Token, Error> {
124        let start = reader.position();
125        match reader.read()?.as_char() {
126            Some(ret) => Ok(Item {
127                span: Span::new(start, start + ret.len_utf8()),
128                value: ret,
129            }),
130            None => Err(reader.error("char")),
131        }
132    }
133}
134
135impl<'input, B> Tokenizer<'input, B> for core::ops::Range<char>
136where
137    B: Buffer<'input>,
138    B::Item: AsChar,
139{
140    type Token = Item<char>;
141
142    fn to_token<'a>(&self, reader: &mut Reader<'_, 'input, B>) -> Result<Self::Token, Error> {
143        let char = reader.parse(Char)?;
144
145        if !self.contains(&char.value) {
146            return Err(reader.error(format!("Expected char in range: {:?}", self)));
147        }
148        Ok(char)
149    }
150}
151
152impl<'input, B> Tokenizer<'input, B> for core::ops::RangeInclusive<char>
153where
154    B: Buffer<'input>,
155    B::Item: AsChar,
156{
157    type Token = Item<char>;
158
159    fn to_token<'a>(&self, reader: &mut Reader<'_, 'input, B>) -> Result<Self::Token, Error> {
160        let char = reader.parse(Char)?;
161
162        if !self.contains(&char.value) {
163            return Err(reader.error(format!("Expected char in range: {:?}", self)));
164        }
165
166        Ok(char)
167    }
168}
169
170impl<'input, L, R, B> Tokenizer<'input, B> for Either<L, R>
171where
172    L: Tokenizer<'input, B>,
173    R: Tokenizer<'input, B>,
174    B: Buffer<'input>,
175{
176    type Token = Either<L::Token, R::Token>;
177
178    fn to_token(&self, reader: &mut Reader<'_, 'input, B>) -> Result<Self::Token, Error> {
179        match self {
180            Self::Left(e) => Ok(Either::Left(e.to_token(reader)?)),
181            Self::Right(e) => Ok(Either::Right(e.to_token(reader)?)),
182        }
183    }
184
185    fn eat(&self, reader: &mut Reader<'_, 'input, B>) -> Result<(), Error> {
186        match self {
187            Self::Left(e) => Ok(e.eat(reader)?),
188            Self::Right(e) => Ok(e.eat(reader)?),
189        }
190    }
191
192    fn peek(&self, reader: &mut Reader<'_, 'input, B>) -> bool {
193        match self {
194            Self::Left(e) => e.peek(reader),
195            Self::Right(e) => e.peek(reader),
196        }
197    }
198}
199
200/// Matches end of feed
201#[derive(Debug, Clone, Copy)]
202pub struct EOF;
203
204impl<'input, S> Tokenizer<'input, S> for EOF
205where
206    S: Buffer<'input>,
207{
208    type Token = usize;
209
210    fn to_token(&self, reader: &mut Reader<'_, 'input, S>) -> Result<Self::Token, Error> {
211        if reader.peek_ch().is_some() {
212            return Err(reader.error("EOF"));
213        }
214        Ok(reader.position())
215    }
216}
217
218macro_rules! tuples {
219    ($first: ident) => {
220        impl<'input, $first, B> Tokenizer<'input, B> for ($first,)
221        where
222            B: Buffer<'input>,
223            $first: Tokenizer<'input, B>,
224        {
225            type Token = ($first::Token, );
226
227            fn to_token(&self, reader: &mut Reader<'_, 'input, B>) -> Result<Self::Token, Error> {
228                Ok((self.0.to_token(reader)?,))
229            }
230
231            fn peek(&self, reader: &mut Reader<'_, 'input, B>) -> bool {
232                self.0.peek(reader)
233            }
234
235            fn eat(&self, reader: &mut Reader<'_, 'input, B>) -> Result<(), Error> {
236                self.0.eat(reader)
237            }
238        }
239    };
240    ($first: ident, $($rest:ident),+) => {
241
242        tuples!($($rest),+);
243
244        #[allow(non_snake_case)]
245        impl<'input, $first, $($rest),+, B> Tokenizer<'input, B> for ($first, $($rest),+)
246        where
247            B: Buffer<'input>,
248            $first: Tokenizer<'input, B>,
249            $(
250                $rest: Tokenizer<'input, B>
251            ),+
252        {
253
254            type Token = ($first::Token, $($rest::Token),+);
255
256            fn to_token(&self, reader: &mut Reader<'_, 'input, B>) -> Result<Self::Token, Error> {
257                let ($first, $($rest),+) = self;
258                Ok((
259                    reader.parse($first)?,
260                    $(
261                        reader.parse($rest)?
262                    ),+
263                ))
264            }
265
266            fn peek(&self, reader: &mut Reader<'_, 'input, B>) -> bool {
267                self.0.peek(reader)
268            }
269
270            fn eat(&self, reader: &mut Reader<'_, 'input, B>) -> Result<(), Error> {
271                let ($first, $($rest),+) = self;
272                reader.eat($first)?;
273                $(
274                    reader.eat($rest)?;
275                )+
276                Ok(())
277            }
278        }
279    };
280}
281
282tuples!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12);
udled/tokenizer.rs

udled/
tokenizer.rs
