udled_tokenizers/
helpers.rs

1use alloc::{vec, vec::Vec};
2use udled::{Error, Item, Reader, Span, Tokenizer};
3
4/// Match a list of T's separated by P's.
5/// Possible to allow trailing P's
6#[derive(Debug, Clone, Copy, Default)]
7pub struct Punctuated<T, P> {
8    item: T,
9    punct: P,
10    trailing: bool,
11}
12
13impl<T, P> Punctuated<T, P> {
14    pub const fn new(item: T, punct: P) -> Punctuated<T, P> {
15        Punctuated {
16            item,
17            punct,
18            trailing: false,
19        }
20    }
21
22    pub const fn with_trailing(mut self, trailing: bool) -> Punctuated<T, P> {
23        self.trailing = trailing;
24        self
25    }
26}
27
28impl<T, P> Tokenizer for Punctuated<T, P>
29where
30    T: Tokenizer,
31    P: Tokenizer,
32{
33    type Token<'a> = Item<Vec<T::Token<'a>>>;
34
35    fn to_token<'a>(&self, reader: &mut Reader<'_, 'a>) -> Result<Self::Token<'a>, Error> {
36        let start = reader.position();
37        let item = reader.parse(&self.item)?;
38
39        let mut output = vec![item];
40        loop {
41            if reader.eof() || !reader.peek(&self.punct)? {
42                break;
43            }
44
45            reader.eat(&self.punct)?;
46
47            if self.trailing && (reader.eof() || !reader.peek(&self.item)?) {
48                break;
49            }
50
51            let item = reader.parse(&self.item)?;
52            output.push(item);
53        }
54
55        let end = reader.position();
56
57        Ok(Item::new(output, Span::new(start, end)))
58    }
59
60    fn eat(&self, reader: &mut Reader<'_, '_>) -> Result<(), Error> {
61        reader.eat(&self.item)?;
62
63        loop {
64            if reader.eof() || !reader.peek(&self.punct)? {
65                break;
66            }
67
68            reader.eat(&self.punct)?;
69
70            if self.trailing && (reader.eof() || !reader.peek(&self.item)?) {
71                break;
72            }
73
74            reader.eat(&self.item)?;
75        }
76
77        Ok(())
78    }
79
80    fn peek(&self, reader: &mut Reader<'_, '_>) -> Result<bool, Error> {
81        reader.peek(&self.item)
82    }
83}
84
85#[cfg(test)]
86mod test {
87    use udled::{Input, Lex};
88
89    use crate::Ident;
90
91    use super::*;
92
93    #[test]
94    fn punctuated() {
95        let mut input = Input::new("ident,identto,");
96
97        let ret = input
98            .parse(Punctuated::new(Ident, ',').with_trailing(true))
99            .unwrap();
100
101        assert_eq!(
102            ret.value,
103            vec![
104                Lex::new("ident", Span::default()),
105                Lex::new("identto", Span::default())
106            ]
107        )
108    }
109}
udled_tokenizers/helpers.rs

udled_tokenizers/
helpers.rs
