udled_tokenizers/
helpers.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
use alloc::{vec, vec::Vec};
use udled::{Error, Item, Reader, Span, Tokenizer};

/// Match a list of T's separated by P's.
/// Possible to allow trailing P's
#[derive(Debug, Clone, Copy, Default)]
pub struct Punctuated<T, P> {
    item: T,
    punct: P,
    trailing: bool,
}

impl<T, P> Punctuated<T, P> {
    pub const fn new(item: T, punct: P) -> Punctuated<T, P> {
        Punctuated {
            item,
            punct,
            trailing: false,
        }
    }

    pub const fn with_trailing(mut self, trailing: bool) -> Punctuated<T, P> {
        self.trailing = trailing;
        self
    }
}

impl<T, P> Tokenizer for Punctuated<T, P>
where
    T: Tokenizer,
    P: Tokenizer,
{
    type Token<'a> = Item<Vec<T::Token<'a>>>;

    fn to_token<'a>(&self, reader: &mut Reader<'_, 'a>) -> Result<Self::Token<'a>, Error> {
        let start = reader.position();
        let item = reader.parse(&self.item)?;

        let mut output = vec![item];
        loop {
            if reader.eof() || !reader.peek(&self.punct)? {
                break;
            }

            reader.eat(&self.punct)?;

            if self.trailing && (reader.eof() || !reader.peek(&self.item)?) {
                break;
            }

            let item = reader.parse(&self.item)?;
            output.push(item);
        }

        let end = reader.position();

        Ok(Item::new(output, Span::new(start, end)))
    }

    fn eat(&self, reader: &mut Reader<'_, '_>) -> Result<(), Error> {
        reader.eat(&self.item)?;

        loop {
            if reader.eof() || !reader.peek(&self.punct)? {
                break;
            }

            reader.eat(&self.punct)?;

            if self.trailing && (reader.eof() || !reader.peek(&self.item)?) {
                break;
            }

            reader.eat(&self.item)?;
        }

        Ok(())
    }

    fn peek(&self, reader: &mut Reader<'_, '_>) -> Result<bool, Error> {
        reader.peek(&self.item)
    }
}

#[cfg(test)]
mod test {
    use udled::{Input, Lex};

    use crate::Ident;

    use super::*;

    #[test]
    fn punctuated() {
        let mut input = Input::new("ident,identto,");

        let ret = input
            .parse(Punctuated::new(Ident, ',').with_trailing(true))
            .unwrap();

        assert_eq!(
            ret.value,
            vec![
                Lex::new("ident", Span::default()),
                Lex::new("identto", Span::default())
            ]
        )
    }
}