udled 0.6.2

Tokenizer and parser
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

use alloc::vec::Vec;

use crate::{tokenizers::Peek, Buffer, Error, Reader, Span, Tokenizer, WithSpan};

#[derive(Debug, Clone, Copy)]
pub enum PuntuatedItem<T, P> {
    Item(T),
    Punct(P),
}

impl<T: WithSpan, P: WithSpan> WithSpan for PuntuatedItem<T, P> {
    fn span(&self) -> Span {
        match self {
            PuntuatedItem::Item(item) => item.span(),
            PuntuatedItem::Punct(punct) => punct.span(),
        }
    }
}

#[derive(Debug, Clone)]
pub struct PunctuatedList<T, P> {
    list: Vec<PuntuatedItem<T, P>>,
    span: Span,
}

impl<T, P> PunctuatedList<T, P> {
    pub fn items(&self) -> impl Iterator<Item = &T> {
        self.list.iter().filter_map(|m| match m {
            PuntuatedItem::Item(item) => Some(item),
            _ => None,
        })
    }

    pub fn into_items(self) -> impl Iterator<Item = T> {
        self.list.into_iter().filter_map(|m| match m {
            PuntuatedItem::Item(item) => Some(item),
            _ => None,
        })
    }

    pub fn iter(&self) -> core::slice::Iter<'_, PuntuatedItem<T, P>> {
        self.list.iter()
    }

    pub fn len(&self) -> usize {
        self.list.len()
    }

    pub fn is_empty(&self) -> bool {
        self.list.is_empty()
    }
}

impl<T, P> WithSpan for PunctuatedList<T, P> {
    fn span(&self) -> Span {
        self.span
    }
}

#[derive(Debug, Clone, Copy)]
pub struct Puntuated<T, P> {
    item: T,
    punct: P,
    non_empty: bool,
}

impl<T, P> Puntuated<T, P> {
    pub const fn new(item: T, punct: P) -> Puntuated<T, P> {
        Puntuated {
            item,
            punct,
            non_empty: false,
        }
    }
}

impl<'input, T, P, B> Tokenizer<'input, B> for Puntuated<T, P>
where
    B: Buffer<'input>,
    T: Tokenizer<'input, B>,
    P: Tokenizer<'input, B>,
{
    type Token = PunctuatedList<T::Token, P::Token>;

    fn to_token(&self, reader: &mut Reader<'_, 'input, B>) -> Result<Self::Token, Error> {
        let start = reader.position();
        let mut output = Vec::new();

        if self.non_empty {
            let item = reader.parse(&self.item)?;
            output.push(PuntuatedItem::Item(item));
            if reader.is(Peek((&self.punct, &self.item))) {
                let punct = reader.parse(&self.punct)?;
                output.push(PuntuatedItem::Punct(punct));
            }
        }

        loop {
            if !reader.is(&self.item) {
                break;
            }

            let item = reader.parse(&self.item)?;

            output.push(PuntuatedItem::Item(item));

            if reader.is(Peek((&self.punct, &self.item))) {
                let punct = reader.parse(&self.punct)?;
                output.push(PuntuatedItem::Punct(punct));
            }
        }

        let end = reader.position();

        Ok(PunctuatedList {
            list: output,
            span: Span::new(start, end),
        })
    }
}