cooklang 0.18.6

Cooklang parser with opt-in extensions
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

//! [Cursor](crate::lexer::Cursor) iterator adapter for it's use in
//! `Parser(super::parser::Parser`).

pub use crate::lexer::TokenKind;
use crate::{lexer::Cursor, span::Span};

pub struct TokenStream<'i> {
    cursor: Cursor<'i>,
    consumed: usize,
}

impl<'i> TokenStream<'i> {
    pub fn new(input: &'i str) -> Self {
        Self {
            cursor: Cursor::new(input),
            consumed: 0,
        }
    }

    pub fn offset(&mut self, offset: usize) {
        self.consumed += offset;
    }
}

impl Iterator for TokenStream<'_> {
    type Item = Token;

    fn next(&mut self) -> Option<Self::Item> {
        let t = self.cursor.advance_token();
        let start = self.consumed;
        self.consumed += t.len as usize;
        if t.kind == TokenKind::Eof && self.cursor.is_eof() {
            None
        } else {
            Some(Token {
                kind: t.kind,
                span: Span::new(start, self.consumed),
            })
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
}

impl Token {
    pub fn len(&self) -> usize {
        self.span.len()
    }
}

#[cfg(test)]
macro_rules! tokens {
    ($($kind:tt . $len:expr),*) => {{
        let mut v = Vec::new();
        let mut _len = 0;
        $(
            v.push($crate::parser::token_stream::Token { kind: $crate::lexer::T![$kind], span: $crate::span::Span::new(_len, _len + $len) });
            _len += $len;
        )*
        v
    }};
}
#[cfg(test)]
pub(crate) use tokens;

#[cfg(test)]
mod tests {
    use crate::lexer::T;

    use super::*;

    #[test]
    fn tokens_macro() {
        let t = tokens![word.3, ws.1];
        assert_eq!(
            t,
            vec![
                Token {
                    kind: T![word],
                    span: Span::new(0, 3)
                },
                Token {
                    kind: T![ws],
                    span: Span::new(3, 4)
                },
            ]
        );
    }
}