moxy-token 0.1.0

Rust token types/parsing/lexing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

use crate::lex::{Cursor, LexError, Scan};
use crate::span::DelimSpan;
use crate::{Delim, Span, TokenStream, TokenTree};

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Group {
    pub(crate) delim: Delim,
    pub(crate) span: DelimSpan,
    pub(crate) tokens: TokenStream,
}

impl Group {
    #[inline]
    pub fn new(delim: Delim, stream: TokenStream) -> Self {
        Self {
            delim,
            span: DelimSpan::new(Span::call_site(), Span::call_site()),
            tokens: stream,
        }
    }

    #[inline]
    pub fn delim(&self) -> Delim {
        self.delim
    }

    #[inline]
    pub fn span(&self) -> DelimSpan {
        self.span
    }

    #[inline]
    pub fn stream(&self) -> TokenStream {
        self.tokens.clone()
    }

    #[inline]
    pub fn set_span(&mut self, span: DelimSpan) {
        self.span = span;
    }

    #[inline]
    pub fn to_token_tree(&self) -> TokenTree {
        TokenTree::Group(self.clone())
    }

    #[inline]
    pub fn into_token_tree(self) -> TokenTree {
        TokenTree::Group(self)
    }
}

impl std::fmt::Display for Group {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self.delim {
            Delim::None => write!(f, "{}", self.tokens),
            d => write!(f, "{}{}{}", d.open(), self.tokens, d.close()),
        }
    }
}

impl Scan for Group {
    fn scan(cursor: Cursor<'_>) -> Result<(Cursor<'_>, Self), LexError> {
        let ch = cursor.first().ok_or(cursor.error())?;
        let delim = Delim::from_open(ch).ok_or(cursor.error())?;
        let c = cursor.advance(ch.len_utf8());
        let (c, inner) = TokenStream::scan(c)?;
        let close_ch = c
            .first()
            .ok_or_else(|| cursor.error().message(format!("unclosed delimiter '{}'", delim.open())))?;

        let close_delim = Delim::from_close(close_ch).ok_or_else(|| {
            c.error()
                .message(format!("expected '{}', found '{}'", delim.close(), close_ch))
        })?;

        if delim != close_delim {
            return Err(c.error().message(format!(
                "mismatched delimiter: expected '{}', found '{}'",
                delim.close(),
                close_ch,
            )));
        }

        let c = c.advance(close_ch.len_utf8());
        let mut group = Self::new(delim, inner);
        group.set_span(DelimSpan::new(cursor.span(), c.span()));

        Ok((c, group))
    }
}

impl crate::ToTokens for Group {
    fn to_tokens(&self, tokens: &mut crate::TokenStream) {
        tokens.extend_one(crate::TokenTree::Group(self.clone()));
    }
}

impl crate::Spanner for Group {
    fn span(&self) -> Span {
        self.span.span()
    }
}

impl crate::Parse for Group {
    fn parse(stream: &mut crate::parser::ParseStream) -> Result<Self, crate::parser::ParseError> {
        match stream.advance() {
            Some(crate::TokenTree::Group(v)) => Ok(v.clone()),
            _ => Err(crate::lex::LexError::new(stream.span()).message("expected Group").into()),
        }
    }
}

#[cfg(feature = "serde")]
impl serde::Serialize for Group {
    fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>
    where
        S: serde::Serializer,
    {
        use serde::ser::SerializeStruct;

        let mut o = s.serialize_struct("Group", 2)?;
        o.serialize_field("delim", &self.delim)?;
        o.serialize_field("tokens", &self.tokens)?;
        o.end()
    }
}