moxy-token 0.4.4

Rust token types and lexing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

use crate::lex::{Cursor, LexError, Scan};
use crate::span::DelimSpan;
use crate::{Delim, Span, TokenStream, TokenTree};

/// A delimited sequence of token trees.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Group {
    pub delim: Delim,
    pub span: DelimSpan,
    pub tokens: TokenStream,
}

impl Group {
    #[inline]
    pub fn new(delim: Delim, stream: TokenStream) -> Self {
        let content_span = stream.span();
        let span = if content_span.is_empty() {
            DelimSpan::new(Span::call_site(), Span::call_site())
        } else {
            DelimSpan::new(content_span, content_span)
        };

        Self {
            delim,
            span,
            tokens: stream,
        }
    }

    #[inline]
    pub fn to_token_tree(&self) -> TokenTree {
        TokenTree::Group(self.clone())
    }

    #[inline]
    pub fn into_token_tree(self) -> TokenTree {
        TokenTree::Group(self)
    }
}

impl std::fmt::Display for Group {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self.delim {
            Delim::None => write!(f, "{}", self.tokens),
            d => write!(f, "{}{}{}", d.open(), self.tokens, d.close()),
        }
    }
}

impl Scan for Group {
    fn scan(cursor: Cursor<'_>) -> Result<(Cursor<'_>, Self), LexError> {
        let ch = cursor.first().ok_or(cursor.error())?;
        let delim = Delim::from_open(ch).ok_or(cursor.error())?;
        let c = cursor.advance_by(ch.len_utf8());
        let (c, inner) = TokenStream::scan(c)?;
        let close_ch = c
            .first()
            .ok_or_else(|| cursor.error().message(format!("unclosed delimiter '{}'", delim.open())))?;

        let close_delim = Delim::from_close(close_ch).ok_or_else(|| {
            c.error()
                .message(format!("expected '{}', found '{}'", delim.close(), close_ch))
        })?;

        if delim != close_delim {
            return Err(c.error().message(format!(
                "mismatched delimiter: expected '{}', found '{}'",
                delim.close(),
                close_ch,
            )));
        }

        let c = c.advance_by(close_ch.len_utf8());
        let mut group = Self::new(delim, inner);
        group.span = DelimSpan::new(cursor.span(), c.span());

        Ok((c, group))
    }
}

impl crate::ToTokens for Group {
    fn to_tokens(&self, tokens: &mut crate::TokenStream) {
        tokens.extend_one(crate::TokenTree::Group(self.clone()));
    }
}

impl crate::Spanner for Group {
    fn span(&self) -> Span {
        self.span.span()
    }
}

#[cfg(feature = "serde")]
impl serde::Serialize for Group {
    fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>
    where
        S: serde::Serializer,
    {
        use serde::ser::SerializeStruct;

        let mut o = s.serialize_struct("Group", 2)?;
        o.serialize_field("delim", &self.delim)?;
        o.serialize_field("tokens", &self.tokens)?;
        o.end()
    }
}