moxy_token/
tree.rs

1use crate::lex::{Cursor, Scan};
2use crate::{Delim, Group, Ident, Keyword, Lit, Punct, Span, Spanner, ToTokens, TokenStream};
3
4/// One Rust token: a group, identifier, literal, punctuation mark, or keyword.
5#[derive(Debug, Clone, PartialEq, Eq)]
6#[cfg_attr(feature = "serde", derive(serde::Serialize), serde(untagged))]
7pub enum TokenTree {
8    Ident(Ident),
9    Keyword(Keyword),
10    Punct(Punct),
11    Literal(Lit),
12    Group(Group),
13}
14
15impl TokenTree {
16    pub fn span(&self) -> Span {
17        match self {
18            Self::Ident(v) => v.span(),
19            Self::Keyword(v) => v.span(),
20            Self::Punct(v) => v.span(),
21            Self::Literal(v) => v.span(),
22            Self::Group(v) => v.span().into(),
23        }
24    }
25
26    pub fn text(&self) -> Option<&str> {
27        match self {
28            Self::Ident(v) => Some(v.text()),
29            Self::Keyword(v) => Some(v.as_str()),
30            _ => None,
31        }
32    }
33
34    pub fn delim(&self) -> Option<Delim> {
35        match self {
36            Self::Group(g) => Some(g.delim()),
37            _ => None,
38        }
39    }
40
41    #[inline]
42    pub fn is_token(&self) -> bool {
43        !matches!(self, Self::Group(_))
44    }
45
46    #[inline]
47    pub fn is_group(&self) -> bool {
48        matches!(self, Self::Group(_))
49    }
50
51    #[inline]
52    pub fn as_group(&self) -> Option<&Group> {
53        match self {
54            Self::Group(v) => Some(v),
55            _ => None,
56        }
57    }
58
59    #[inline]
60    pub fn is_ident(&self) -> bool {
61        matches!(self, Self::Ident(_))
62    }
63
64    #[inline]
65    pub fn as_ident(&self) -> Option<&Ident> {
66        match self {
67            Self::Ident(v) => Some(v),
68            _ => None,
69        }
70    }
71
72    #[inline]
73    pub fn is_literal(&self) -> bool {
74        matches!(self, Self::Literal(_))
75    }
76
77    #[inline]
78    pub fn as_literal(&self) -> Option<&Lit> {
79        match self {
80            Self::Literal(v) => Some(v),
81            _ => None,
82        }
83    }
84}
85
86impl From<Ident> for TokenTree {
87    #[inline]
88    fn from(value: Ident) -> Self {
89        Self::Ident(value)
90    }
91}
92
93impl From<Keyword> for TokenTree {
94    #[inline]
95    fn from(value: Keyword) -> Self {
96        Self::Keyword(value)
97    }
98}
99
100impl From<Punct> for TokenTree {
101    #[inline]
102    fn from(value: Punct) -> Self {
103        Self::Punct(value)
104    }
105}
106
107impl From<Lit> for TokenTree {
108    #[inline]
109    fn from(value: Lit) -> Self {
110        Self::Literal(value)
111    }
112}
113
114impl From<Group> for TokenTree {
115    #[inline]
116    fn from(value: Group) -> Self {
117        Self::Group(value)
118    }
119}
120
121impl IntoIterator for TokenTree {
122    type Item = Self;
123    type IntoIter = std::iter::Once<Self>;
124
125    fn into_iter(self) -> Self::IntoIter {
126        std::iter::once(self)
127    }
128}
129
130impl std::fmt::Display for TokenTree {
131    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
132        match self {
133            Self::Ident(v) => write!(f, "{}", v),
134            Self::Keyword(v) => write!(f, "{}", v),
135            Self::Punct(v) => write!(f, "{}", v),
136            Self::Literal(v) => write!(f, "{}", v),
137            Self::Group(v) => write!(f, "{}", v),
138        }
139    }
140}
141
142impl ToTokens for TokenTree {
143    fn to_tokens(&self, tokens: &mut TokenStream) {
144        tokens.extend_one(self.clone());
145    }
146}
147
148impl Spanner for TokenTree {
149    fn span(&self) -> Span {
150        self.span()
151    }
152}
153
154impl ToTokens for &str {
155    fn to_tokens(&self, tokens: &mut TokenStream) {
156        use std::str::FromStr;
157
158        if let Ok(ts) = TokenStream::from_str(self) {
159            ts.to_tokens(tokens);
160        }
161    }
162}
163
164impl ToTokens for String {
165    fn to_tokens(&self, tokens: &mut TokenStream) {
166        crate::Lit::string(self).to_tokens(tokens);
167    }
168}
169
170impl ToTokens for &mut String {
171    fn to_tokens(&self, tokens: &mut TokenStream) {
172        crate::Lit::string(self).to_tokens(tokens);
173    }
174}
175
176/// Lex a run of `proc_macro` punctuation chars into moxy [`Punct`] tokens,
177/// preserving each token's compiler span and joint spacing.
178pub(crate) fn scan_puncts_spanned(run: &[(char, Span, crate::Spacing)], tokens: &mut TokenStream) {
179    let text: String = run.iter().map(|(c, _, _)| *c).collect();
180    let mut cursor = Cursor::new(&text, 0);
181    let mut idx = 0usize;
182
183    while !cursor.is_empty() {
184        match <Punct as Scan>::scan(cursor) {
185            Ok((next, mut op)) => {
186                let consumed = op.as_str().chars().count();
187                let span = run[idx].1.join(run[idx + consumed - 1].1);
188                op.set_span(span);
189                op.set_spacing(run[idx + consumed - 1].2);
190                tokens.extend_one(TokenTree::Punct(op));
191                idx += consumed;
192                cursor = next;
193            }
194            Err(_) => break,
195        }
196    }
197}
moxy_token/tree.rs

moxy_token/
tree.rs
