moxy_token/
tree.rs

1use crate::lex::{Cursor, Scan};
2use crate::{Delim, Group, Ident, Keyword, Lit, Punctuation, Span, Spanner, ToTokens, TokenStream};
3
4#[derive(Debug, Clone, PartialEq, Eq)]
5#[cfg_attr(feature = "serde", derive(serde::Serialize), serde(untagged))]
6pub enum TokenTree {
7    Ident(Ident),
8    Keyword(Keyword),
9    Punct(Punctuation),
10    Literal(Lit),
11    Group(Group),
12}
13
14impl TokenTree {
15    pub fn span(&self) -> Span {
16        match self {
17            Self::Ident(v) => v.span(),
18            Self::Keyword(v) => v.span(),
19            Self::Punct(v) => v.span(),
20            Self::Literal(v) => v.span(),
21            Self::Group(v) => v.span().into(),
22        }
23    }
24
25    pub fn text(&self) -> Option<&str> {
26        match self {
27            Self::Ident(v) => Some(v.text()),
28            Self::Keyword(v) => Some(v.as_str()),
29            _ => None,
30        }
31    }
32
33    pub fn delim(&self) -> Option<Delim> {
34        match self {
35            Self::Group(g) => Some(g.delim()),
36            _ => None,
37        }
38    }
39
40    #[inline]
41    pub fn is_token(&self) -> bool {
42        !matches!(self, Self::Group(_))
43    }
44
45    #[inline]
46    pub fn is_group(&self) -> bool {
47        matches!(self, Self::Group(_))
48    }
49
50    #[inline]
51    pub fn as_group(&self) -> Option<&Group> {
52        match self {
53            Self::Group(v) => Some(v),
54            _ => None,
55        }
56    }
57
58    #[inline]
59    pub fn is_ident(&self) -> bool {
60        matches!(self, Self::Ident(_))
61    }
62
63    #[inline]
64    pub fn as_ident(&self) -> Option<&Ident> {
65        match self {
66            Self::Ident(v) => Some(v),
67            _ => None,
68        }
69    }
70
71    #[inline]
72    pub fn is_literal(&self) -> bool {
73        matches!(self, Self::Literal(_))
74    }
75
76    #[inline]
77    pub fn as_literal(&self) -> Option<&Lit> {
78        match self {
79            Self::Literal(v) => Some(v),
80            _ => None,
81        }
82    }
83}
84
85impl From<Ident> for TokenTree {
86    #[inline]
87    fn from(value: Ident) -> Self {
88        Self::Ident(value)
89    }
90}
91
92impl From<Keyword> for TokenTree {
93    #[inline]
94    fn from(value: Keyword) -> Self {
95        Self::Keyword(value)
96    }
97}
98
99impl From<Punctuation> for TokenTree {
100    #[inline]
101    fn from(value: Punctuation) -> Self {
102        Self::Punct(value)
103    }
104}
105
106impl From<Lit> for TokenTree {
107    #[inline]
108    fn from(value: Lit) -> Self {
109        Self::Literal(value)
110    }
111}
112
113impl From<Group> for TokenTree {
114    #[inline]
115    fn from(value: Group) -> Self {
116        Self::Group(value)
117    }
118}
119
120impl IntoIterator for TokenTree {
121    type Item = Self;
122    type IntoIter = std::iter::Once<Self>;
123
124    fn into_iter(self) -> Self::IntoIter {
125        std::iter::once(self)
126    }
127}
128
129impl std::fmt::Display for TokenTree {
130    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
131        match self {
132            Self::Ident(v) => write!(f, "{}", v),
133            Self::Keyword(v) => write!(f, "{}", v),
134            Self::Punct(v) => write!(f, "{}", v),
135            Self::Literal(v) => write!(f, "{}", v),
136            Self::Group(v) => write!(f, "{}", v),
137        }
138    }
139}
140
141impl ToTokens for TokenTree {
142    fn to_tokens(&self, tokens: &mut TokenStream) {
143        tokens.extend_one(self.clone());
144    }
145}
146
147impl Spanner for TokenTree {
148    fn span(&self) -> Span {
149        self.span()
150    }
151}
152
153impl ToTokens for &str {
154    fn to_tokens(&self, tokens: &mut TokenStream) {
155        use std::str::FromStr;
156
157        if let Ok(ts) = TokenStream::from_str(self) {
158            ts.to_tokens(tokens);
159        }
160    }
161}
162
163impl ToTokens for String {
164    fn to_tokens(&self, tokens: &mut TokenStream) {
165        crate::Lit::string(self).to_tokens(tokens);
166    }
167}
168
169impl ToTokens for &mut String {
170    fn to_tokens(&self, tokens: &mut TokenStream) {
171        crate::Lit::string(self).to_tokens(tokens);
172    }
173}
174
175/// Lex a run of consecutive `proc_macro` punctuation chars into moxy
176/// [`Punctuation`] tokens, preserving each token's real compiler span.
177///
178/// `proc_macro` emits multi-char operators (`=>`, `::`, `&&`) as single-char
179/// puncts; moxy models them as one variant. We longest-match against the run's
180/// text (reusing `<Punctuation as Scan>`) and retag each matched variant with
181/// the joined spans of the chars it consumed.
182pub(crate) fn scan_puncts_spanned(run: &[(char, Span)], tokens: &mut TokenStream) {
183    let text: String = run.iter().map(|(c, _)| *c).collect();
184    let mut cursor = Cursor::new(&text, 0);
185    let mut idx = 0usize;
186
187    while !cursor.is_empty() {
188        match <Punctuation as Scan>::scan(cursor) {
189            Ok((next, mut op)) => {
190                let consumed = op.as_str().chars().count();
191                let span = run[idx].1.join(run[idx + consumed - 1].1);
192                op.set_span(span);
193                tokens.extend_one(TokenTree::Punct(op));
194                idx += consumed;
195                cursor = next;
196            }
197            Err(_) => break,
198        }
199    }
200}
moxy_token/tree.rs

moxy_token/
tree.rs
