macros_utils/
tokens.rs

1use std::{
2    collections::VecDeque,
3    fmt::{Debug, Display, Formatter},
4    str::FromStr,
5};
6
7use proc_macro2::{Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
8use quote::{quote, ToTokens, TokenStreamExt};
9
10use crate::{
11    parsers::{
12        get_byte_at, parse_lit_byte, parse_lit_byte_str, parse_lit_byte_str_raw, parse_lit_char,
13        parse_lit_float, parse_lit_int, parse_lit_str, parse_lit_str_raw,
14    },
15    MacroStream, ParseError, ParseErrorKind, ParseResult,
16};
17
18/// The delimiter of a group of tokens
19#[derive(Clone, Copy, Debug, Eq, PartialEq)]
20pub enum Delimiter {
21    /// `( ... )`
22    Parenthesis,
23    /// `{ ... }`
24    Brace,
25    /// `[ ... ]`
26    Bracket,
27    /// `Ø ... Ø`
28    /// An invisible delimiter around something like $var
29    /// Ensures that if $var is substituted in as 1 + 2
30    /// order of operations is preserved
31    None,
32}
33
34impl Display for Delimiter {
35    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
36        match self {
37            Self::Parenthesis => write!(f, "parenthesis"),
38            Self::Brace => write!(f, "braces"),
39            Self::Bracket => write!(f, "brackets"),
40            Self::None => write!(f, "none"),
41        }
42    }
43}
44
45impl From<Delimiter> for proc_macro2::Delimiter {
46    fn from(delimiter: Delimiter) -> Self {
47        match delimiter {
48            Delimiter::Parenthesis => Self::Parenthesis,
49            Delimiter::Brace => Self::Brace,
50            Delimiter::Bracket => Self::Bracket,
51            Delimiter::None => Self::None,
52        }
53    }
54}
55impl From<&Delimiter> for proc_macro2::Delimiter {
56    fn from(delimiter: &Delimiter) -> Self {
57        match delimiter {
58            Delimiter::Parenthesis => Self::Parenthesis,
59            Delimiter::Brace => Self::Brace,
60            Delimiter::Bracket => Self::Bracket,
61            Delimiter::None => Self::None,
62        }
63    }
64}
65
66impl From<proc_macro2::Delimiter> for Delimiter {
67    fn from(delimiter: proc_macro2::Delimiter) -> Self {
68        match delimiter {
69            proc_macro2::Delimiter::Parenthesis => Self::Parenthesis,
70            proc_macro2::Delimiter::Brace => Self::Brace,
71            proc_macro2::Delimiter::Bracket => Self::Bracket,
72            proc_macro2::Delimiter::None => Self::None,
73        }
74    }
75}
76impl From<&proc_macro2::Delimiter> for Delimiter {
77    fn from(delimiter: &proc_macro2::Delimiter) -> Self {
78        match delimiter {
79            proc_macro2::Delimiter::Parenthesis => Self::Parenthesis,
80            proc_macro2::Delimiter::Brace => Self::Brace,
81            proc_macro2::Delimiter::Bracket => Self::Bracket,
82            proc_macro2::Delimiter::None => Self::None,
83        }
84    }
85}
86
87/// A token in a macro.
88#[derive(Clone, Debug)]
89pub enum Token {
90    Ident {
91        name: String,
92        span: Span,
93    },
94
95    Group {
96        delimiter: Delimiter,
97        stream: MacroStream,
98        span: Span,
99    },
100
101    Literal {
102        kind: LiteralKind,
103        value: String,
104        span: Span,
105        suffix: String,
106        token: Option<Literal>,
107    },
108
109    /// either a single character for something like `+`
110    /// or a longer string for something like `+=` or `+===`
111    Punctuation {
112        value: char,
113        spacing: Spacing,
114        span: Span,
115    },
116}
117
118impl PartialEq for Token {
119    fn eq(&self, other: &Self) -> bool {
120        match (self, other) {
121            (
122                Self::Ident { name, .. },
123                Self::Ident {
124                    name: other_name, ..
125                },
126            ) => name == other_name,
127            (
128                Self::Group {
129                    delimiter, stream, ..
130                },
131                Self::Group {
132                    delimiter: other_delimiter,
133                    stream: other_stream,
134                    ..
135                },
136            ) => delimiter == other_delimiter && stream == other_stream,
137            (
138                Self::Literal {
139                    kind,
140                    value,
141                    suffix,
142                    ..
143                },
144                Self::Literal {
145                    kind: other_kind,
146                    value: other_value,
147                    suffix: other_suffix,
148                    ..
149                },
150            ) => kind == other_kind && value == other_value && suffix == other_suffix,
151            (
152                Self::Punctuation { value, .. },
153                Self::Punctuation {
154                    value: other_value, ..
155                },
156            ) => value == other_value,
157            _ => false,
158        }
159    }
160}
161
162impl Eq for Token {}
163
164/// The kind of literal.
165#[derive(Clone, Debug, Eq, PartialEq)]
166pub enum LiteralKind {
167    Byte,
168    Char,
169    Integer,
170    Float,
171    Str,
172    // the u8 is the number of `#` symbols used in the raw string
173    StrRaw(u8),
174    ByteStr,
175    // the u8 is the number of `#` symbols used in the raw string
176    ByteStrRaw(u8),
177}
178
179impl LiteralKind {
180    fn to_ident(&self) -> Token {
181        Token::Ident {
182            name: match self {
183                Self::Byte => "Byte",
184                Self::Char => "Char",
185                Self::Integer => "Integer",
186                Self::Float => "Float",
187                Self::Str => "String",
188                Self::StrRaw(_) => "StrRaw",
189                Self::ByteStr => "ByteStr",
190                Self::ByteStrRaw(_) => "ByteStrRaw",
191            }
192            .to_string(),
193            span: Span::call_site(),
194        }
195    }
196}
197
198impl Token {
199    pub fn to_token_stream(&self) -> TokenStream {
200        match self {
201            Self::Group { .. } => quote!(),
202            Self::Ident { name, .. } => {
203                quote! {
204                    macros_utils::Token::Ident {
205                        name: #name.to_string(),
206                        span: macros_utils::call_site(),
207                    }
208                }
209            },
210            Self::Literal {
211                kind,
212                suffix,
213                value,
214                ..
215            } => {
216                let kind = kind.to_ident();
217                quote! {
218                    macros_utils::Token::Literal {
219                        kind: macros_utils::LiteralKind::#kind,
220                        value: #value.to_string(),
221                        span: macros_utils::call_site(),
222                        suffix: #suffix.to_string(),
223                        token: None,
224                    }
225                }
226            },
227            Self::Punctuation { value, .. } => {
228                quote! {
229                    macros_utils::Token::Punctuation {
230                        value: #value.to_string(),
231                        span: macros_utils::call_site(),
232                    }
233                }
234            },
235        }
236    }
237
238    pub fn from_tokens(queue: &mut VecDeque<TokenTree>) -> ParseResult<Self> {
239        let token = queue.pop_front().unwrap();
240        Ok(match token {
241            TokenTree::Ident(ident) => Self::Ident {
242                name: ident.to_string(),
243                span: ident.span(),
244            },
245            TokenTree::Group(group) => Self::Group {
246                delimiter: group.delimiter().into(),
247                stream: MacroStream::from_tokens(group.stream())?,
248                span: group.span(),
249            },
250            TokenTree::Literal(lit) => {
251                let literal = lit.to_string();
252                match get_byte_at(&literal, 0) {
253                    b'"' => {
254                        let (value, suffix) = parse_lit_str(&literal)?;
255                        Self::Literal {
256                            kind: LiteralKind::Str,
257                            value,
258                            span: lit.span(),
259                            suffix,
260                            token: Some(lit),
261                        }
262                    },
263                    b'r' => {
264                        let (value, suffix, hashtags) = parse_lit_str_raw(&literal)?;
265                        Self::Literal {
266                            kind: LiteralKind::StrRaw(hashtags),
267                            value,
268                            span: lit.span(),
269                            suffix,
270                            token: Some(lit),
271                        }
272                    },
273                    b'b' => match get_byte_at(&literal, 1) {
274                        b'"' => {
275                            let (value, suffix) = parse_lit_byte_str(&literal)?;
276                            Self::Literal {
277                                kind: LiteralKind::ByteStr,
278                                value,
279                                span: lit.span(),
280                                suffix,
281                                token: Some(lit),
282                            }
283                        },
284                        b'r' => {
285                            let (value, suffix, hashtags) = parse_lit_byte_str_raw(&literal)?;
286                            Self::Literal {
287                                kind: LiteralKind::ByteStrRaw(hashtags),
288                                value,
289                                span: lit.span(),
290                                suffix,
291                                token: Some(lit),
292                            }
293                        },
294                        b'\'' => {
295                            let (value, suffix) = parse_lit_byte(&literal)?;
296                            Self::Literal {
297                                kind: LiteralKind::Byte,
298                                value,
299                                span: lit.span(),
300                                suffix,
301                                token: Some(lit),
302                            }
303                        },
304                        _ => {
305                            return Err(ParseError::new(
306                                lit.span(),
307                                ParseErrorKind::UnknownLiteral(literal),
308                            ))
309                        },
310                    },
311                    b'\'' => {
312                        let (value, suffix) = parse_lit_char(&literal)?;
313                        Self::Literal {
314                            kind: LiteralKind::Char,
315                            value,
316                            span: lit.span(),
317                            suffix,
318                            token: Some(lit),
319                        }
320                    },
321                    b'0'..=b'9' | b'-' => {
322                        if let Some((value, suffix)) = parse_lit_float(&literal)? {
323                            Self::Literal {
324                                kind: LiteralKind::Float,
325                                value,
326                                span: lit.span(),
327                                suffix,
328                                token: Some(lit),
329                            }
330                        } else {
331                            let (value, suffix) = parse_lit_int(&literal)?;
332                            Self::Literal {
333                                kind: LiteralKind::Integer,
334                                value,
335                                span: lit.span(),
336                                suffix,
337                                token: Some(lit),
338                            }
339                        }
340                    },
341                    _ => {
342                        return Err(ParseError::new(
343                            lit.span(),
344                            ParseErrorKind::UnknownLiteral(literal),
345                        ))
346                    },
347                }
348            },
349            TokenTree::Punct(p) => Self::Punctuation {
350                value: p.as_char(),
351                spacing: p.spacing(),
352                span: p.span(),
353            },
354        })
355    }
356
357    pub fn ident(&self) -> Option<&str> {
358        if let Token::Ident { name, .. } = self {
359            Some(name)
360        } else {
361            None
362        }
363    }
364
365    pub fn group(&self) -> Option<&MacroStream> {
366        if let Token::Group { stream, .. } = self {
367            Some(stream)
368        } else {
369            None
370        }
371    }
372
373    pub fn lit_suffix(&self) -> Option<&str> {
374        if let Token::Literal { suffix, .. } = self {
375            Some(suffix)
376        } else {
377            None
378        }
379    }
380
381    pub fn span(&self) -> Span {
382        match self {
383            Token::Ident { span, .. } => *span,
384            Token::Group { span, .. } => *span,
385            Token::Literal { span, .. } => *span,
386            Token::Punctuation { span, .. } => *span,
387        }
388    }
389
390    pub fn punctuation(&self) -> Option<&char> {
391        if let Token::Punctuation { value, .. } = self {
392            Some(value)
393        } else {
394            None
395        }
396    }
397
398    pub fn lit_byte(&self) -> Option<u8> {
399        if let Token::Literal {
400            kind: LiteralKind::Byte,
401            value,
402            ..
403        } = self
404        {
405            if let Ok(value) = value.parse::<u8>() {
406                return Some(value);
407            }
408        }
409        None
410    }
411
412    pub fn lit_char(&self) -> Option<char> {
413        if let Token::Literal {
414            kind: LiteralKind::Char,
415            value,
416            ..
417        } = self
418        {
419            if let Ok(value) = value.parse::<char>() {
420                return Some(value);
421            }
422        }
423        None
424    }
425
426    pub fn lit_integer<I>(&self) -> Option<I>
427    where
428        I: FromStr,
429    {
430        if let Token::Literal {
431            kind: LiteralKind::Integer,
432            value,
433            ..
434        } = self
435        {
436            if let Ok(value) = value.parse::<I>() {
437                return Some(value);
438            }
439        }
440        None
441    }
442
443    pub fn lit_float<F>(&self) -> Option<F>
444    where
445        F: FromStr,
446    {
447        if let Token::Literal {
448            kind: LiteralKind::Float,
449            value,
450            ..
451        } = self
452        {
453            if let Ok(value) = value.parse::<F>() {
454                return Some(value);
455            }
456        }
457        None
458    }
459
460    pub fn lit_str(&self) -> Option<&str> {
461        if let Token::Literal {
462            kind: LiteralKind::Str,
463            value,
464            ..
465        } = self
466        {
467            Some(value)
468        } else {
469            None
470        }
471    }
472
473    pub fn lit_str_raw(&self) -> Option<&str> {
474        if let Token::Literal {
475            kind: LiteralKind::StrRaw(_),
476            value,
477            ..
478        } = self
479        {
480            Some(value)
481        } else {
482            None
483        }
484    }
485
486    pub fn lit_byte_str(&self) -> Option<&[u8]> {
487        if let Token::Literal {
488            kind: LiteralKind::ByteStr,
489            value,
490            ..
491        } = self
492        {
493            Some(value.as_bytes())
494        } else {
495            None
496        }
497    }
498
499    pub fn lit_byte_str_raw(&self) -> Option<&[u8]> {
500        if let Token::Literal {
501            kind: LiteralKind::ByteStrRaw(_),
502            value,
503            ..
504        } = self
505        {
506            Some(value.as_bytes())
507        } else {
508            None
509        }
510    }
511}
512
513/// Note: Converting a Literal will result in the loss of the suffix and typically also specific information regarding what type it is, the value itself will not be lost (large u128 numbers exceeding 127 bits may lose their last bit though).
514impl ToTokens for Token {
515    fn to_tokens(&self, tokens: &mut TokenStream) {
516        tokens.append::<TokenTree>(match self {
517            Self::Group {
518                delimiter,
519                stream,
520                span,
521            } => {
522                let mut token = Group::new(delimiter.into(), stream.to_token_stream());
523                token.set_span(*span);
524                token.into()
525            },
526            Self::Ident { name, span } => Ident::new(name, *span).into(),
527            Self::Literal {
528                kind,
529                value,
530                token,
531                span,
532                ..
533            } => match token {
534                Some(lit) => lit.clone().into(),
535                None => {
536                    let mut token = match kind {
537                        LiteralKind::Byte => Literal::u8_unsuffixed(value.parse::<u8>().unwrap()),
538                        LiteralKind::ByteStr => Literal::byte_string(value.as_bytes()),
539                        LiteralKind::ByteStrRaw(_) => Literal::byte_string(value.as_bytes()),
540                        LiteralKind::Char => Literal::character(value.parse::<char>().unwrap()),
541                        LiteralKind::Float => {
542                            Literal::f64_unsuffixed(value.parse::<f64>().unwrap())
543                        },
544                        LiteralKind::Integer => {
545                            Literal::i128_unsuffixed(value.parse::<i128>().unwrap())
546                        },
547                        LiteralKind::Str => Literal::string(value),
548                        LiteralKind::StrRaw(_) => Literal::string(value),
549                    };
550                    token.set_span(*span);
551                    token.into()
552                },
553            },
554            Self::Punctuation {
555                value,
556                span,
557                spacing,
558            } => {
559                let mut token = Punct::new(*value, *spacing);
560                token.set_span(*span);
561                token.into()
562            },
563        });
564    }
565}
macros_utils/tokens.rs

macros_utils/
tokens.rs
