flat_token/
lib.rs

1extern crate proc_macro2;
2
3use proc_macro2::{
4    Delimiter, Ident, Literal, Punct, Span, TokenStream, TokenTree,
5};
6
7#[derive(Clone, Debug)]
8pub enum FlatToken {
9    Delim(char, Span),
10    Ident(Ident),
11    Punct(Punct),
12    Literal(Literal),
13}
14
15impl FlatToken {
16    pub fn span(&self) -> Span {
17        match self {
18            &FlatToken::Delim(_, span) => span,
19            FlatToken::Ident(tt) => tt.span(),
20            FlatToken::Punct(tt) => tt.span(),
21            FlatToken::Literal(tt) => tt.span(),
22        }
23    }
24}
25
26fn explicit_delimiters(delim: Delimiter) -> Option<(char, char)> {
27    match delim {
28        Delimiter::Parenthesis => Some(('(', ')')),
29        Delimiter::Brace => Some(('{', '}')),
30        Delimiter::Bracket => Some(('[', ']')),
31        // FIXME(eddyb) maybe encode implicit delimiters somehow?
32        // One way could be to have an opaque `FlatToken` variant,
33        // containing the entire group, instead of exposing its contents.
34        Delimiter::None => None
35    }
36}
37
38pub fn flatten(stream: TokenStream, out: &mut Vec<FlatToken>) {
39    for tt in stream {
40        let flat = match tt {
41            TokenTree::Group(tt) => {
42                let stream = tt.stream();
43                let spans = (tt.span_open(), tt.span_close());
44                let delimiters = explicit_delimiters(tt.delimiter());
45                if let Some((open, _)) = delimiters {
46                    out.push(FlatToken::Delim(open, spans.0));
47                }
48                flatten(stream, out);
49                if let Some((_, close)) = delimiters {
50                    FlatToken::Delim(close, spans.1)
51                } else {
52                    continue;
53                }
54            }
55            TokenTree::Ident(tt) => FlatToken::Ident(tt),
56            TokenTree::Punct(tt) => FlatToken::Punct(tt),
57            TokenTree::Literal(tt) => FlatToken::Literal(tt),
58        };
59        out.push(flat);
60    }
61}
flat_token/lib.rs

flat_token/
lib.rs
