ra_ap_syntax_bridge/
to_parser_input.rs

1//! Convert macro-by-example tokens which are specific to macro expansion into a
2//! format that works for our parser.
3
4use std::fmt;
5use std::hash::Hash;
6
7use rustc_hash::FxHashMap;
8use span::{Edition, SpanData};
9use syntax::{SyntaxKind, SyntaxKind::*, T};
10
11pub fn to_parser_input<Ctx: Copy + fmt::Debug + PartialEq + Eq + Hash>(
12    buffer: tt::TokenTreesView<'_, SpanData<Ctx>>,
13    span_to_edition: &mut dyn FnMut(Ctx) -> Edition,
14) -> parser::Input {
15    let mut res = parser::Input::with_capacity(buffer.len());
16
17    let mut current = buffer.cursor();
18    let mut syntax_context_to_edition_cache = FxHashMap::default();
19    let mut ctx_edition =
20        |ctx| *syntax_context_to_edition_cache.entry(ctx).or_insert_with(|| span_to_edition(ctx));
21
22    while !current.eof() {
23        let tt = current.token_tree();
24
25        // Check if it is lifetime
26        if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tt
27            && punct.char == '\''
28        {
29            current.bump();
30            match current.token_tree() {
31                Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => {
32                    res.push(LIFETIME_IDENT, ctx_edition(ident.span.ctx));
33                    current.bump();
34                    continue;
35                }
36                _ => panic!("Next token must be ident"),
37            }
38        }
39
40        match tt {
41            Some(tt::TokenTree::Leaf(leaf)) => {
42                match leaf {
43                    tt::Leaf::Literal(lit) => {
44                        let kind = match lit.kind {
45                            tt::LitKind::Byte => SyntaxKind::BYTE,
46                            tt::LitKind::Char => SyntaxKind::CHAR,
47                            tt::LitKind::Integer => SyntaxKind::INT_NUMBER,
48                            tt::LitKind::Float => SyntaxKind::FLOAT_NUMBER,
49                            tt::LitKind::Str | tt::LitKind::StrRaw(_) => SyntaxKind::STRING,
50                            tt::LitKind::ByteStr | tt::LitKind::ByteStrRaw(_) => {
51                                SyntaxKind::BYTE_STRING
52                            }
53                            tt::LitKind::CStr | tt::LitKind::CStrRaw(_) => SyntaxKind::C_STRING,
54                            tt::LitKind::Err(_) => SyntaxKind::ERROR,
55                        };
56                        res.push(kind, ctx_edition(lit.span.ctx));
57
58                        if kind == FLOAT_NUMBER && !lit.symbol.as_str().ends_with('.') {
59                            // Tag the token as joint if it is float with a fractional part
60                            // we use this jointness to inform the parser about what token split
61                            // event to emit when we encounter a float literal in a field access
62                            res.was_joint();
63                        }
64                    }
65                    tt::Leaf::Ident(ident) => {
66                        let edition = ctx_edition(ident.span.ctx);
67                        match ident.sym.as_str() {
68                            "_" => res.push(T![_], edition),
69                            i if i.starts_with('\'') => res.push(LIFETIME_IDENT, edition),
70                            _ if ident.is_raw.yes() => res.push(IDENT, edition),
71                            text => match SyntaxKind::from_keyword(text, edition) {
72                                Some(kind) => res.push(kind, edition),
73                                None => {
74                                    let contextual_keyword =
75                                        SyntaxKind::from_contextual_keyword(text, edition)
76                                            .unwrap_or(SyntaxKind::IDENT);
77                                    res.push_ident(contextual_keyword, edition);
78                                }
79                            },
80                        }
81                    }
82                    tt::Leaf::Punct(punct) => {
83                        let kind = SyntaxKind::from_char(punct.char)
84                            .unwrap_or_else(|| panic!("{punct:#?} is not a valid punct"));
85                        res.push(kind, ctx_edition(punct.span.ctx));
86                        if punct.spacing == tt::Spacing::Joint {
87                            res.was_joint();
88                        }
89                    }
90                }
91                current.bump();
92            }
93            Some(tt::TokenTree::Subtree(subtree)) => {
94                if let Some(kind) = match subtree.delimiter.kind {
95                    tt::DelimiterKind::Parenthesis => Some(T!['(']),
96                    tt::DelimiterKind::Brace => Some(T!['{']),
97                    tt::DelimiterKind::Bracket => Some(T!['[']),
98                    tt::DelimiterKind::Invisible => None,
99                } {
100                    res.push(kind, ctx_edition(subtree.delimiter.open.ctx));
101                }
102                current.bump();
103            }
104            None => {
105                let subtree = current.end();
106                if let Some(kind) = match subtree.delimiter.kind {
107                    tt::DelimiterKind::Parenthesis => Some(T![')']),
108                    tt::DelimiterKind::Brace => Some(T!['}']),
109                    tt::DelimiterKind::Bracket => Some(T![']']),
110                    tt::DelimiterKind::Invisible => None,
111                } {
112                    res.push(kind, ctx_edition(subtree.delimiter.close.ctx));
113                }
114            }
115        };
116    }
117
118    res
119}
ra_ap_syntax_bridge/to_parser_input.rs

ra_ap_syntax_bridge/
to_parser_input.rs
