ra_ap_tt/
lib.rs

1//! `tt` crate defines a `TokenTree` data structure: this is the interface (both
2//! input and output) of macros.
3//!
4//! The `TokenTree` is semantically a tree, but for performance reasons it is stored as a flat structure.
5
6#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
7
8#[cfg(not(feature = "in-rust-tree"))]
9extern crate ra_ap_rustc_lexer as rustc_lexer;
10#[cfg(feature = "in-rust-tree")]
11extern crate rustc_lexer;
12
13pub mod buffer;
14pub mod iter;
15
16use std::fmt;
17
18use buffer::Cursor;
19use intern::Symbol;
20use iter::{TtElement, TtIter};
21use stdx::{impl_from, itertools::Itertools as _};
22
23pub use text_size::{TextRange, TextSize};
24
25pub const MAX_GLUED_PUNCT_LEN: usize = 3;
26
27#[derive(Clone, PartialEq, Debug)]
28pub struct Lit {
29    pub kind: LitKind,
30    pub symbol: Symbol,
31    pub suffix: Option<Symbol>,
32}
33
34#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
35pub enum IdentIsRaw {
36    No,
37    Yes,
38}
39impl IdentIsRaw {
40    pub fn yes(self) -> bool {
41        matches!(self, IdentIsRaw::Yes)
42    }
43    pub fn no(&self) -> bool {
44        matches!(self, IdentIsRaw::No)
45    }
46    pub fn as_str(self) -> &'static str {
47        match self {
48            IdentIsRaw::No => "",
49            IdentIsRaw::Yes => "r#",
50        }
51    }
52    pub fn split_from_symbol(sym: &str) -> (Self, &str) {
53        if let Some(sym) = sym.strip_prefix("r#") {
54            (IdentIsRaw::Yes, sym)
55        } else {
56            (IdentIsRaw::No, sym)
57        }
58    }
59}
60
61#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
62pub enum LitKind {
63    Byte,
64    Char,
65    Integer, // e.g. `1`, `1u8`, `1f32`
66    Float,   // e.g. `1.`, `1.0`, `1e3f32`
67    Str,
68    StrRaw(u8), // raw string delimited by `n` hash symbols
69    ByteStr,
70    ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols
71    CStr,
72    CStrRaw(u8),
73    Err(()),
74}
75
76#[derive(Debug, Clone, PartialEq, Eq, Hash)]
77pub enum TokenTree<S = u32> {
78    Leaf(Leaf<S>),
79    Subtree(Subtree<S>),
80}
81impl_from!(Leaf<S>, Subtree<S> for TokenTree);
82impl<S: Copy> TokenTree<S> {
83    pub fn first_span(&self) -> S {
84        match self {
85            TokenTree::Leaf(l) => *l.span(),
86            TokenTree::Subtree(s) => s.delimiter.open,
87        }
88    }
89}
90
91#[derive(Debug, Clone, PartialEq, Eq, Hash)]
92pub enum Leaf<S> {
93    Literal(Literal<S>),
94    Punct(Punct<S>),
95    Ident(Ident<S>),
96}
97
98impl<S> Leaf<S> {
99    pub fn span(&self) -> &S {
100        match self {
101            Leaf::Literal(it) => &it.span,
102            Leaf::Punct(it) => &it.span,
103            Leaf::Ident(it) => &it.span,
104        }
105    }
106}
107impl_from!(Literal<S>, Punct<S>, Ident<S> for Leaf);
108
109#[derive(Debug, Clone, PartialEq, Eq, Hash)]
110pub struct Subtree<S> {
111    pub delimiter: Delimiter<S>,
112    /// Number of following token trees that belong to this subtree, excluding this subtree.
113    pub len: u32,
114}
115
116impl<S> Subtree<S> {
117    pub fn usize_len(&self) -> usize {
118        self.len as usize
119    }
120}
121
122#[derive(Clone, PartialEq, Eq, Hash)]
123pub struct TopSubtree<S>(pub Box<[TokenTree<S>]>);
124
125impl<S: Copy> TopSubtree<S> {
126    pub fn empty(span: DelimSpan<S>) -> Self {
127        Self(Box::new([TokenTree::Subtree(Subtree {
128            delimiter: Delimiter::invisible_delim_spanned(span),
129            len: 0,
130        })]))
131    }
132
133    pub fn invisible_from_leaves<const N: usize>(delim_span: S, leaves: [Leaf<S>; N]) -> Self {
134        let mut builder = TopSubtreeBuilder::new(Delimiter::invisible_spanned(delim_span));
135        builder.extend(leaves);
136        builder.build()
137    }
138
139    pub fn from_token_trees(delimiter: Delimiter<S>, token_trees: TokenTreesView<'_, S>) -> Self {
140        let mut builder = TopSubtreeBuilder::new(delimiter);
141        builder.extend_with_tt(token_trees);
142        builder.build()
143    }
144
145    pub fn from_subtree(subtree: SubtreeView<'_, S>) -> Self {
146        Self(subtree.0.into())
147    }
148
149    pub fn view(&self) -> SubtreeView<'_, S> {
150        SubtreeView::new(&self.0)
151    }
152
153    pub fn iter(&self) -> TtIter<'_, S> {
154        self.view().iter()
155    }
156
157    pub fn top_subtree(&self) -> &Subtree<S> {
158        self.view().top_subtree()
159    }
160
161    pub fn top_subtree_delimiter_mut(&mut self) -> &mut Delimiter<S> {
162        let TokenTree::Subtree(subtree) = &mut self.0[0] else {
163            unreachable!("the first token tree is always the top subtree");
164        };
165        &mut subtree.delimiter
166    }
167
168    pub fn token_trees(&self) -> TokenTreesView<'_, S> {
169        self.view().token_trees()
170    }
171}
172
173#[derive(Debug, Clone, PartialEq, Eq, Hash)]
174pub struct TopSubtreeBuilder<S> {
175    unclosed_subtree_indices: Vec<usize>,
176    token_trees: Vec<TokenTree<S>>,
177    last_closed_subtree: Option<usize>,
178}
179
180impl<S: Copy> TopSubtreeBuilder<S> {
181    pub fn new(top_delimiter: Delimiter<S>) -> Self {
182        let mut result = Self {
183            unclosed_subtree_indices: Vec::new(),
184            token_trees: Vec::new(),
185            last_closed_subtree: None,
186        };
187        let top_subtree = TokenTree::Subtree(Subtree { delimiter: top_delimiter, len: 0 });
188        result.token_trees.push(top_subtree);
189        result
190    }
191
192    pub fn open(&mut self, delimiter_kind: DelimiterKind, open_span: S) {
193        self.unclosed_subtree_indices.push(self.token_trees.len());
194        self.token_trees.push(TokenTree::Subtree(Subtree {
195            delimiter: Delimiter {
196                open: open_span,
197                close: open_span, // Will be overwritten on close.
198                kind: delimiter_kind,
199            },
200            len: 0,
201        }));
202    }
203
204    pub fn close(&mut self, close_span: S) {
205        let last_unclosed_index = self
206            .unclosed_subtree_indices
207            .pop()
208            .expect("attempt to close a `tt::Subtree` when none is open");
209        let subtree_len = (self.token_trees.len() - last_unclosed_index - 1) as u32;
210        let TokenTree::Subtree(subtree) = &mut self.token_trees[last_unclosed_index] else {
211            unreachable!("unclosed token tree is always a subtree");
212        };
213        subtree.len = subtree_len;
214        subtree.delimiter.close = close_span;
215        self.last_closed_subtree = Some(last_unclosed_index);
216    }
217
218    /// You cannot call this consecutively, it will only work once after close.
219    pub fn remove_last_subtree_if_invisible(&mut self) {
220        let Some(last_subtree_idx) = self.last_closed_subtree else { return };
221        if let TokenTree::Subtree(Subtree {
222            delimiter: Delimiter { kind: DelimiterKind::Invisible, .. },
223            ..
224        }) = self.token_trees[last_subtree_idx]
225        {
226            self.token_trees.remove(last_subtree_idx);
227            self.last_closed_subtree = None;
228        }
229    }
230
231    pub fn push(&mut self, leaf: Leaf<S>) {
232        self.token_trees.push(TokenTree::Leaf(leaf));
233    }
234
235    pub fn extend(&mut self, leaves: impl IntoIterator<Item = Leaf<S>>) {
236        self.token_trees.extend(leaves.into_iter().map(TokenTree::Leaf));
237    }
238
239    /// This does not check the token trees are valid, beware!
240    pub fn extend_tt_dangerous(&mut self, tt: impl IntoIterator<Item = TokenTree<S>>) {
241        self.token_trees.extend(tt);
242    }
243
244    pub fn extend_with_tt(&mut self, tt: TokenTreesView<'_, S>) {
245        self.token_trees.extend(tt.0.iter().cloned());
246    }
247
248    /// Like [`Self::extend_with_tt()`], but makes sure the new tokens will never be
249    /// joint with whatever comes after them.
250    pub fn extend_with_tt_alone(&mut self, tt: TokenTreesView<'_, S>) {
251        if let Some((last, before_last)) = tt.0.split_last() {
252            self.token_trees.reserve(tt.0.len());
253            self.token_trees.extend(before_last.iter().cloned());
254            let last = if let TokenTree::Leaf(Leaf::Punct(last)) = last {
255                let mut last = *last;
256                last.spacing = Spacing::Alone;
257                TokenTree::Leaf(Leaf::Punct(last))
258            } else {
259                last.clone()
260            };
261            self.token_trees.push(last);
262        }
263    }
264
265    pub fn expected_delimiters(&self) -> impl Iterator<Item = &Delimiter<S>> {
266        self.unclosed_subtree_indices.iter().rev().map(|&subtree_idx| {
267            let TokenTree::Subtree(subtree) = &self.token_trees[subtree_idx] else {
268                unreachable!("unclosed token tree is always a subtree")
269            };
270            &subtree.delimiter
271        })
272    }
273
274    /// Builds, and remove the top subtree if it has only one subtree child.
275    pub fn build_skip_top_subtree(mut self) -> TopSubtree<S> {
276        let top_tts = TokenTreesView::new(&self.token_trees[1..]);
277        match top_tts.try_into_subtree() {
278            Some(_) => {
279                assert!(
280                    self.unclosed_subtree_indices.is_empty(),
281                    "attempt to build an unbalanced `TopSubtreeBuilder`"
282                );
283                TopSubtree(self.token_trees.drain(1..).collect())
284            }
285            None => self.build(),
286        }
287    }
288
289    pub fn build(mut self) -> TopSubtree<S> {
290        assert!(
291            self.unclosed_subtree_indices.is_empty(),
292            "attempt to build an unbalanced `TopSubtreeBuilder`"
293        );
294        let total_len = self.token_trees.len() as u32;
295        let TokenTree::Subtree(top_subtree) = &mut self.token_trees[0] else {
296            unreachable!("first token tree is always a subtree");
297        };
298        top_subtree.len = total_len - 1;
299        TopSubtree(self.token_trees.into_boxed_slice())
300    }
301
302    pub fn restore_point(&self) -> SubtreeBuilderRestorePoint {
303        SubtreeBuilderRestorePoint {
304            unclosed_subtree_indices_len: self.unclosed_subtree_indices.len(),
305            token_trees_len: self.token_trees.len(),
306            last_closed_subtree: self.last_closed_subtree,
307        }
308    }
309
310    pub fn restore(&mut self, restore_point: SubtreeBuilderRestorePoint) {
311        self.unclosed_subtree_indices.truncate(restore_point.unclosed_subtree_indices_len);
312        self.token_trees.truncate(restore_point.token_trees_len);
313        self.last_closed_subtree = restore_point.last_closed_subtree;
314    }
315}
316
317#[derive(Clone, Copy)]
318pub struct SubtreeBuilderRestorePoint {
319    unclosed_subtree_indices_len: usize,
320    token_trees_len: usize,
321    last_closed_subtree: Option<usize>,
322}
323
324#[derive(Clone, Copy)]
325pub struct TokenTreesView<'a, S>(&'a [TokenTree<S>]);
326
327impl<'a, S: Copy> TokenTreesView<'a, S> {
328    pub fn new(tts: &'a [TokenTree<S>]) -> Self {
329        if cfg!(debug_assertions) {
330            tts.iter().enumerate().for_each(|(idx, tt)| {
331                if let TokenTree::Subtree(tt) = &tt {
332                    // `<` and not `<=` because `Subtree.len` does not include the subtree node itself.
333                    debug_assert!(
334                        idx + tt.usize_len() < tts.len(),
335                        "`TokenTreeView::new()` was given a cut-in-half list"
336                    );
337                }
338            });
339        }
340        Self(tts)
341    }
342
343    pub fn iter(&self) -> TtIter<'a, S> {
344        TtIter::new(self.0)
345    }
346
347    pub fn cursor(&self) -> Cursor<'a, S> {
348        Cursor::new(self.0)
349    }
350
351    pub fn len(&self) -> usize {
352        self.0.len()
353    }
354
355    pub fn is_empty(&self) -> bool {
356        self.0.is_empty()
357    }
358
359    pub fn try_into_subtree(self) -> Option<SubtreeView<'a, S>> {
360        if let Some(TokenTree::Subtree(subtree)) = self.0.first()
361            && subtree.usize_len() == (self.0.len() - 1)
362        {
363            return Some(SubtreeView::new(self.0));
364        }
365        None
366    }
367
368    pub fn strip_invisible(self) -> TokenTreesView<'a, S> {
369        self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self)
370    }
371
372    /// This returns a **flat** structure of tokens (subtrees will be represented by a single node
373    /// preceding their children), so it isn't suited for most use cases, only for matching leaves
374    /// at the beginning/end with no subtrees before them. If you need a structured pass, use [`TtIter`].
375    pub fn flat_tokens(&self) -> &'a [TokenTree<S>] {
376        self.0
377    }
378
379    pub fn split(
380        self,
381        mut split_fn: impl FnMut(TtElement<'a, S>) -> bool,
382    ) -> impl Iterator<Item = TokenTreesView<'a, S>> {
383        let mut subtree_iter = self.iter();
384        let mut need_to_yield_even_if_empty = true;
385
386        std::iter::from_fn(move || {
387            if subtree_iter.is_empty() && !need_to_yield_even_if_empty {
388                return None;
389            };
390
391            need_to_yield_even_if_empty = false;
392            let savepoint = subtree_iter.savepoint();
393            let mut result = subtree_iter.from_savepoint(savepoint);
394            while let Some(tt) = subtree_iter.next() {
395                if split_fn(tt) {
396                    need_to_yield_even_if_empty = true;
397                    break;
398                }
399                result = subtree_iter.from_savepoint(savepoint);
400            }
401            Some(result)
402        })
403    }
404}
405
406impl<S: fmt::Debug + Copy> fmt::Debug for TokenTreesView<'_, S> {
407    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
408        let mut iter = self.iter();
409        while let Some(tt) = iter.next() {
410            print_debug_token(f, 0, tt)?;
411            if !iter.is_empty() {
412                writeln!(f)?;
413            }
414        }
415        Ok(())
416    }
417}
418
419impl<S: Copy> fmt::Display for TokenTreesView<'_, S> {
420    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
421        return token_trees_display(f, self.iter());
422
423        fn subtree_display<S>(
424            subtree: &Subtree<S>,
425            f: &mut fmt::Formatter<'_>,
426            iter: TtIter<'_, S>,
427        ) -> fmt::Result {
428            let (l, r) = match subtree.delimiter.kind {
429                DelimiterKind::Parenthesis => ("(", ")"),
430                DelimiterKind::Brace => ("{", "}"),
431                DelimiterKind::Bracket => ("[", "]"),
432                DelimiterKind::Invisible => ("", ""),
433            };
434            f.write_str(l)?;
435            token_trees_display(f, iter)?;
436            f.write_str(r)?;
437            Ok(())
438        }
439
440        fn token_trees_display<S>(f: &mut fmt::Formatter<'_>, iter: TtIter<'_, S>) -> fmt::Result {
441            let mut needs_space = false;
442            for child in iter {
443                if needs_space {
444                    f.write_str(" ")?;
445                }
446                needs_space = true;
447
448                match child {
449                    TtElement::Leaf(Leaf::Punct(p)) => {
450                        needs_space = p.spacing == Spacing::Alone;
451                        fmt::Display::fmt(p, f)?;
452                    }
453                    TtElement::Leaf(leaf) => fmt::Display::fmt(leaf, f)?,
454                    TtElement::Subtree(subtree, subtree_iter) => {
455                        subtree_display(subtree, f, subtree_iter)?
456                    }
457                }
458            }
459            Ok(())
460        }
461    }
462}
463
464#[derive(Clone, Copy)]
465// Invariant: always starts with `Subtree` that covers the entire thing.
466pub struct SubtreeView<'a, S>(&'a [TokenTree<S>]);
467
468impl<'a, S: Copy> SubtreeView<'a, S> {
469    pub fn new(tts: &'a [TokenTree<S>]) -> Self {
470        if cfg!(debug_assertions) {
471            let TokenTree::Subtree(subtree) = &tts[0] else {
472                panic!("first token tree must be a subtree in `SubtreeView`");
473            };
474            assert_eq!(
475                subtree.usize_len(),
476                tts.len() - 1,
477                "subtree must cover the entire `SubtreeView`"
478            );
479        }
480        Self(tts)
481    }
482
483    pub fn as_token_trees(self) -> TokenTreesView<'a, S> {
484        TokenTreesView::new(self.0)
485    }
486
487    pub fn iter(&self) -> TtIter<'a, S> {
488        TtIter::new(&self.0[1..])
489    }
490
491    pub fn top_subtree(&self) -> &'a Subtree<S> {
492        let TokenTree::Subtree(subtree) = &self.0[0] else {
493            unreachable!("the first token tree is always the top subtree");
494        };
495        subtree
496    }
497
498    pub fn strip_invisible(&self) -> TokenTreesView<'a, S> {
499        if self.top_subtree().delimiter.kind == DelimiterKind::Invisible {
500            TokenTreesView::new(&self.0[1..])
501        } else {
502            TokenTreesView::new(self.0)
503        }
504    }
505
506    pub fn token_trees(&self) -> TokenTreesView<'a, S> {
507        TokenTreesView::new(&self.0[1..])
508    }
509}
510
511impl<S: fmt::Debug + Copy> fmt::Debug for SubtreeView<'_, S> {
512    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
513        fmt::Debug::fmt(&TokenTreesView(self.0), f)
514    }
515}
516
517impl<S: Copy> fmt::Display for SubtreeView<'_, S> {
518    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
519        fmt::Display::fmt(&TokenTreesView(self.0), f)
520    }
521}
522
523#[derive(Debug, Copy, Clone, PartialEq)]
524pub struct DelimSpan<S> {
525    pub open: S,
526    pub close: S,
527}
528
529impl<Span: Copy> DelimSpan<Span> {
530    pub fn from_single(sp: Span) -> Self {
531        DelimSpan { open: sp, close: sp }
532    }
533
534    pub fn from_pair(open: Span, close: Span) -> Self {
535        DelimSpan { open, close }
536    }
537}
538#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
539pub struct Delimiter<S> {
540    pub open: S,
541    pub close: S,
542    pub kind: DelimiterKind,
543}
544
545impl<S: Copy> Delimiter<S> {
546    pub const fn invisible_spanned(span: S) -> Self {
547        Delimiter { open: span, close: span, kind: DelimiterKind::Invisible }
548    }
549
550    pub const fn invisible_delim_spanned(span: DelimSpan<S>) -> Self {
551        Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible }
552    }
553
554    pub fn delim_span(&self) -> DelimSpan<S> {
555        DelimSpan { open: self.open, close: self.close }
556    }
557}
558
559#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
560pub enum DelimiterKind {
561    Parenthesis,
562    Brace,
563    Bracket,
564    Invisible,
565}
566
567#[derive(Debug, Clone, PartialEq, Eq, Hash)]
568pub struct Literal<S> {
569    // escaped
570    pub symbol: Symbol,
571    pub span: S,
572    pub kind: LitKind,
573    pub suffix: Option<Symbol>,
574}
575
576pub fn token_to_literal<S>(text: &str, span: S) -> Literal<S>
577where
578    S: Copy,
579{
580    use rustc_lexer::LiteralKind;
581
582    let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple();
583    let Some((rustc_lexer::Token {
584        kind: rustc_lexer::TokenKind::Literal { kind, suffix_start },
585        ..
586    },)) = token
587    else {
588        return Literal {
589            span,
590            symbol: Symbol::intern(text),
591            kind: LitKind::Err(()),
592            suffix: None,
593        };
594    };
595
596    let (kind, start_offset, end_offset) = match kind {
597        LiteralKind::Int { .. } => (LitKind::Integer, 0, 0),
598        LiteralKind::Float { .. } => (LitKind::Float, 0, 0),
599        LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize),
600        LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize),
601        LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize),
602        LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize),
603        LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize),
604        LiteralKind::RawStr { n_hashes } => (
605            LitKind::StrRaw(n_hashes.unwrap_or_default()),
606            2 + n_hashes.unwrap_or_default() as usize,
607            1 + n_hashes.unwrap_or_default() as usize,
608        ),
609        LiteralKind::RawByteStr { n_hashes } => (
610            LitKind::ByteStrRaw(n_hashes.unwrap_or_default()),
611            3 + n_hashes.unwrap_or_default() as usize,
612            1 + n_hashes.unwrap_or_default() as usize,
613        ),
614        LiteralKind::RawCStr { n_hashes } => (
615            LitKind::CStrRaw(n_hashes.unwrap_or_default()),
616            3 + n_hashes.unwrap_or_default() as usize,
617            1 + n_hashes.unwrap_or_default() as usize,
618        ),
619    };
620
621    let (lit, suffix) = text.split_at(suffix_start as usize);
622    let lit = &lit[start_offset..lit.len() - end_offset];
623    let suffix = match suffix {
624        "" | "_" => None,
625        // ill-suffixed literals
626        _ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => {
627            return Literal {
628                span,
629                symbol: Symbol::intern(text),
630                kind: LitKind::Err(()),
631                suffix: None,
632            };
633        }
634        suffix => Some(Symbol::intern(suffix)),
635    };
636
637    Literal { span, symbol: Symbol::intern(lit), kind, suffix }
638}
639
640#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
641pub struct Punct<S> {
642    pub char: char,
643    pub spacing: Spacing,
644    pub span: S,
645}
646
647/// Indicates whether a token can join with the following token to form a
648/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
649/// guide pretty-printing, which is where the `JointHidden` value (which isn't
650/// part of `proc_macro::Spacing`) comes in useful.
651#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
652pub enum Spacing {
653    /// The token cannot join with the following token to form a compound
654    /// token.
655    ///
656    /// In token streams parsed from source code, the compiler will use `Alone`
657    /// for any token immediately followed by whitespace, a non-doc comment, or
658    /// EOF.
659    ///
660    /// When constructing token streams within the compiler, use this for each
661    /// token that (a) should be pretty-printed with a space after it, or (b)
662    /// is the last token in the stream. (In the latter case the choice of
663    /// spacing doesn't matter because it is never used for the last token. We
664    /// arbitrarily use `Alone`.)
665    ///
666    /// Converts to `proc_macro::Spacing::Alone`, and
667    /// `proc_macro::Spacing::Alone` converts back to this.
668    Alone,
669
670    /// The token can join with the following token to form a compound token.
671    ///
672    /// In token streams parsed from source code, the compiler will use `Joint`
673    /// for any token immediately followed by punctuation (as determined by
674    /// `Token::is_punct`).
675    ///
676    /// When constructing token streams within the compiler, use this for each
677    /// token that (a) should be pretty-printed without a space after it, and
678    /// (b) is followed by a punctuation token.
679    ///
680    /// Converts to `proc_macro::Spacing::Joint`, and
681    /// `proc_macro::Spacing::Joint` converts back to this.
682    Joint,
683
684    /// The token can join with the following token to form a compound token,
685    /// but this will not be visible at the proc macro level. (This is what the
686    /// `Hidden` means; see below.)
687    ///
688    /// In token streams parsed from source code, the compiler will use
689    /// `JointHidden` for any token immediately followed by anything not
690    /// covered by the `Alone` and `Joint` cases: an identifier, lifetime,
691    /// literal, delimiter, doc comment.
692    ///
693    /// When constructing token streams, use this for each token that (a)
694    /// should be pretty-printed without a space after it, and (b) is followed
695    /// by a non-punctuation token.
696    ///
697    /// Converts to `proc_macro::Spacing::Alone`, but
698    /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`.
699    /// Because of that, pretty-printing of `TokenStream`s produced by proc
700    /// macros is unavoidably uglier (with more whitespace between tokens) than
701    /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed
702    /// source code, internally constructed token streams, and token streams
703    /// produced by declarative macros).
704    JointHidden,
705}
706
707/// Identifier or keyword.
708#[derive(Debug, Clone, PartialEq, Eq, Hash)]
709pub struct Ident<S> {
710    pub sym: Symbol,
711    pub span: S,
712    pub is_raw: IdentIsRaw,
713}
714
715impl<S> Ident<S> {
716    pub fn new(text: &str, span: S) -> Self {
717        // let raw_stripped = IdentIsRaw::split_from_symbol(text.as_ref());
718        let (is_raw, text) = IdentIsRaw::split_from_symbol(text);
719        Ident { sym: Symbol::intern(text), span, is_raw }
720    }
721}
722
723fn print_debug_subtree<S: fmt::Debug>(
724    f: &mut fmt::Formatter<'_>,
725    subtree: &Subtree<S>,
726    level: usize,
727    iter: TtIter<'_, S>,
728) -> fmt::Result {
729    let align = "  ".repeat(level);
730
731    let Delimiter { kind, open, close } = &subtree.delimiter;
732    let delim = match kind {
733        DelimiterKind::Invisible => "$$",
734        DelimiterKind::Parenthesis => "()",
735        DelimiterKind::Brace => "{}",
736        DelimiterKind::Bracket => "[]",
737    };
738
739    write!(f, "{align}SUBTREE {delim} ",)?;
740    write!(f, "{open:#?}")?;
741    write!(f, " ")?;
742    write!(f, "{close:#?}")?;
743    for child in iter {
744        writeln!(f)?;
745        print_debug_token(f, level + 1, child)?;
746    }
747
748    Ok(())
749}
750
751fn print_debug_token<S: fmt::Debug>(
752    f: &mut fmt::Formatter<'_>,
753    level: usize,
754    tt: TtElement<'_, S>,
755) -> fmt::Result {
756    let align = "  ".repeat(level);
757
758    match tt {
759        TtElement::Leaf(leaf) => match leaf {
760            Leaf::Literal(lit) => {
761                write!(
762                    f,
763                    "{}LITERAL {:?} {}{} {:#?}",
764                    align,
765                    lit.kind,
766                    lit.symbol,
767                    lit.suffix.as_ref().map(|it| it.as_str()).unwrap_or(""),
768                    lit.span
769                )?;
770            }
771            Leaf::Punct(punct) => {
772                write!(
773                    f,
774                    "{}PUNCH   {} [{}] {:#?}",
775                    align,
776                    punct.char,
777                    if punct.spacing == Spacing::Alone { "alone" } else { "joint" },
778                    punct.span
779                )?;
780            }
781            Leaf::Ident(ident) => {
782                write!(
783                    f,
784                    "{}IDENT   {}{} {:#?}",
785                    align,
786                    ident.is_raw.as_str(),
787                    ident.sym,
788                    ident.span
789                )?;
790            }
791        },
792        TtElement::Subtree(subtree, subtree_iter) => {
793            print_debug_subtree(f, subtree, level, subtree_iter)?;
794        }
795    }
796
797    Ok(())
798}
799
800impl<S: fmt::Debug + Copy> fmt::Debug for TopSubtree<S> {
801    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
802        fmt::Debug::fmt(&self.view(), f)
803    }
804}
805
806impl<S: fmt::Display + Copy> fmt::Display for TopSubtree<S> {
807    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
808        fmt::Display::fmt(&self.view(), f)
809    }
810}
811
812impl<S> fmt::Display for Leaf<S> {
813    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
814        match self {
815            Leaf::Ident(it) => fmt::Display::fmt(it, f),
816            Leaf::Literal(it) => fmt::Display::fmt(it, f),
817            Leaf::Punct(it) => fmt::Display::fmt(it, f),
818        }
819    }
820}
821
822impl<S> fmt::Display for Ident<S> {
823    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
824        fmt::Display::fmt(&self.is_raw.as_str(), f)?;
825        fmt::Display::fmt(&self.sym, f)
826    }
827}
828
829impl<S> fmt::Display for Literal<S> {
830    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
831        match self.kind {
832            LitKind::Byte => write!(f, "b'{}'", self.symbol),
833            LitKind::Char => write!(f, "'{}'", self.symbol),
834            LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", self.symbol),
835            LitKind::Str => write!(f, "\"{}\"", self.symbol),
836            LitKind::ByteStr => write!(f, "b\"{}\"", self.symbol),
837            LitKind::CStr => write!(f, "c\"{}\"", self.symbol),
838            LitKind::StrRaw(num_of_hashes) => {
839                let num_of_hashes = num_of_hashes as usize;
840                write!(
841                    f,
842                    r#"r{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
843                    "",
844                    text = self.symbol
845                )
846            }
847            LitKind::ByteStrRaw(num_of_hashes) => {
848                let num_of_hashes = num_of_hashes as usize;
849                write!(
850                    f,
851                    r#"br{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
852                    "",
853                    text = self.symbol
854                )
855            }
856            LitKind::CStrRaw(num_of_hashes) => {
857                let num_of_hashes = num_of_hashes as usize;
858                write!(
859                    f,
860                    r#"cr{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
861                    "",
862                    text = self.symbol
863                )
864            }
865        }?;
866        if let Some(suffix) = &self.suffix {
867            write!(f, "{suffix}")?;
868        }
869        Ok(())
870    }
871}
872
873impl<S> fmt::Display for Punct<S> {
874    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
875        fmt::Display::fmt(&self.char, f)
876    }
877}
878
879impl<S> Subtree<S> {
880    /// Count the number of tokens recursively
881    pub fn count(&self) -> usize {
882        self.usize_len()
883    }
884}
885
886impl<S> TopSubtree<S> {
887    /// A simple line string used for debugging
888    pub fn subtree_as_debug_string(&self, subtree_idx: usize) -> String {
889        fn debug_subtree<S>(
890            output: &mut String,
891            subtree: &Subtree<S>,
892            iter: &mut std::slice::Iter<'_, TokenTree<S>>,
893        ) {
894            let delim = match subtree.delimiter.kind {
895                DelimiterKind::Brace => ("{", "}"),
896                DelimiterKind::Bracket => ("[", "]"),
897                DelimiterKind::Parenthesis => ("(", ")"),
898                DelimiterKind::Invisible => ("$", "$"),
899            };
900
901            output.push_str(delim.0);
902            let mut last = None;
903            let mut idx = 0;
904            while idx < subtree.len {
905                let child = iter.next().unwrap();
906                debug_token_tree(output, child, last, iter);
907                last = Some(child);
908                idx += 1;
909            }
910
911            output.push_str(delim.1);
912        }
913
914        fn debug_token_tree<S>(
915            output: &mut String,
916            tt: &TokenTree<S>,
917            last: Option<&TokenTree<S>>,
918            iter: &mut std::slice::Iter<'_, TokenTree<S>>,
919        ) {
920            match tt {
921                TokenTree::Leaf(it) => {
922                    let s = match it {
923                        Leaf::Literal(it) => it.symbol.to_string(),
924                        Leaf::Punct(it) => it.char.to_string(),
925                        Leaf::Ident(it) => format!("{}{}", it.is_raw.as_str(), it.sym),
926                    };
927                    match (it, last) {
928                        (Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => {
929                            output.push(' ');
930                            output.push_str(&s);
931                        }
932                        (Leaf::Punct(_), Some(TokenTree::Leaf(Leaf::Punct(punct)))) => {
933                            if punct.spacing == Spacing::Alone {
934                                output.push(' ');
935                                output.push_str(&s);
936                            } else {
937                                output.push_str(&s);
938                            }
939                        }
940                        _ => output.push_str(&s),
941                    }
942                }
943                TokenTree::Subtree(it) => debug_subtree(output, it, iter),
944            }
945        }
946
947        let mut res = String::new();
948        debug_token_tree(
949            &mut res,
950            &self.0[subtree_idx],
951            None,
952            &mut self.0[subtree_idx + 1..].iter(),
953        );
954        res
955    }
956}
957
958pub fn pretty<S>(mut tkns: &[TokenTree<S>]) -> String {
959    fn tokentree_to_text<S>(tkn: &TokenTree<S>, tkns: &mut &[TokenTree<S>]) -> String {
960        match tkn {
961            TokenTree::Leaf(Leaf::Ident(ident)) => {
962                format!("{}{}", ident.is_raw.as_str(), ident.sym)
963            }
964            TokenTree::Leaf(Leaf::Literal(literal)) => format!("{literal}"),
965            TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char),
966            TokenTree::Subtree(subtree) => {
967                let (subtree_content, rest) = tkns.split_at(subtree.usize_len());
968                let content = pretty(subtree_content);
969                *tkns = rest;
970                let (open, close) = match subtree.delimiter.kind {
971                    DelimiterKind::Brace => ("{", "}"),
972                    DelimiterKind::Bracket => ("[", "]"),
973                    DelimiterKind::Parenthesis => ("(", ")"),
974                    DelimiterKind::Invisible => ("", ""),
975                };
976                format!("{open}{content}{close}")
977            }
978        }
979    }
980
981    let mut last = String::new();
982    let mut last_to_joint = true;
983
984    while let Some((tkn, rest)) = tkns.split_first() {
985        tkns = rest;
986        last = [last, tokentree_to_text(tkn, &mut tkns)].join(if last_to_joint { "" } else { " " });
987        last_to_joint = false;
988        if let TokenTree::Leaf(Leaf::Punct(punct)) = tkn
989            && punct.spacing == Spacing::Joint
990        {
991            last_to_joint = true;
992        }
993    }
994    last
995}
ra_ap_tt/lib.rs

ra_ap_tt/
lib.rs
