easy_macros_helpers/
token_stream_to_consistent_string.rs

1use proc_macro2::{Delimiter, TokenStream, TokenTree};
2
3/// Converts a token stream to a consistent string representation without spaces.
4///
5/// The standard `TokenStream::to_string()` method can produce inconsistent output
6/// depending on the context - sometimes including spaces between tokens, sometimes not.
7/// This function ensures a consistent, space-free representation that's reliable
8/// for comparisons, hashing, or other operations requiring deterministic output.
9///
10/// # Context-Dependent Behavior of `TokenStream::to_string()`
11///
12/// - When operating on `syn::File`: spaces generally appear between tokens
13/// - Inside procedural macros: spaces generally don't appear
14/// - Other contexts: behavior may vary
15///
16/// # Arguments
17///
18/// * `tokens` - The token stream to convert to a consistent string
19///
20/// # Returns
21///
22/// A string representation with no spaces between tokens, ensuring consistent
23/// output regardless of the original context
24///
25/// # Examples
26///
27#[doc = docify::embed!("src/examples.rs", token_stream_consistent_string_example)]
28///
29/// # Token Processing
30///
31/// The function handles different token types:
32/// - **Groups**: Processes delimiters (`()`, `{}`, `[]`) and recursively processes contents
33/// - **Identifiers**: Removes leading/trailing whitespace
34/// - **Punctuation**: Removes leading/trailing whitespace  
35/// - **Literals**: Removes leading/trailing whitespace
36///
37/// # Use Cases
38///
39/// - Comparing token streams for equality regardless of spacing
40/// - Debugging token streams with predictable output
41pub fn token_stream_to_consistent_string(tokens: TokenStream) -> String {
42    let mut result_str = String::new();
43
44    for token in tokens.into_iter() {
45        match token {
46            TokenTree::Group(group) => {
47                match group.delimiter() {
48                    Delimiter::Parenthesis => {
49                        result_str.push('(');
50                    }
51                    Delimiter::Brace => {
52                        result_str.push('{');
53                    }
54                    Delimiter::Bracket => {
55                        result_str.push('[');
56                    }
57                    Delimiter::None => {}
58                }
59                result_str.push_str(&token_stream_to_consistent_string(group.stream()));
60                match group.delimiter() {
61                    Delimiter::Parenthesis => {
62                        result_str.push(')');
63                    }
64                    Delimiter::Brace => {
65                        result_str.push('}');
66                    }
67                    Delimiter::Bracket => {
68                        result_str.push(']');
69                    }
70                    Delimiter::None => {}
71                }
72            }
73            TokenTree::Ident(ident) => {
74                result_str.push_str(ident.to_string().trim_start().trim_end());
75            }
76            TokenTree::Punct(punct) => {
77                result_str.push_str(punct.to_string().trim_start().trim_end());
78            }
79            TokenTree::Literal(literal) => {
80                result_str.push_str(literal.to_string().trim_start().trim_end());
81            }
82        }
83    }
84
85    result_str
86}
easy_macros_helpers/token_stream_to_consistent_string.rs

easy_macros_helpers/
token_stream_to_consistent_string.rs
