easy_macros_helpers/
readable_token_stream.rs

1/// Formats a token stream string by removing unnecessary whitespace while preserving readability.
2///
3/// This function processes the string representation of token streams to remove
4/// redundant spaces while keeping necessary spacing for readability. It's particularly
5/// useful for cleaning up generated code or preparing token streams for debugging.
6///
7/// # Arguments
8///
9/// * `tokens_str` - A string representation of tokens (e.g., from `TokenStream :: to_string ()`)
10///
11/// # Returns
12///
13/// A cleaned string with unnecessary whitespace removed but readability preserved
14///
15/// # Whitespace Rules
16///
17/// The function removes spaces in these cases:
18/// - Multiple consecutive spaces are collapsed to one
19/// - Spaces after opening delimiters: `(`, `!`, `&`, `[`, `<`, `>`, `.`
20/// - Spaces before closing delimiters and punctuation: `.`, `,`, `(`, `[`, `:`, `;`, `!`, `<`, `>`, `?`
21/// - Spaces between consecutive closing delimiters: `))`, `}}`, `]]`
22///
23/// # Examples
24///
25#[doc = docify::embed!("src/examples.rs", readable_token_stream_example)]
26/// # Use Cases
27///
28/// - Cleaning up token streams for debugging output
29/// - Formatting generated code for better readability
30/// - Preparing code for display in error messages
31///
32/// # Safety
33///
34/// This function includes an assertion to ensure that only whitespace is removed,
35/// not actual token content. If this assertion fails, it indicates a bug in the
36/// whitespace removal logic.
37pub fn readable_token_stream(tokens_str: &str) -> String {
38    let mut result = String::new();
39
40    let mut char_iter_future = tokens_str.chars();
41    char_iter_future.next();
42
43    let char_iter_current = tokens_str.chars();
44
45    let char_iter_future = char_iter_future.map(Some).chain(std::iter::once(None));
46
47    let iters_zipped = char_iter_current.zip(char_iter_future);
48
49    let mut last_char = ' ';
50
51    for (c, future_c) in iters_zipped {
52        match c {
53            ' ' => {
54                if last_char == ' ' {
55                    continue;
56                }
57                match (last_char, future_c) {
58                    ('>', Some('>' | '(' | '{' | '[' | ',' | ']' | ':' | ';')) => {
59                        continue;
60                    }
61                    ('>', _) => {
62                        result.push(c);
63                        last_char = c;
64                    }
65                    ('(' | '!' | '&' | '[' | '<' | '.', _)
66                    | (_, None | Some('.' | ',' | '(' | '[' | ':' | ';' | '!' | '<' | '>' | '?'))
67                    | (')', Some(')'))
68                    | ('}', Some('}'))
69                    | (']', Some(']')) => {
70                        continue;
71                    }
72                    _ => {
73                        result.push(' ');
74                        last_char = ' ';
75                    }
76                }
77            }
78            _ => {
79                result.push(c);
80                last_char = c;
81            }
82        }
83    }
84
85    //Test if we only removed whitespace
86    #[cfg(test)]
87    assert_eq!(
88        result.replace(|c: char| c.is_whitespace(), ""),
89        tokens_str.replace(|c: char| c.is_whitespace(), ""),
90        "Only whitespace should be removed from token stream | Result: `{result}` | Original: `{tokens_str}`"
91    );
92
93    result
94}
easy_macros_helpers/readable_token_stream.rs

easy_macros_helpers/
readable_token_stream.rs
