tokel_std/string.rs
1//! String and text-manipulation Tokel [`Transformer`]s.
2//!
3//! This module provides transformers for modifying the textual representation
4//! and casing of token streams.
5//!
6//! # Available Transformers
7//!
8//! | Transformer | Argument Type | Description |
9//! |-----------------|-------------------------|-------------|
10//! | [`Concatenate`] | [`syn::parse::Nothing`] | Concatenates all input tokens into a single identifier or group. |
11//! | [`ToString`] | [`syn::parse::Nothing`] | Concatenates all input tokens into a single identifier or group. |
12//! | [`Case`] | [`CaseStyle`] | Converts identifiers and string-like tokens to a target case style. |
13//!
14//! # Argument Types
15//!
16//! * [`syn::parse::Nothing`] - No argument is required.
17//! * [`CaseStyle`] - A specific case formatting rule: `pascal`, `camel`, or `snake`.
18//!
19//! # Examples
20//!
21//! **Basic Usage:**
22//! * `[< hello _ world >]:concatenate` -> `hello_world`
23//! * `[< hello _ world >]:case[[pascal]]` -> `Hello _ World`
24//! * `[< some_value >]:case[[camel]]` -> `someValue`
25//!
26//! **Nested & Composed Usage:**
27//! Transformers can be evaluated inside arguments of other transformers. Inner expressions are always evaluated first.
28//! * `[< a b c >]:intersperse[[[< x y >]:concatenate]]` -> `a xy b xy c`
29//! * `[< a b >]:push_left[[[< hello world >]:concatenate]]` -> `helloworld a b`
30//! * `[< greet >]:push_right[[[< hello world >]:case[[pascal]]]]` -> `greet HelloWorld`
31//!
32//! **Literal Transformations:**
33//! Case transformations apply seamlessly to string literals and identifiers alike:
34//! * `[< "hello" world >]:case[[snake]]` -> `hello world`
35//!
36//! # Remarks
37//!
38//! * [`Concatenate`] directly glues the textual representations of tokens together. Token groups are processed recursively, meaning any nested tokens are flattened into the final result.
39//! * [`Case`] targets identifier-like tokens, string literals, and boolean literals. It safely preserves punctuation and non-identifier tokens where possible.
40
41use std::{
42 iter::{self, Peekable},
43 str::FromStr,
44};
45
46use proc_macro2::{Group, Ident, Literal, TokenStream, TokenTree};
47
48use quote::ToTokens;
49
50use syn::{
51 Lit,
52 parse::{Nothing, Parse, ParseStream},
53 spanned::Spanned,
54};
55
56use heck::{AsLowerCamelCase, AsPascalCase, AsSnekCase};
57
58use tokel_engine::prelude::{Pass, Registry, Transformer};
59
60/// A transformer that concatenates all elegible input tokens into a single identifier.
61///
62/// It ignores standard spacing and simply glues the string representations
63/// of the tokens together.
64///
65/// By "token", this implies identifier and string literals (not including byte literals, c-strings, or other string type).
66///
67/// This performs a rolling approach, physically contiguous tokens of the same type will be concatenated into one of the same token type.
68///
69/// # Example
70///
71/// `[< hello _ world "what" "ever" . "buddy" >]:concatenate` -> `hello_world "whatever" . "buddy"`
72#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
73pub struct Concatenate;
74
75impl Pass for Concatenate {
76 type Argument = Nothing;
77
78 fn through(&mut self, input: TokenStream, _: Self::Argument) -> syn::Result<TokenStream> {
79 struct ConcatIter(Peekable<<TokenStream as IntoIterator>::IntoIter>);
80
81 impl ConcatIter {
82 fn stream(stream: TokenStream) -> syn::Result<TokenStream> {
83 let mut nested_iter = Self(stream.into_iter().peekable());
84
85 let mut nested_tokens = Vec::new();
86
87 loop {
88 match nested_iter.next() {
89 Some(Ok(tree)) => nested_tokens.push(tree),
90 Some(Err(error)) => return Err(error),
91 None => break,
92 }
93 }
94
95 Ok(nested_tokens.into_iter().collect::<TokenStream>())
96 }
97 }
98
99 impl Iterator for ConcatIter {
100 type Item = syn::Result<TokenTree>;
101
102 fn next(&mut self) -> Option<Self::Item> {
103 let Self(inner_iter) = self;
104
105 match inner_iter.peek() {
106 Some(TokenTree::Ident(..) | TokenTree::Literal(..) | TokenTree::Group(..)) => {
107 match inner_iter.next() {
108 Some(TokenTree::Ident(ident_start)) => {
109 let ref mut ident_str = String::new();
110
111 let ref mut ident_tokens = TokenStream::new();
112
113 ident_str.push_str(ident_start.to_string().as_str());
114 ident_tokens
115 .extend(iter::once(ident_start).map(Ident::into_token_stream));
116
117 while let Some(TokenTree::Ident(..)) = inner_iter.peek() {
118 let Some(TokenTree::Ident(ident_extra)) = inner_iter.next()
119 else {
120 unreachable!()
121 };
122
123 ident_tokens.extend(
124 iter::once(ident_extra.clone())
125 .map(Ident::into_token_stream),
126 );
127
128 ident_str.push_str(ident_extra.to_string().as_str());
129 }
130
131 let mut ident = syn::parse_str::<Ident>(ident_str).ok()?;
132
133 ident.set_span(ident_tokens.span());
134
135 Some(Ok(TokenTree::Ident(ident)))
136 }
137 Some(TokenTree::Literal(lit)) => {
138 if let Lit::Str(lit_str) = Lit::new(lit.clone()) {
139 let mut concatenated_str = lit_str.value();
140
141 while let Some(TokenTree::Literal(peeked_lit)) =
142 inner_iter.peek()
143 {
144 if let Lit::Str(peeked_str) = Lit::new(peeked_lit.clone()) {
145 let Some(..) = inner_iter.next() else {
146 unreachable!()
147 };
148
149 concatenated_str.push_str(peeked_str.value().as_str());
150 } else {
151 break;
152 }
153 }
154
155 let mut lit = Literal::string(&concatenated_str);
156
157 lit.set_span(lit_str.span());
158
159 Some(Ok(TokenTree::Literal(lit)))
160 } else {
161 Some(Ok(TokenTree::Literal(lit)))
162 }
163 }
164 Some(TokenTree::Group(inner_group)) => {
165 let (delimiter, stream, span) = (
166 inner_group.delimiter(),
167 inner_group.stream(),
168 inner_group.span(),
169 );
170
171 let stream = match Self::stream(stream) {
172 Ok(stream) => stream,
173 Err(error) => return Some(Err(error)),
174 };
175
176 let mut group = Group::new(delimiter, stream);
177
178 group.set_span(span);
179
180 Some(Ok(TokenTree::Group(group)))
181 }
182 Some(..) | None => unreachable!(),
183 }
184 }
185 Some(..) | None => inner_iter.next().map(Ok),
186 }
187 }
188 }
189
190 ConcatIter::stream(input)
191 }
192}
193
194/// The target case style to transform the identifiers to.
195#[derive(Debug, Copy, Clone)]
196pub enum CaseStyle {
197 /// `PascalCase`.
198 Pascal,
199
200 /// `camelCase`.
201 Camel,
202
203 /// `snake_case`.
204 Snake,
205
206 /// `UPPERCASE`
207 Upper,
208
209 /// `lowercase`
210 Lower,
211}
212
213impl Parse for CaseStyle {
214 fn parse(input: ParseStream) -> syn::Result<Self> {
215 let case_ident = input.parse::<Ident>()?;
216
217 let _: Nothing = input.parse()?;
218
219 match case_ident.to_string().as_str() {
220 "pascal" => Ok(Self::Pascal),
221 "camel" => Ok(Self::Camel),
222 "snake" => Ok(Self::Snake),
223 "upper" => Ok(Self::Upper),
224 "lower" => Ok(Self::Lower),
225 _ => {
226 return Err(syn::Error::new_spanned(
227 case_ident,
228 "unsupported case, supported ones are: `pascal`, `camel`, `snake`, `upper`, `lower`",
229 ));
230 }
231 }
232 }
233}
234
235/// A transformer that changes the case of incoming identifiers, as instructed.
236///
237/// # Example
238///
239/// `[< hello _ world >]:case[[pascal]]` -> `Hello _ World`
240#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
241pub struct Case;
242
243impl Pass for Case {
244 type Argument = CaseStyle;
245
246 fn through(&mut self, input: TokenStream, style: Self::Argument) -> syn::Result<TokenStream> {
247 fn apply_case(string: String, case: CaseStyle) -> String {
248 match case {
249 CaseStyle::Pascal => AsPascalCase(string).to_string(),
250 CaseStyle::Camel => AsLowerCamelCase(string).to_string(),
251 CaseStyle::Snake => AsSnekCase(string).to_string(),
252 CaseStyle::Upper => string.to_uppercase(),
253 CaseStyle::Lower => string.to_lowercase(),
254 }
255 }
256
257 fn apply(input: TokenStream, case: CaseStyle) -> syn::Result<TokenStream> {
258 input
259 .into_iter()
260 .try_fold(TokenStream::new(), |mut acc, target_tree| {
261 let target_output = match target_tree {
262 TokenTree::Literal(target_lit) => {
263 match syn::parse2::<Lit>(target_lit.into_token_stream())? {
264 Lit::Str(inner) => {
265 TokenStream::from_str(apply_case(inner.value(), case).as_str())?
266 }
267 Lit::Bool(lit) => TokenStream::from_str(
268 apply_case(lit.value.to_string(), case).as_str(),
269 )?,
270
271 lit @ _ => lit.into_token_stream(),
272 }
273 }
274 TokenTree::Ident(target_ident) => TokenStream::from_str(
275 apply_case(target_ident.to_string(), case).as_str(),
276 )?,
277 TokenTree::Group(group) => group
278 .stream()
279 .into_iter()
280 .map(|tree| apply(tree.into_token_stream(), case))
281 .try_fold(TokenStream::new(), |mut acc, result| {
282 result.map(|stream| {
283 acc.extend(stream);
284 acc
285 })
286 })
287 .map(|a| {
288 let mut new_group = Group::new(group.delimiter(), a);
289
290 new_group.set_span(group.span());
291
292 new_group
293 })
294 .map(TokenTree::Group)
295 .map(ToTokens::into_token_stream)?,
296
297 target_tree @ _ => target_tree.into_token_stream(),
298 };
299
300 acc.extend(target_output);
301
302 Ok(acc)
303 })
304 }
305
306 apply(input, style)
307 }
308}
309
310/// A transformer that converts every non-nested token tree into a string.
311///
312/// This does not further modify literals that are already strings.
313///
314/// # Example
315///
316/// `[< hello _ world >]:to_string` -> `"hello" "_" "world"`
317pub struct ToString;
318
319impl Pass for ToString {
320 type Argument = syn::parse::Nothing;
321
322 fn through(&mut self, input: TokenStream, _: Self::Argument) -> syn::Result<TokenStream> {
323 struct ToStringIter(<TokenStream as IntoIterator>::IntoIter);
324
325 impl ToStringIter {
326 fn stream(stream: TokenStream) -> TokenStream {
327 Self(stream.into_iter()).collect::<TokenStream>()
328 }
329 }
330
331 impl Iterator for ToStringIter {
332 type Item = TokenTree;
333
334 fn next(&mut self) -> Option<Self::Item> {
335 let Self(inner_iter) = self;
336
337 let Some(token_tree) = inner_iter.next() else {
338 return None;
339 };
340
341 Some(match token_tree {
342 TokenTree::Group(group) => {
343 let (delimiter, stream, span) =
344 (group.delimiter(), group.stream(), group.span());
345
346 let mut group = Group::new(delimiter, Self::stream(stream));
347
348 group.set_span(span);
349
350 TokenTree::Group(group)
351 }
352 TokenTree::Ident(ident) => {
353 let mut lit = Literal::string(ident.to_string().as_str());
354
355 lit.set_span(ident.span());
356
357 TokenTree::Literal(lit)
358 }
359 TokenTree::Punct(punct) => {
360 let mut lit = Literal::string(punct.to_string().as_str());
361
362 lit.set_span(punct.span());
363
364 TokenTree::Literal(lit)
365 }
366 TokenTree::Literal(literal) => {
367 // NOTE: If already a string-like literal, keep it as it is.
368 if let Lit::CStr(..) | Lit::ByteStr(..) | Lit::Char(..) | Lit::Str(..) =
369 Lit::new(literal.clone())
370 {
371 TokenTree::Literal(literal)
372 } else {
373 let mut lit = Literal::string(literal.to_string().as_str());
374
375 lit.set_span(literal.span());
376
377 TokenTree::Literal(lit)
378 }
379 }
380 })
381 }
382 }
383
384 Ok(ToStringIter::stream(input))
385 }
386}
387
388/// Inserts all `string`-related [`Transformer`]s into the specified [`Registry`].
389///
390/// # Errors
391///
392/// This will fail if at least one standard `string`-related [`Transformer`] is already present by-name in the [`Registry`].
393///
394/// On failure, there is no guarantee that other non-colliding transformers have not been registered.
395#[inline]
396pub fn register(registry: &mut Registry) -> Result<(), Box<dyn Transformer>> {
397 registry
398 .try_insert("concatenate", Concatenate)
399 .map_err(Box::new)
400 .map_err(|t| t as Box<dyn Transformer>)?;
401
402 registry
403 .try_insert("case", Case)
404 .map_err(Box::new)
405 .map_err(|t| t as Box<dyn Transformer>)?;
406
407 registry
408 .try_insert("to_string", ToString)
409 .map_err(Box::new)
410 .map_err(|t| t as Box<dyn Transformer>)?;
411
412 Ok(())
413}