tokel_std/string.rs
1//! String and text-manipulation Tokel [`Transformer`]s.
2//!
3//! This module provides transformers for modifying the textual representation
4//! and casing of token streams.
5//!
6//! # Available Transformers
7//!
8//! | Transformer | Argument Type | Description |
9//! |-----------------|-------------------------|-------------|
10//! | [`Concatenate`] | [`syn::parse::Nothing`] | Concatenates all input tokens into a single identifier or group. |
11//! | [`Case`] | [`CaseStyle`] | Converts identifiers and string-like tokens to a target case style. |
12//!
13//! # Argument Types
14//!
15//! * [`syn::parse::Nothing`] - No argument is required.
16//! * [`CaseStyle`] - A specific case formatting rule: `pascal`, `camel`, or `snake`.
17//!
18//! # Examples
19//!
20//! **Basic Usage:**
21//! * `[< hello _ world >]:concatenate` -> `hello_world`
22//! * `[< hello _ world >]:case[[pascal]]` -> `Hello _ World`
23//! * `[< some_value >]:case[[camel]]` -> `someValue`
24//!
25//! **Nested & Composed Usage:**
26//! Transformers can be evaluated inside arguments of other transformers. Inner expressions are always evaluated first.
27//! * `[< a b c >]:intersperse[[[< x y >]:concatenate]]` -> `a xy b xy c`
28//! * `[< a b >]:push_left[[[< hello world >]:concatenate]]` -> `helloworld a b`
29//! * `[< greet >]:push_right[[[< hello world >]:case[[pascal]]]]` -> `greet HelloWorld`
30//!
31//! **Literal Transformations:**
32//! Case transformations apply seamlessly to string literals and identifiers alike:
33//! * `[< "hello" world >]:case[[snake]]` -> `hello world`
34//!
35//! # Remarks
36//!
37//! * [`Concatenate`] directly glues the textual representations of tokens together. Token groups are processed recursively, meaning any nested tokens are flattened into the final result.
38//! * [`Case`] targets identifier-like tokens, string literals, and boolean literals. It safely preserves punctuation and non-identifier tokens where possible.
39
40use std::{
41 iter::{self, Peekable},
42 str::FromStr,
43};
44
45use proc_macro2::{Group, Ident, Literal, TokenStream, TokenTree};
46
47use quote::ToTokens;
48
49use syn::{
50 Lit,
51 parse::{Nothing, Parse, ParseStream},
52 spanned::Spanned,
53};
54
55use heck::{AsLowerCamelCase, AsPascalCase, AsSnekCase};
56
57use tokel_engine::prelude::{Pass, Registry, Transformer};
58
59/// A transformer that concatenates all elegible input tokens into a single identifier.
60///
61/// It ignores standard spacing and simply glues the string representations
62/// of the tokens together.
63///
64/// By "token", this implies identifier and string literals (not including byte literals, c-strings, or other string type).
65///
66/// This performs a rolling approach, physically contiguous tokens of the same type will be concatenated into one of the same token type.
67///
68/// # Example
69///
70/// `[< hello _ world "what" "ever" . "buddy" >]:concatenate` -> `hello_world "whatever" . "buddy"`
71#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
72pub struct Concatenate;
73
74impl Pass for Concatenate {
75 type Argument = Nothing;
76
77 fn through(&mut self, input: TokenStream, _: Self::Argument) -> syn::Result<TokenStream> {
78 struct ConcatIter(Peekable<<TokenStream as IntoIterator>::IntoIter>);
79
80 impl ConcatIter {
81 fn stream(stream: TokenStream) -> syn::Result<TokenStream> {
82 let mut nested_iter = Self(stream.into_iter().peekable());
83
84 let mut nested_tokens = Vec::new();
85
86 loop {
87 match nested_iter.next() {
88 Some(Ok(tree)) => nested_tokens.push(tree),
89 Some(Err(error)) => return Err(error),
90 None => break,
91 }
92 }
93
94 Ok(nested_tokens.into_iter().collect::<TokenStream>())
95 }
96 }
97
98 impl Iterator for ConcatIter {
99 type Item = syn::Result<TokenTree>;
100
101 fn next(&mut self) -> Option<Self::Item> {
102 let Self(inner_iter) = self;
103
104 match inner_iter.peek() {
105 Some(TokenTree::Ident(..) | TokenTree::Literal(..) | TokenTree::Group(..)) => {
106 match inner_iter.next() {
107 Some(TokenTree::Ident(ident_start)) => {
108 let ref mut ident_str = String::new();
109
110 let ref mut ident_tokens = TokenStream::new();
111
112 ident_str.push_str(ident_start.to_string().as_str());
113 ident_tokens
114 .extend(iter::once(ident_start).map(Ident::into_token_stream));
115
116 while let Some(TokenTree::Ident(..)) = inner_iter.peek() {
117 let Some(TokenTree::Ident(ident_extra)) = inner_iter.next()
118 else {
119 unreachable!()
120 };
121
122 ident_tokens.extend(
123 iter::once(ident_extra.clone())
124 .map(Ident::into_token_stream),
125 );
126
127 ident_str.push_str(ident_extra.to_string().as_str());
128 }
129
130 let mut ident = syn::parse_str::<Ident>(ident_str).ok()?;
131
132 ident.set_span(ident_tokens.span());
133
134 Some(Ok(TokenTree::Ident(ident)))
135 }
136 Some(TokenTree::Literal(lit)) => {
137 if let Lit::Str(lit_str) = Lit::new(lit.clone()) {
138 let mut concatenated_str = lit_str.value();
139
140 while let Some(TokenTree::Literal(peeked_lit)) =
141 inner_iter.peek()
142 {
143 if let Lit::Str(peeked_str) = Lit::new(peeked_lit.clone()) {
144 let _ = inner_iter.next();
145
146 concatenated_str.push_str(peeked_str.value().as_str());
147 } else {
148 break;
149 }
150 }
151
152 Some(Ok(TokenTree::Literal(Literal::string(
153 concatenated_str.as_str(),
154 ))))
155 } else {
156 Some(Ok(TokenTree::Literal(lit)))
157 }
158 }
159 Some(TokenTree::Group(..)) => {
160 let Some(TokenTree::Group(inner_group)) = inner_iter.next() else {
161 unreachable!()
162 };
163
164 let (delimiter, stream, span) = (
165 inner_group.delimiter(),
166 inner_group.stream(),
167 inner_group.span(),
168 );
169
170 let stream = match Self::stream(stream) {
171 Ok(stream) => stream,
172 Err(error) => return Some(Err(error)),
173 };
174
175 let mut group = Group::new(delimiter, stream);
176
177 group.set_span(span);
178
179 Some(Ok(TokenTree::Group(group)))
180 }
181 Some(..) | None => unreachable!(),
182 }
183 }
184 Some(..) | None => inner_iter.next().map(Ok),
185 }
186 }
187 }
188
189 ConcatIter::stream(input)
190 }
191}
192
193/// The target case style to transform the identifiers to.
194#[derive(Debug, Copy, Clone)]
195pub enum CaseStyle {
196 /// `PascalCase`.
197 Pascal,
198
199 /// `camelCase`.
200 Camel,
201
202 /// `snake_case`.
203 Snake,
204
205 /// `UPPERCASE`
206 Upper,
207
208 /// `lowercase`
209 Lower,
210}
211
212impl Parse for CaseStyle {
213 fn parse(input: ParseStream) -> syn::Result<Self> {
214 let case_ident = input.parse::<Ident>()?;
215
216 let _: Nothing = input.parse()?;
217
218 match case_ident.to_string().as_str() {
219 "pascal" => Ok(Self::Pascal),
220 "camel" => Ok(Self::Camel),
221 "snake" => Ok(Self::Snake),
222 "upper" => Ok(Self::Upper),
223 "lower" => Ok(Self::Lower),
224 _ => {
225 return Err(syn::Error::new_spanned(
226 case_ident,
227 "unsupported case, supported ones are: `pascal`, `camel`, `snake`, `upper`, `lower`",
228 ));
229 }
230 }
231 }
232}
233
234/// A transformer that changes the case of incoming identifiers, as instructed.
235///
236/// # Example
237///
238/// `[< hello _ world >]:case[[pascal]]` -> `Hello _ World`
239#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
240pub struct Case;
241
242impl Pass for Case {
243 type Argument = CaseStyle;
244
245 fn through(&mut self, input: TokenStream, style: Self::Argument) -> syn::Result<TokenStream> {
246 fn apply_case(string: String, case: CaseStyle) -> String {
247 match case {
248 CaseStyle::Pascal => AsPascalCase(string).to_string(),
249 CaseStyle::Camel => AsLowerCamelCase(string).to_string(),
250 CaseStyle::Snake => AsSnekCase(string).to_string(),
251 CaseStyle::Upper => string.to_uppercase(),
252 CaseStyle::Lower => string.to_lowercase(),
253 }
254 }
255
256 fn apply(input: TokenStream, case: CaseStyle) -> syn::Result<TokenStream> {
257 input
258 .into_iter()
259 .try_fold(TokenStream::new(), |mut acc, target_tree| {
260 let target_output = match target_tree {
261 TokenTree::Literal(target_lit) => {
262 match syn::parse2::<Lit>(target_lit.into_token_stream())? {
263 Lit::Str(inner) => {
264 TokenStream::from_str(apply_case(inner.value(), case).as_str())?
265 }
266 Lit::Bool(lit) => TokenStream::from_str(
267 apply_case(lit.value.to_string(), case).as_str(),
268 )?,
269
270 lit @ _ => lit.into_token_stream(),
271 }
272 }
273 TokenTree::Ident(target_ident) => TokenStream::from_str(
274 apply_case(target_ident.to_string(), case).as_str(),
275 )?,
276 TokenTree::Group(group) => group
277 .stream()
278 .into_iter()
279 .map(|tree| apply(tree.into_token_stream(), case))
280 .try_fold(TokenStream::new(), |mut acc, result| {
281 result.map(|stream| {
282 acc.extend(stream);
283 acc
284 })
285 })
286 .map(|a| {
287 let mut new_group = Group::new(group.delimiter(), a);
288
289 new_group.set_span(group.span());
290
291 new_group
292 })
293 .map(TokenTree::Group)
294 .map(ToTokens::into_token_stream)?,
295
296 target_tree @ _ => target_tree.into_token_stream(),
297 };
298
299 acc.extend(target_output);
300
301 Ok(acc)
302 })
303 }
304
305 apply(input, style)
306 }
307}
308
309/// A transformer that converts every non-nested token tree into a string.
310///
311/// This does not further modify literals that are already strings.
312///
313/// # Example
314///
315/// `[< hello _ world >]:to_string` -> `"hello" "_" "world"`
316pub struct ToString;
317
318impl Pass for ToString {
319 type Argument = syn::parse::Nothing;
320
321 fn through(&mut self, input: TokenStream, _: Self::Argument) -> syn::Result<TokenStream> {
322 struct ToStringIter(<TokenStream as IntoIterator>::IntoIter);
323
324 impl ToStringIter {
325 fn stream(stream: TokenStream) -> TokenStream {
326 Self(stream.into_iter()).collect::<TokenStream>()
327 }
328 }
329
330 impl Iterator for ToStringIter {
331 type Item = TokenTree;
332
333 fn next(&mut self) -> Option<Self::Item> {
334 let Self(inner_iter) = self;
335
336 let Some(token_tree) = inner_iter.next() else {
337 return None;
338 };
339
340 Some(match token_tree {
341 TokenTree::Group(group) => {
342 let (delimiter, stream, span) =
343 (group.delimiter(), group.stream(), group.span());
344
345 let mut group = Group::new(delimiter, Self::stream(stream));
346
347 group.set_span(span);
348
349 TokenTree::Group(group)
350 }
351 TokenTree::Ident(ident) => {
352 let mut lit = Literal::string(ident.to_string().as_str());
353
354 lit.set_span(ident.span());
355
356 TokenTree::Literal(lit)
357 }
358 TokenTree::Punct(punct) => {
359 let mut lit = Literal::string(punct.to_string().as_str());
360
361 lit.set_span(punct.span());
362
363 TokenTree::Literal(lit)
364 }
365 TokenTree::Literal(literal) => {
366 // NOTE: If already a string-like literal, keep it as it is.
367 if let Lit::CStr(..) | Lit::ByteStr(..) | Lit::Char(..) | Lit::Str(..) =
368 Lit::new(literal.clone())
369 {
370 TokenTree::Literal(literal)
371 } else {
372 let mut lit = Literal::string(literal.to_string().as_str());
373
374 lit.set_span(literal.span());
375
376 TokenTree::Literal(lit)
377 }
378 }
379 })
380 }
381 }
382
383 Ok(ToStringIter::stream(input))
384 }
385}
386
387/// Inserts all `string`-related [`Transformer`]s into the specified [`Registry`].
388///
389/// # Errors
390///
391/// This will fail if at least one standard `string`-related [`Transformer`] is already present by-name in the [`Registry`].
392///
393/// On failure, there is no guarantee that other non-colliding transformers have not been registered.
394#[inline]
395pub fn register(registry: &mut Registry) -> Result<(), Box<dyn Transformer>> {
396 registry
397 .try_insert("concatenate", Concatenate)
398 .map_err(Box::new)
399 .map_err(|t| t as Box<dyn Transformer>)?;
400
401 registry
402 .try_insert("case", Case)
403 .map_err(Box::new)
404 .map_err(|t| t as Box<dyn Transformer>)?;
405
406 registry
407 .try_insert("to_string", ToString)
408 .map_err(Box::new)
409 .map_err(|t| t as Box<dyn Transformer>)?;
410
411 Ok(())
412}