macro_rules_rt/
transcriber.rs

1use crate::{
2    matcher::{
3        MacroRepOp, MacroRepSep, MatchStringBuilder, MatchTokensBuilder, PatternItems, RawMatch,
4    },
5    token_entry::TokenStringBuilder,
6    utils::{to_close_str, to_open_str, RangeBuilder},
7    ParseStreamEx, Rule, Source,
8};
9use proc_macro2::{Delimiter, Group, Ident, Span, TokenStream, TokenTree};
10use quote::{ToTokens, TokenStreamExt};
11use std::{ops::Range, str::FromStr};
12use structmeta::{Parse, ToTokens};
13use syn::{
14    ext::IdentExt,
15    parse::{Parse, ParseStream},
16    spanned::Spanned,
17    token, Error, Result, Token,
18};
19
20/// Replacement pattern.
21///
22/// `Transcriber` corresponds to `MacroTranscriber` (excluding outermost brace) in [`Macros By Example`](https://doc.rust-lang.org/reference/macros-by-example.html).
23///
24/// A `Transcriber` created using [`FromStr::from_str`] preserves whitespace in the original string as much as possible.
25#[derive(Debug, Clone)]
26pub struct Transcriber {
27    items: TranscriberItems,
28    is_ready_string: bool,
29}
30
31impl FromStr for Transcriber {
32    type Err = Error;
33
34    fn from_str(s: &str) -> Result<Self> {
35        let (source, input) = Source::from_str(s)?;
36        let mut to = ParseStreamEx::parse_from_tokens(input, 0, Self::parse_ex)?;
37        to.items.ready_string(&source);
38        to.is_ready_string = true;
39        Ok(to)
40    }
41}
42
43impl Transcriber {
44    /// Parse a transcriber.
45    ///
46    /// Unlike [`FromStr::from_str`], non-token information, such as whitespace, is lost.
47    ///
48    /// `Transcriber` does not implement [`Parse`](syn::parse::Parse) to prevent information loss using [`parse_str`](syn::parse_str).
49    pub fn parse(input: ParseStream) -> Result<Self> {
50        Self::parse_ex(&mut ParseStreamEx::new(input, 0))
51    }
52    fn parse_ex(input: &mut ParseStreamEx) -> Result<Self> {
53        Ok(Self {
54            items: TranscriberItems::parse(input)?,
55            is_ready_string: false,
56        })
57    }
58    pub(crate) fn attach(&mut self, p: &PatternItems) -> Result<()> {
59        self.items.attach(p)
60    }
61    pub(crate) fn apply_tokens_to(&self, m: &RawMatch, b: &mut MatchTokensBuilder) {
62        self.items.apply_tokens_to(m, b)
63    }
64    pub(crate) fn apply_string(
65        &self,
66        m: &RawMatch,
67        rule: &Rule,
68        tes_len: usize,
69        b: &mut TokenStringBuilder,
70    ) {
71        let mut b = MatchStringBuilder {
72            b,
73            rule,
74            tes_len,
75            is_ready_string: self.is_ready_string,
76        };
77        self.items.apply_string(m, &mut b)
78    }
79}
80
81#[derive(Debug, Clone)]
82struct TranscriberItems {
83    items: Vec<TranscriberItem>,
84}
85
86impl TranscriberItems {
87    fn parse(input: &mut ParseStreamEx) -> Result<Self> {
88        let mut items = Vec::new();
89        let mut tokens = Vec::new();
90        let mut tes_range = RangeBuilder::new();
91        while !input.is_empty() {
92            if input.peek(token::Paren) || input.peek(token::Brace) || input.peek(token::Bracket) {
93                push_tokens(&mut tokens, &mut items, &mut tes_range);
94                let g = input.parse_group(|g, input| {
95                    Ok(TranscriberGroup {
96                        delimiter: g.group.delimiter(),
97                        content: Self::parse(input)?,
98                        tes_range_open: g.tes_range_open,
99                        tes_range_close: g.tes_range_close,
100                        span: g.group.span(),
101                    })
102                })?;
103                items.push(TranscriberItem::Group(g));
104                continue;
105            }
106            if input.peek(Token![$]) {
107                if input.peek2(Ident::peek_any) {
108                    push_tokens(&mut tokens, &mut items, &mut tes_range);
109                    items.push(TranscriberItem::Var(input.parse()?));
110                    continue;
111                }
112                if input.peek2(token::Paren) {
113                    push_tokens(&mut tokens, &mut items, &mut tes_range);
114                    items.push(TranscriberItem::Rep(TranscriberRep::parse(input)?));
115                    continue;
116                }
117            }
118            let tes_start = input.tes_offset;
119            let token: TokenTree = input.parse().unwrap();
120            let tes_end = input.tes_offset;
121            tes_range.push(tes_start..tes_end);
122            tokens.push(token);
123        }
124        push_tokens(&mut tokens, &mut items, &mut tes_range);
125        Ok(Self { items })
126    }
127    fn ready_string(&mut self, source: &Source) {
128        self.ready_string_with(source, &mut RangeBuilder::new());
129    }
130    fn ready_string_with(&mut self, source: &Source, tes_range: &mut RangeBuilder) {
131        for item in &mut self.items {
132            item.ready_string_with(source, tes_range);
133        }
134    }
135    fn attach(&mut self, p: &PatternItems) -> Result<()> {
136        for i in &mut self.items {
137            i.attach(p)?;
138        }
139        Ok(())
140    }
141    fn get_var(&self) -> Option<MacroTranscriberVar> {
142        for i in &self.items {
143            if let Some(b) = i.get_var() {
144                return Some(b);
145            }
146        }
147        None
148    }
149    fn apply_tokens_to(&self, m: &RawMatch, b: &mut MatchTokensBuilder) {
150        for item in &self.items {
151            item.apply_tokens_to(m, b);
152        }
153    }
154    fn apply_string(&self, m: &RawMatch, b: &mut MatchStringBuilder) {
155        for item in &self.items {
156            item.apply_string(m, b)
157        }
158    }
159}
160fn push_tokens(
161    tokens: &mut Vec<TokenTree>,
162    items: &mut Vec<TranscriberItem>,
163    tes_range: &mut RangeBuilder,
164) {
165    if let Some(tes_range) = tes_range.take() {
166        if !tokens.is_empty() {
167            let tokens = TokenStream::from_iter(tokens.drain(..));
168            items.push(TranscriberItem::Tokens(TranscriberTokens {
169                tokens,
170                tes_range,
171            }));
172        }
173    }
174    items.push(TranscriberItem::String(String::new()));
175}
176
177#[derive(Debug, Clone)]
178enum TranscriberItem {
179    Tokens(TranscriberTokens),
180    Group(TranscriberGroup),
181    String(String),
182    Var(TranscriberVar),
183    Rep(TranscriberRep),
184}
185impl TranscriberItem {
186    fn ready_string_with(&mut self, source: &Source, tes_range: &mut RangeBuilder) {
187        match self {
188            Self::Tokens(t) => tes_range.push(t.tes_range.clone()),
189            Self::Group(g) => g.ready_string_with(source, tes_range),
190            Self::String(ref mut s) => {
191                if let Some(tes_range) = tes_range.take() {
192                    let mut b = TokenStringBuilder::new(source);
193                    b.push_tes(tes_range);
194                    *s = b.s;
195                }
196            }
197            Self::Var(_) => {}
198            Self::Rep(r) => r.content.ready_string(source),
199        }
200    }
201    fn attach(&mut self, p: &PatternItems) -> Result<()> {
202        match self {
203            Self::Tokens(_) | Self::String(_) => Ok(()),
204            Self::Group(g) => g.content.attach(p),
205            Self::Var(v) => v.attach(p),
206            Self::Rep(r) => r.attach(p),
207        }
208    }
209    fn get_var(&self) -> Option<MacroTranscriberVar> {
210        match self {
211            TranscriberItem::Tokens(_) => None,
212            TranscriberItem::Group(g) => g.content.get_var(),
213            TranscriberItem::String(_) => None,
214            TranscriberItem::Var(v) => Some(v.var.clone()),
215            TranscriberItem::Rep(r) => Some(r.var.clone()),
216        }
217    }
218    fn apply_tokens_to(&self, m: &RawMatch, b: &mut MatchTokensBuilder) {
219        match self {
220            TranscriberItem::Tokens(t) => t.tokens.to_tokens(b.tokens),
221            TranscriberItem::String(_) => {}
222            TranscriberItem::Group(g) => g.apply_tokens_to(m, b),
223            TranscriberItem::Var(v) => v.apply_tokens_to(m, b),
224            TranscriberItem::Rep(r) => r.apply_tokens_to(m, b),
225        }
226    }
227
228    fn apply_string(&self, m: &RawMatch, b: &mut MatchStringBuilder) {
229        match self {
230            TranscriberItem::Tokens(tokens) => tokens.apply_string(b),
231            TranscriberItem::Group(g) => g.apply_string(m, b),
232            TranscriberItem::String(s) => b.b.push_str(s),
233            TranscriberItem::Var(v) => v.apply_string(m, b),
234            TranscriberItem::Rep(r) => r.apply_string(m, b),
235        }
236    }
237}
238
239#[derive(Debug, Clone)]
240struct TranscriberTokens {
241    tokens: TokenStream,
242    tes_range: Range<usize>,
243}
244impl TranscriberTokens {
245    fn apply_string(&self, b: &mut MatchStringBuilder) {
246        if !b.is_ready_string {
247            b.b.push_tokens(&self.tokens)
248        }
249    }
250}
251
252#[derive(Debug, Clone)]
253struct TranscriberGroup {
254    delimiter: Delimiter,
255    content: TranscriberItems,
256    span: Span,
257    tes_range_open: Range<usize>,
258    tes_range_close: Range<usize>,
259}
260impl TranscriberGroup {
261    fn ready_string_with(&mut self, source: &Source, tes_range: &mut RangeBuilder) {
262        tes_range.push(self.tes_range_open.clone());
263        self.content.ready_string_with(source, tes_range);
264        tes_range.push(self.tes_range_close.clone());
265    }
266
267    fn apply_tokens_to(&self, m: &RawMatch, b: &mut MatchTokensBuilder) {
268        let mut stream = TokenStream::new();
269        self.content.apply_tokens_to(
270            m,
271            &mut MatchTokensBuilder {
272                tokens: &mut stream,
273                ..*b
274            },
275        );
276        let mut g = Group::new(self.delimiter, stream);
277        g.set_span(self.span);
278        b.tokens.append(g);
279    }
280
281    fn apply_string(&self, m: &RawMatch, b: &mut MatchStringBuilder) {
282        if !b.is_ready_string {
283            b.b.push_str(to_open_str(self.delimiter));
284        }
285        self.content.apply_string(m, b);
286        if !b.is_ready_string {
287            b.b.push_str(to_close_str(self.delimiter));
288        }
289    }
290}
291
292#[derive(Parse, ToTokens, Debug, Clone)]
293struct MacroTranscriberVar {
294    dollar_token: Token![$],
295    name: Ident,
296}
297
298#[derive(Debug, Clone)]
299struct TranscriberVar {
300    var: MacroTranscriberVar,
301    var_index: usize,
302}
303impl Parse for TranscriberVar {
304    fn parse(input: ParseStream) -> Result<Self> {
305        Ok(Self {
306            var: input.parse()?,
307            var_index: usize::MAX,
308        })
309    }
310}
311
312impl TranscriberVar {
313    fn attach(&mut self, p: &PatternItems) -> Result<()> {
314        let name = self.var.name.to_string();
315        let span = self.var.span();
316        if let Some(b) = p.vars.get(&name) {
317            if b.depth != 0 {
318                bail!(span, "variable '{name}' is still repeating at this depth",);
319            }
320            self.var_index = b.var_index_or_rep_index;
321            Ok(())
322        } else {
323            bail!(span, "attempted to repeat an expression containing no syntax variables matched as repeating at this depth")
324        }
325    }
326    fn apply_tokens_to(&self, m: &RawMatch, b: &mut MatchTokensBuilder) {
327        m.vars[self.var_index].apply_tokens_to(b)
328    }
329
330    fn apply_string(&self, m: &RawMatch, b: &mut MatchStringBuilder) {
331        m.vars[self.var_index].apply_string(b)
332    }
333}
334
335#[derive(Debug, Clone)]
336struct TranscriberRep {
337    content: TranscriberItems,
338    sep: MacroRepSep,
339    op: MacroRepOp,
340    span: Span,
341    var: MacroTranscriberVar,
342    rep_index: usize,
343}
344impl TranscriberRep {
345    fn parse(input: &mut ParseStreamEx) -> Result<Self> {
346        let _dollar_token: Token![$] = input.parse()?;
347        input.expect(token::Paren)?;
348        let content = input.parse_group(|_g, input| TranscriberItems::parse(input))?;
349        let sep = input.parse()?;
350        let op: MacroRepOp = input.parse()?;
351        let span = _dollar_token.span();
352        let span = span.join(op.span()).unwrap_or(span);
353        let Some(var) = content.get_var() else {
354            bail!(span, "attempted to repeat an expression containing no syntax variables");
355        };
356        Ok(Self {
357            content,
358            sep,
359            op,
360            span,
361            var,
362            rep_index: usize::MAX,
363        })
364    }
365    fn attach(&mut self, p: &PatternItems) -> Result<()> {
366        let var_name = self.var.name.to_string();
367        if let Some(b) = p.vars.get(&var_name) {
368            if b.depth > 0 {
369                self.rep_index = b.var_index_or_rep_index;
370                if let Some(r) = p.find_rep(&var_name) {
371                    if self.op != r.op {
372                        bail!(
373                            self.span,
374                            "mismatch repeat operator. expected {:?}, found {:?}",
375                            r.op,
376                            self.op
377                        );
378                    }
379                    return self.content.attach(&r.content);
380                }
381            }
382        }
383        bail!(self.var.span(), "attempted to repeat an expression containing no syntax variables matched as repeating at this depth")
384    }
385
386    fn apply_tokens_to(&self, m: &RawMatch, b: &mut MatchTokensBuilder) {
387        let mut is_next = false;
388        for m in &m.reps[self.rep_index].0 {
389            if is_next {
390                if let Some(sep) = &self.sep.0 {
391                    sep.to_tokens(b.tokens);
392                }
393            }
394            is_next = true;
395            self.content.apply_tokens_to(m, b)
396        }
397    }
398
399    fn apply_string(&self, m: &RawMatch, b: &mut MatchStringBuilder) {
400        let mut is_next = false;
401        for m in &m.reps[self.rep_index].0 {
402            if is_next {
403                if let Some(sep) = &self.sep.0 {
404                    b.b.push_tokens(&sep.to_token_stream());
405                }
406            }
407            is_next = true;
408            self.content.apply_string(m, b)
409        }
410    }
411}
macro_rules_rt/transcriber.rs

macro_rules_rt/
transcriber.rs
