logos_codegen/parser/
mod.rs

1use proc_macro2::{Span, TokenStream, TokenTree};
2use quote::quote;
3use std::borrow::Cow;
4use syn::spanned::Spanned;
5use syn::{Attribute, GenericParam, Ident, Lit, LitBool, Meta, Type};
6
7use crate::error::Errors;
8use crate::leaf::{Callback, InlineCallback};
9use crate::util::{expect_punct, MaybeVoid};
10use crate::LOGOS_ATTR;
11
12mod definition;
13mod error_type;
14mod ignore_flags;
15mod nested;
16mod subpattern;
17mod type_params;
18
19pub use self::definition::{Definition, Literal};
20pub use self::error_type::ErrorType;
21pub use self::ignore_flags::IgnoreFlags;
22use self::nested::{AttributeParser, Nested, NestedValue};
23pub use self::subpattern::Subpatterns;
24use self::type_params::{replace_lifetime, traverse_type, TypeParams};
25
26#[derive(Default)]
27pub struct Parser {
28    pub errors: Errors,
29    pub utf8_mode: Option<LitBool>,
30    pub skips: Vec<Definition>,
31    pub extras: MaybeVoid,
32    pub subpatterns: Vec<(Ident, Literal)>,
33    pub error_type: Option<ErrorType>,
34    pub logos_path: Option<TokenStream>,
35    pub export_path: Option<String>,
36    types: TypeParams,
37}
38
39impl Parser {
40    pub fn parse_generic(&mut self, param: GenericParam) {
41        match param {
42            GenericParam::Lifetime(lt) => {
43                self.types.explicit_lifetime(lt, &mut self.errors);
44            }
45            GenericParam::Type(ty) => {
46                self.types.add(ty.ident);
47            }
48            GenericParam::Const(c) => {
49                self.err("Logos doesn't support const generics.", c.span());
50            }
51        }
52    }
53
54    pub fn generics(&mut self) -> Option<TokenStream> {
55        self.types.generics(&mut self.errors)
56    }
57
58    fn parse_attr(&mut self, attr: &mut Attribute) -> Option<AttributeParser> {
59        match &mut attr.meta {
60            Meta::List(list) => {
61                let tokens = std::mem::replace(&mut list.tokens, TokenStream::new());
62
63                Some(AttributeParser::new(tokens))
64            }
65            _ => None,
66        }
67    }
68
69    /// Try to parse the main `#[logos(...)]`, does nothing if
70    /// the attribute's name isn't `logos`.
71    pub fn try_parse_logos(&mut self, attr: &mut Attribute) {
72        if !attr.path().is_ident(LOGOS_ATTR) {
73            return;
74        }
75
76        let nested = match self.parse_attr(attr) {
77            Some(tokens) => tokens,
78            None => {
79                self.err("Expected #[logos(...)]", attr.span());
80                return;
81            }
82        };
83
84        for nested in nested {
85            let (name, value) = match nested {
86                Nested::Named(name, value) => (name, value),
87                Nested::Unexpected(tokens) | Nested::Unnamed(tokens) => {
88                    self.err("Invalid nested attribute", tokens.span());
89                    continue;
90                }
91            };
92
93            let span = name.span();
94
95            match name.to_string().as_str() {
96                "crate" => match value {
97                    NestedValue::Assign(logos_path) => self.logos_path = Some(logos_path),
98                    _ => {
99                        self.err("Expected: #[logos(crate = path::to::logos)]", span);
100                    }
101                },
102                "error" => match value {
103                    NestedValue::Assign(value) => {
104                        let span = value.span();
105
106                        let error_ty = ErrorType::new(value);
107
108                        if let Some(previous) = self.error_type.replace(error_ty) {
109                            self.err("Error type can be defined only once", span)
110                                .err("Previous definition here", previous.span());
111                        }
112                    }
113                    NestedValue::Group(value) => {
114                        let span = value.span();
115                        let mut nested = AttributeParser::new(value);
116                        let ty = match nested.parsed::<Type>() {
117                            Some(Ok(ty)) => ty,
118                            Some(Err(e)) => {
119                                self.err(e.to_string(), e.span());
120                                return;
121                            }
122                            None => {
123                                self.err("Expected #[logos(error(SomeType))]", span);
124                                return;
125                            }
126                        };
127
128                        let mut error_type = {
129                            use quote::ToTokens;
130                            ErrorType::new(ty.into_token_stream())
131                        };
132
133                        for (position, next) in nested.enumerate() {
134                            match next {
135                                Nested::Unexpected(tokens) => {
136                                    self.err("Unexpected token in attribute", tokens.span());
137                                }
138                                Nested::Unnamed(tokens) => match position {
139                                    0 => error_type.callback = self.parse_callback(tokens),
140                                    _ => {
141                                        self.err(
142                                            "\
143                                            Expected a named argument at this position\n\
144                                            \n\
145                                            hint: If you are trying to define a callback here use: callback = ...\
146                                            ",
147                                            tokens.span(),
148                                        );
149                                    }
150                                },
151                                Nested::Named(name, value) => {
152                                    error_type.named_attr(name, value, self);
153                                }
154                            }
155                        }
156
157                        if let Some(previous) = self.error_type.replace(error_type) {
158                            self.err("Error type can be defined only once", span)
159                                .err("Previous definition here", previous.span());
160                        }
161                    }
162                    _ => {
163                        self.err(
164                            concat!(
165                                "Expected: #[logos(error = SomeType)] or ",
166                                "#[logos(error(SomeType[, callback))]"
167                            ),
168                            span,
169                        );
170                    }
171                },
172                "export_dir" => match value {
173                    NestedValue::Assign(value) => {
174                        let span = value.span();
175
176                        match syn::parse2::<Literal>(value) {
177                            Ok(Literal::Utf8(str)) => {
178                                if let Some(previous) = self.export_path.replace(str.value()) {
179                                    self.err("Export path can be defined only once", span)
180                                        .err("Previous definition here", previous.span());
181                                }
182                            }
183                            Ok(_) => {
184                                self.err("Expected a &str", span);
185                            }
186                            Err(e) => {
187                                self.err(e.to_string(), span);
188                            }
189                        }
190                    }
191                    _ => {
192                        self.err(
193                            "Expected #[logos(export_dir = \"path/to/export/dir\")]",
194                            span,
195                        );
196                    }
197                },
198                "extras" => match value {
199                    NestedValue::Assign(value) => {
200                        let span = value.span();
201
202                        if let MaybeVoid::Some(previous) = self.extras.replace(value) {
203                            self.err("Extras can be defined only once", span)
204                                .err("Previous definition here", previous.span());
205                        }
206                    }
207                    _ => {
208                        self.err("Expected: #[logos(extras = SomeType)]", span);
209                    }
210                },
211                "skip" => match value {
212                    NestedValue::Literal(lit) => {
213                        if let Some(literal) = self.parse_literal(Lit::new(lit)) {
214                            self.skips.push(Definition::new(literal));
215                        }
216                    }
217                    NestedValue::Group(tokens) => {
218                        let token_span = tokens.span();
219                        if let Some(skip) = self.parse_definition(AttributeParser::new(tokens)) {
220                            self.skips.push(skip);
221                        } else {
222                            self.err(
223                                "Expected #[logos(skip(\"regex literal\"[, [callback = ] callback, priority = priority]))]",
224                                token_span,
225                            );
226                        }
227                    }
228                    _ => {
229                        self.err(
230                            "Expected: #[logos(skip \"regex literal\")] or #[logos(skip(...))]",
231                            span,
232                        );
233                    }
234                },
235                "source" => {
236                    self.err(
237                        "The `source` attribute is deprecated. Use the `utf8` attribute instead",
238                        span,
239                    );
240                }
241                "subpattern" => match value {
242                    NestedValue::KeywordAssign(name, value) => {
243                        match syn::parse2::<Literal>(value) {
244                            Ok(lit) => {
245                                self.subpatterns.push((name, lit));
246                            }
247                            Err(e) => {
248                                self.errors.err(e.to_string(), e.span());
249                            }
250                        };
251                    }
252                    _ => {
253                        self.err(r#"Expected: #[logos(subpattern name = r"regex")]"#, span);
254                    }
255                },
256                "type" => match value {
257                    NestedValue::KeywordAssign(generic, ty) => {
258                        self.types.set(generic, ty, &mut self.errors);
259                    }
260                    _ => {
261                        self.err("Expected: #[logos(type T = SomeType)]", span);
262                    }
263                },
264                "utf8" => match value {
265                    NestedValue::Assign(value) => {
266                        let span = value.span();
267
268                        match syn::parse2::<LitBool>(value) {
269                            Ok(lit) => {
270                                if let Some(previous) = self.utf8_mode.replace(lit) {
271                                    self.err("Utf8 mode can be defined only once", span)
272                                        .err("Previous definition here", previous.span());
273                                }
274                            }
275                            Err(e) => {
276                                self.err(format!("Expected a boolean literal: {e}"), span);
277                            }
278                        }
279                    }
280                    _ => {
281                        self.err("Expected: #[logos(utf8 = true)]", span);
282                    }
283                },
284                name => {
285                    self.err(format!("Unknown nested attribute #[logos({name})]",), span);
286                }
287            }
288        }
289    }
290
291    pub fn parse_literal(&mut self, lit: Lit) -> Option<Literal> {
292        match lit {
293            Lit::Str(string) => Some(Literal::Utf8(string)),
294            Lit::ByteStr(bytes) => Some(Literal::Bytes(bytes)),
295            _ => {
296                self.err("Expected a &str or &[u8] slice", lit.span());
297
298                None
299            }
300        }
301    }
302
303    /// Parse attribute definition of a token:
304    ///
305    /// + `#[token(literal[, callback])]`
306    /// + `#[regex(literal[, callback])]`
307    pub fn parse_definition_attr(&mut self, attr: &mut Attribute) -> Option<Definition> {
308        let nested = self.parse_attr(attr)?;
309
310        self.parse_definition(nested)
311    }
312
313    fn parse_definition(&mut self, mut nested: AttributeParser) -> Option<Definition> {
314        let literal = match nested.parsed::<Lit>()? {
315            Ok(lit) => self.parse_literal(lit)?,
316            Err(err) => {
317                self.err(err.to_string(), err.span());
318
319                return None;
320            }
321        };
322
323        let mut def = Definition::new(literal);
324
325        for (position, next) in nested.enumerate() {
326            match next {
327                Nested::Unexpected(tokens) => {
328                    self.err("Unexpected token in attribute", tokens.span());
329                }
330                Nested::Unnamed(tokens) => match position {
331                    0 => def.callback = self.parse_callback(tokens),
332                    _ => {
333                        self.err(
334                            "\
335                            Expected a named argument at this position\n\
336                            \n\
337                            hint: If you are trying to define a callback here use: callback = ...\
338                            ",
339                            tokens.span(),
340                        );
341                    }
342                },
343                Nested::Named(name, value) => {
344                    def.named_attr(name, value, self);
345                }
346            }
347        }
348
349        Some(def)
350    }
351
352    fn parse_callback(&mut self, tokens: TokenStream) -> Option<Callback> {
353        let span = tokens.span();
354        let mut tokens = tokens.into_iter();
355
356        if let Some(tt) = expect_punct(tokens.next(), '|') {
357            let mut label = TokenStream::from(tt);
358
359            label.extend(tokens);
360
361            return Some(Callback::Label(label));
362        }
363
364        let first = tokens.next();
365        let error = expect_punct(tokens.next(), '|');
366
367        let arg = match (error, first) {
368            (None, Some(TokenTree::Ident(arg))) => arg,
369            _ => {
370                self.err(
371                    "Inline callbacks must use closure syntax with exactly one parameter",
372                    span,
373                );
374                return None;
375            }
376        };
377
378        let body = match tokens.next() {
379            Some(TokenTree::Group(group)) => group.stream(),
380            Some(first) => {
381                let mut body = TokenStream::from(first);
382
383                body.extend(tokens);
384                body
385            }
386            None => {
387                self.err("Callback missing a body", span);
388                return None;
389            }
390        };
391
392        let inline = InlineCallback { arg, body, span };
393
394        Some(inline.into())
395    }
396
397    /// Checks if `ty` is a declared generic param, if so replaces it
398    /// with a concrete type defined using #[logos(type T = Type)]
399    ///
400    /// If no matching generic param is found, all lifetimes are fixed
401    /// to the source lifetime
402    pub fn get_type(&self, ty: &mut Type) -> TokenStream {
403        traverse_type(ty, &mut |ty| {
404            if let Type::Path(tp) = ty {
405                // Skip types that begin with `self::`
406                if tp.qself.is_none() {
407                    // If `ty` is a generic type parameter, try to find
408                    // its concrete type defined with #[logos(type T = Type)]
409                    if let Some(substitute) = self.types.find(&tp.path) {
410                        *ty = substitute;
411                    }
412                }
413            }
414            // If `ty` is a concrete type, fix its lifetimes to 'source
415            replace_lifetime(ty);
416        });
417
418        quote!(#ty)
419    }
420
421    pub fn err<M>(&mut self, message: M, span: Span) -> &mut Errors
422    where
423        M: Into<Cow<'static, str>>,
424    {
425        self.errors.err(message, span)
426    }
427}
logos_codegen/parser/mod.rs

logos_codegen/parser/
mod.rs
