regex_tokenizer_impl/
lib.rs

1extern crate proc_macro;
2
3mod compilation_error;
4mod parse_declaration;
5
6use proc_macro::TokenStream;
7use quote::quote;
8
9/// Parse a tokenizer DSL, where three kinds of statements are admitted:
10/// - `IDENTIFIER`: a valid Rust identified used to give a name to the Tokenizer and related internal types
11/// - `"regex" => Type`: "regex" is a valid regular expression, Type is a valid Rust identifier. Tokens that match the regex will be considered of type `Type`
12/// - `"regex" => _`: "regex" is a valid regular expression and tokens that match it are considered separators and ignored.
13///
14/// # Examples
15/// ```
16/// tokenizer! {
17///     Test
18///
19///     r"[a-zA-Z]\w*" => Identifier
20///     r"\d+" => Number
21///     r"\s+" => _
22/// }
23/// ```
24#[proc_macro]
25pub fn tokenizer(_item: TokenStream) -> TokenStream {
26    let data = match parse_declaration::parse(_item.into()) {
27        Ok(data) => data,
28        Err(error) => return error.into(),
29    };
30
31    let parser = data.get_parser();
32
33    quote! {
34        #parser
35    }
36    .into()
37}
regex_tokenizer_impl/lib.rs

regex_tokenizer_impl/
lib.rs
