alpaca_lexer_derive/
lib.rs

1#![recursion_limit="128"]
2
3extern crate proc_macro;
4extern crate syn;
5#[macro_use]
6extern crate quote;
7
8use proc_macro::TokenStream;
9
10#[proc_macro_derive(Token, attributes(expr, eof, skip))]
11pub fn token(input: TokenStream) -> TokenStream {
12    // Construct a string representation of the type definition
13    let s = input.to_string();
14    
15    // Parse the string representation
16    let ast = syn::parse_derive_input(&s).unwrap();
17
18    // Build the impl
19    let gen = impl_token(&ast);
20    
21    // Return the generated impl
22    gen.parse().unwrap()
23}
24
25fn impl_token(ast: &syn::DeriveInput) -> quote::Tokens {
26    let body = match ast.body {
27        syn::Body::Enum(ref b) => {
28            b
29        },
30        syn::Body::Struct(_) => {
31            panic!("#[derive(Token)] only supports Enums");
32        }
33    };
34    let name = &ast.ident;
35    let mut eof_variant = None;
36    let mut ids1 = Vec::new();
37    let mut exprs1 = Vec::new();
38    let mut types = Vec::new();
39    let mut skips = vec![false; body.len()];
40    for (k,v) in body.iter().enumerate() {
41        for a in &v.attrs {
42            if a.name() == "eof" {
43                if let syn::MetaItem::Word(_) = a.value {
44                    if eof_variant.is_some() {
45                        panic!("#[derive(Token)] error: more than one variant has the `#[eof] attribute`");
46                    }
47                    eof_variant = Some(&v.ident);
48                }
49            }
50            else if a.name() == "expr" {
51                if let syn::MetaItem::NameValue(_, ref l) = a.value {
52                    if let syn::Lit::Str(ref s, _) = *l {
53                        let _type = match v.data {
54                            syn::VariantData::Tuple(ref types) => {
55                                if types.len()!=1 {
56                                    panic!("#[derive(Token)] error: Only one tuple member allowed");
57                                }
58                                Some(&types.first().unwrap().ty)
59                            },
60                            syn::VariantData::Struct(_) => {
61                                    panic!("#[derive(Token)] error: Struct variants not allowed");
62                            },
63                            _ => { None }
64                        };
65                        ids1.push(&v.ident);
66                        exprs1.push(format!("^{}",s));
67                        types.push(_type);
68                    }
69                }
70            }
71            else if a.name() == "skip" {
72                skips[k] = true;
73            }
74        }
75    }
76    let re_ids1: Vec<_> = ids1.iter().map(|i| syn::Ident::new(format!("RE_{}",i.as_ref()))).collect();
77    let re_ids2 = re_ids1.clone();
78    let eof_variant = eof_variant.expect("#[derive(Token)] error: exactly one variant must have the `#[eof]` attribute");
79
80    let make_tokens = ids1.iter().zip(types.iter()).zip(skips.iter()).zip(exprs1.iter()).map(|(((id,ty),skip),expr)| {
81        if *skip {
82            return quote! {
83                continue 'main;
84            };
85        }
86        match *ty {
87            Some(ref t) => {
88                let ty_tok = quote! {
89                    #t
90                };
91                let ty_str = ty_tok.as_str();
92                quote! {
93                    let to_parse = t.as_str();
94                    let val: #t = match to_parse.parse() {
95                        Ok(v) => v,
96                        Err(_) => {
97                            return Some(Err(Error::InvalidToken{
98                                parsed: to_parse.to_owned(),
99                                regex: #expr.to_owned(),
100                                ty: #ty_str.to_owned(),
101                            }));
102                        }
103                    };
104                    return Some(Ok(Token::#id(val)));
105                }
106            },
107            None => {
108                quote! {
109                    return Some(Ok(Token::#id));
110                }
111            }
112        }
113    }).collect::<Vec<_>>();
114    quote! {
115        use ::std;
116        use ::regex;
117        pub struct TokenIterator<'input> {
118            eof_reached: bool,
119            input: &'input str,
120        }
121        impl<'input> Iterator for TokenIterator<'input> {
122            type Item = Result< #name >;
123            fn next(&mut self) -> Option<Self::Item> {
124                lazy_static! {
125                    #(
126                        static ref #re_ids1: regex::Regex = {
127                            regex::Regex::new(#exprs1).unwrap()
128                        };
129                    )*
130                }
131                'main: loop {
132                    if self.eof_reached {
133                        return None;
134                    }
135                    if self.input.is_empty() {
136                        self.eof_reached = true;
137                        return Some(Ok(Token::#eof_variant));
138                    }
139                    #(
140                        if let Some(t) = #re_ids2.find(self.input) {
141                            self.input = &self.input[t.end()..];
142                            #make_tokens
143                        }
144                     )*
145                    self.eof_reached = true;
146                    return Some(Err(Error::InvalidInput{unparsed:self.input.to_owned()}))
147                }
148            }
149        }
150        pub fn scan(input: &str) -> TokenIterator {
151            TokenIterator {
152                input,
153                eof_reached: false,
154            }
155        }
156        #[derive(Debug, Fail)]
157        pub enum Error {
158            #[fail(display = "Failed to parse token of type {} with regex {}: {}",ty,regex,parsed )]
159            InvalidToken {
160                parsed: String,
161                regex: String,
162                ty: String,
163            },
164            #[fail(display = "No rule in the lexer matches: {}", unparsed)]
165            InvalidInput {
166                unparsed: String
167            }
168        }
169        pub type Result<T> = std::result::Result<T, Error>;
170    }
171}
alpaca_lexer_derive/lib.rs

alpaca_lexer_derive/
lib.rs
