alpaca_lexer_derive/
lib.rs

#![recursion_limit="128"]

extern crate proc_macro;
extern crate syn;
#[macro_use]
extern crate quote;

use proc_macro::TokenStream;

#[proc_macro_derive(Token, attributes(expr, eof, skip))]
pub fn token(input: TokenStream) -> TokenStream {
    // Construct a string representation of the type definition
    let s = input.to_string();
    
    // Parse the string representation
    let ast = syn::parse_derive_input(&s).unwrap();

    // Build the impl
    let gen = impl_token(&ast);
    
    // Return the generated impl
    gen.parse().unwrap()
}

fn impl_token(ast: &syn::DeriveInput) -> quote::Tokens {
    let body = match ast.body {
        syn::Body::Enum(ref b) => {
            b
        },
        syn::Body::Struct(_) => {
            panic!("#[derive(Token)] only supports Enums");
        }
    };
    let name = &ast.ident;
    let mut eof_variant = None;
    let mut ids1 = Vec::new();
    let mut exprs1 = Vec::new();
    let mut types = Vec::new();
    let mut skips = vec![false; body.len()];
    for (k,v) in body.iter().enumerate() {
        for a in &v.attrs {
            if a.name() == "eof" {
                if let syn::MetaItem::Word(_) = a.value {
                    if eof_variant.is_some() {
                        panic!("#[derive(Token)] error: more than one variant has the `#[eof] attribute`");
                    }
                    eof_variant = Some(&v.ident);
                }
            }
            else if a.name() == "expr" {
                if let syn::MetaItem::NameValue(_, ref l) = a.value {
                    if let syn::Lit::Str(ref s, _) = *l {
                        let _type = match v.data {
                            syn::VariantData::Tuple(ref types) => {
                                if types.len()!=1 {
                                    panic!("#[derive(Token)] error: Only one tuple member allowed");
                                }
                                Some(&types.first().unwrap().ty)
                            },
                            syn::VariantData::Struct(_) => {
                                    panic!("#[derive(Token)] error: Struct variants not allowed");
                            },
                            _ => { None }
                        };
                        ids1.push(&v.ident);
                        exprs1.push(format!("^{}",s));
                        types.push(_type);
                    }
                }
            }
            else if a.name() == "skip" {
                skips[k] = true;
            }
        }
    }
    let re_ids1: Vec<_> = ids1.iter().map(|i| syn::Ident::new(format!("RE_{}",i.as_ref()))).collect();
    let re_ids2 = re_ids1.clone();
    let eof_variant = eof_variant.expect("#[derive(Token)] error: exactly one variant must have the `#[eof]` attribute");

    let make_tokens = ids1.iter().zip(types.iter()).zip(skips.iter()).zip(exprs1.iter()).map(|(((id,ty),skip),expr)| {
        if *skip {
            return quote! {
                continue 'main;
            };
        }
        match *ty {
            Some(ref t) => {
                let ty_tok = quote! {
                    #t
                };
                let ty_str = ty_tok.as_str();
                quote! {
                    let to_parse = t.as_str();
                    let val: #t = match to_parse.parse() {
                        Ok(v) => v,
                        Err(_) => {
                            return Some(Err(Error::InvalidToken{
                                parsed: to_parse.to_owned(),
                                regex: #expr.to_owned(),
                                ty: #ty_str.to_owned(),
                            }));
                        }
                    };
                    return Some(Ok(Token::#id(val)));
                }
            },
            None => {
                quote! {
                    return Some(Ok(Token::#id));
                }
            }
        }
    }).collect::<Vec<_>>();
    quote! {
        use ::std;
        use ::regex;
        pub struct TokenIterator<'input> {
            eof_reached: bool,
            input: &'input str,
        }
        impl<'input> Iterator for TokenIterator<'input> {
            type Item = Result< #name >;
            fn next(&mut self) -> Option<Self::Item> {
                lazy_static! {
                    #(
                        static ref #re_ids1: regex::Regex = {
                            regex::Regex::new(#exprs1).unwrap()
                        };
                    )*
                }
                'main: loop {
                    if self.eof_reached {
                        return None;
                    }
                    if self.input.is_empty() {
                        self.eof_reached = true;
                        return Some(Ok(Token::#eof_variant));
                    }
                    #(
                        if let Some(t) = #re_ids2.find(self.input) {
                            self.input = &self.input[t.end()..];
                            #make_tokens
                        }
                     )*
                    self.eof_reached = true;
                    return Some(Err(Error::InvalidInput{unparsed:self.input.to_owned()}))
                }
            }
        }
        pub fn scan(input: &str) -> TokenIterator {
            TokenIterator {
                input,
                eof_reached: false,
            }
        }
        #[derive(Debug, Fail)]
        pub enum Error {
            #[fail(display = "Failed to parse token of type {} with regex {}: {}",ty,regex,parsed )]
            InvalidToken {
                parsed: String,
                regex: String,
                ty: String,
            },
            #[fail(display = "No rule in the lexer matches: {}", unparsed)]
            InvalidInput {
                unparsed: String
            }
        }
        pub type Result<T> = std::result::Result<T, Error>;
    }
}
alpaca_lexer_derive/lib.rs

alpaca_lexer_derive/
lib.rs
