lr_rust_parsing_macro 0.1.2

parsing_macro sub library for lr_rust.
Documentation
mod lexer;

use crate::lexer::{process_productions, Production};
use proc_macro2::{Ident, Punct, Spacing, Span, TokenStream};
use quote::{quote, quote_spanned, ToTokens, TokenStreamExt};
use lr_rust_shared_structs::{ParseTable, Trie, RegexDFA};
use std::{collections::BTreeSet, process::exit};
use syn::{
    parse::{discouraged::Speculative, Parse},
    spanned::Spanned,
    Error, ExprClosure, Token, Type, Visibility,
};

const ERR_STATE_NOT_SPECIFIED: &'static str =
    "ERROR: You must specify the lexer state with State(...)";
const ERR_KIND_NOT_SPECIFIED: &'static str =
    "ERROR: You must specify the lexer kind with Kind(...)";
const ERR_MISSING_STATE_TYPE: &'static str =
    "ERROR: Expected parenthesized lexer state type after State element.";
const ERR_MISSING_OUT_TYPE: &'static str =
    "ERROR: Expected parenthesized output type after Output element.";
const ERR_NO_OUT_TYPE: &'static str = "ERROR: You must specify the output type with Output(...)";
const ERR_GEN_FN_NOT_SPECIFIED: &'static str =
    "ERROR: You must specify the generated function with GeneretedFn(...)";

extern crate proc_macro;

/// The parser macro takes comma separated arguments of three types:
/// - State: This argument must only be passed once. It is of the form State(<StateTypeName>),
///   and specifies the state which will be maintained during the lexing stage of your parser
/// - Output: Specifies the output enum produced by parser. Every single Rule must have a name
///   which corresponds to a member of this enum.
/// - GeneratedFn the name of the function generated by the parser. visibility modifiers are allowd
/// - Kind the name of the generated enum representing node kinds. visibility modifiers are allowd
/// - Elements: This argument must appear at least once. It specifies the various elements of
///   your language, and takes three sub-forms:
///   - Regex: Specifies a regex defined lexeme within your language. It must be of the form:
///     <LexemeName> => Regex(<ProducedNodeType>, "my-regex.*") <callback>
///     where the callback is a closure of form:
///       |state: &mut State, matched_text: &str| -> Option<Node>
///     returning None allows creation of update patterns which serve to eat input and update state
///   - Literal: Specifies a literal defined lexeme within your language. It is of the same form as
///     a Regex, except it is specified as:
///       <LexemeName> => Literal(... <callback>)
///   - Rule: Specifies a non-leaf node in your AST. It must be of the form:
///     <RuleName> => Rule(<rule_1>, <rule_2>, ... )
///     where each rule is of the form: elem1 elem2 elem3 ... <optional-callback>
///     each elem is the name of a (not necessarily previously) defined token, and
///     the optional callback is a closure of the form:
///       |state: &mut State, elem1: Node, elem2: Node, ...| -> Node {...}
#[proc_macro]
pub fn parser(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
    let input = syn::parse(input).expect("Proc Macro errored parsing input.");
    match parser2(input) {
        Result::Ok(output) => output.into(),
        Result::Err(err) => {
            eprintln!("{}", err);
            exit(69);
        }
    }
}

trait Context {
    fn context(self, cx: &str) -> Self;
}

impl<T> Context for syn::Result<T> {
    fn context(self, cx: &str) -> Self {
        self.map_err(|e| Error::new(e.span(), format!("{cx}: {e}")))
    }
}

struct MacroBody {
    out_type: TokenStream,
    state_type: TokenStream,
    generated_fn: TokenStream,
    kind_def: TokenStream,
    kind_type: Type,
    regex: RegexDFA,
    trie: Trie,
    productions: Vec<Production>,
    parser: ParseTable,
}

impl Parse for MacroBody {
    fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
        let mut productions: Vec<Production> = vec![];
        let mut out_type = None;
        let mut state_type = None;
        let mut kind_type = None;
        let mut kind_vis = None;
        let mut generated_fn_vis = None;
        let mut generated_fn_name = None;
        while !input.is_empty() {
            let fork = input.fork();
            if let Result::Ok(ident) = fork.parse::<Ident>()
                && ["GeneratedFn", "Kind", "State", "Output"].contains(&ident.to_string().as_str())
            {
                input.advance_to(&fork);
                let ident_str = ident.to_string();
                if ident_str == "State" {
                    let content;
                    syn::parenthesized!(content in input);
                    state_type = Some(content.parse::<Type>().context(ERR_MISSING_STATE_TYPE)?);
                } else if ident_str == "Output" {
                    let content;
                    syn::parenthesized!(content in input);
                    out_type = Some(content.parse::<Type>().context(ERR_MISSING_OUT_TYPE)?);
                } else if ident_str == "Kind" {
                    let content;
                    syn::parenthesized!(content in input);
                    kind_vis = content.parse::<Visibility>().ok();
                    kind_type = Some(content.parse::<Type>().context(ERR_MISSING_OUT_TYPE)?);
                } else if ident_str == "GeneratedFn" {
                    let content;
                    syn::parenthesized!(content in input);
                    generated_fn_vis = content.parse::<Visibility>().ok();
                    generated_fn_name =
                        Some(content.parse::<Ident>().context(ERR_MISSING_OUT_TYPE)?);
                } else {
                    panic!();
                }
            } else {
                productions.push(input.parse()?);
            }
            input.parse::<Token![,]>().unwrap();
        }

        let (regex, trie, parser) = process_productions(&productions);
        let tot_span = input.span();
        let out_type = out_type.ok_or(Error::new(tot_span, ERR_NO_OUT_TYPE))?;
        let state_type = state_type.ok_or(Error::new(tot_span, ERR_STATE_NOT_SPECIFIED))?;
        let kind_type = kind_type.ok_or(Error::new(tot_span, ERR_KIND_NOT_SPECIFIED))?;
        let generated_fn_name =
            generated_fn_name.ok_or(Error::new(tot_span, ERR_GEN_FN_NOT_SPECIFIED))?;

        let result = Ok(Self {
            out_type: quote! { #out_type },
            state_type: quote! { #state_type },
            kind_def: quote! { #kind_vis enum #kind_type },
            kind_type: kind_type,
            generated_fn: quote! {#generated_fn_vis fn #generated_fn_name() },
            regex,
            productions,
            trie,
            parser,
        });
        result
    }
}

fn parser2(input: TokenStream) -> Result<TokenStream, Error> {
    let MacroBody {
        state_type,
        out_type,
        kind_type,
        kind_def,
        generated_fn,
        regex,
        trie,
        productions,
        parser,
    } = syn::parse2(input)?;

    let make_rule_callback = |maybe_user_callback: Option<&ExprClosure>,
                              num_generated: usize,
                              num_args: usize|
     -> (TokenStream, Ident) {
        let callback_name = Ident::new(&format!("__gen_{}", num_generated), Span::call_site());
        let user_callback = maybe_user_callback
            .map(|c| c.to_token_stream())
            .unwrap_or_else(|| {
                let mut closure_args = quote! { node_1 };
                for _ in 0..(num_args - 1) {
                    closure_args.append_all(quote! {, _});
                }
                quote! { |_, #closure_args| node_1 }
            });

        let callback_args_rev = (0..num_args)
            .map(|i| Ident::new(&format!("node_{}", num_args - i), user_callback.span()));
        let stack_pops_iter = callback_args_rev
            .clone()
            .map(|s| quote! { let #s = node_stack.pop().unwrap(); });
        let stack_pops = quote! { #(#stack_pops_iter)* };
        let callback_args_iter = callback_args_rev.rev();
        let callback_args = quote! { #(#callback_args_iter),* };
        let callback = quote_spanned! { user_callback.span() =>
            fn #callback_name(state: &mut #state_type, node_stack: &mut Vec<#out_type>) -> #out_type {
                #stack_pops
                let user_callback = #user_callback;
                user_callback(state, #callback_args)
            }
        };
        (callback, callback_name)
    };

    let mut lexeme_callback_defs = TokenStream::new();
    let mut lexeme_callback_names = vec![];
    let mut error_callback_defs = TokenStream::new();
    let mut error_callback_names = vec![];
    let mut rule_callback_defs = TokenStream::new();
    let mut rule_callback_names = vec![];
    let mut num_generated = 0;
    let mut cur_rule = 0;

    for production in productions.iter() {
        match &production {
            Production::Lexeme { callback, .. } | Production::Update { callback, .. } => {
                let callback_name =
                    Ident::new(&format!("__gen_{}", num_generated), Span::call_site());
                let callback = quote_spanned! { callback.span() =>
                    fn #callback_name(state: &mut #state_type, s: &str) -> Option<(#out_type, usize)> {
                        let user_callback = #callback;
                        user_callback(state, s)
                    }
                };
                num_generated += 1;
                lexeme_callback_defs.append_all(callback);
                lexeme_callback_names.push(callback_name);
            }
            Production::None { .. } => {}
            Production::Rule { rules, error, .. } => {
                for (_, callback) in rules {
                    let (callback, callback_name) = make_rule_callback(
                        callback.as_ref(),
                        num_generated,
                        parser.rule_lens[cur_rule].0,
                    );
                    num_generated += 1;
                    cur_rule += 1;
                    rule_callback_defs.append_all(callback);
                    rule_callback_names.push(callback_name);
                }
                if let Some(user_callback) = error {
                    let callback_name =
                        Ident::new(&format!("__gen_{}", num_generated), Span::call_site());
                    let callback = quote_spanned! { user_callback.span() =>
                        fn #callback_name(state: &mut #state_type, vec: &Vec<#out_type>) -> #out_type {
                            let user_callback = #user_callback;
                            user_callback(state, s)
                        }
                    };
                    error_callback_defs.append_all(callback);
                    error_callback_names.push(callback_name);
                    num_generated += 1;
                }
            }
        }
    }

    let mut lexeme_callbacks = TokenStream::new();
    lexeme_callbacks.append_separated(
        lexeme_callback_names.into_iter().map(|name| {
            quote! {
                #name as fn(&mut #state_type, &str) -> Option<(#out_type, usize)>
            }
        }),
        Punct::new(',', Spacing::Alone),
    );

    let mut error_callbacks = TokenStream::new();
    error_callbacks.append_separated(
        error_callback_names.into_iter().map(|name| {
            quote! {
                #name as fn(&mut #state_type, &mut Vec<#out_type>) -> #out_type
            }
        }),
        Punct::new(',', Spacing::Alone),
    );

    let mut rule_callbacks = TokenStream::new();
    rule_callbacks.append_separated(
        rule_callback_names.into_iter().map(|name| {
            quote! {
                #name as fn(&mut #state_type, &mut Vec<#out_type>) -> #out_type
            }
        }),
        Punct::new(',', Spacing::Alone),
    );

    let kinds = productions
        .iter()
        .scan(BTreeSet::new(), |seen, prod| {
            if let Some((name, name_raw)) = prod.get_name()
                && !seen.contains(name_raw)
            {
                seen.insert(name_raw);
                return Some(Some(name));
            }
            return Some(None);
        })
        .filter_map(|k| k)
        .collect::<Vec<_>>();
    let kind_def = quote! {
        #[derive(Clone, Copy, Debug)]
        #[repr(u16)]
        #kind_def { #(#kinds),* }

        impl Into<usize> for #kind_type {
            fn into(self) -> usize {
                self as usize
            }
        }
    };

    Ok(quote! {
        #kind_def

        #generated_fn ->
            Result<lr_rust::Engine<
                #out_type,
                #state_type,
                >,
                &'static str
            >
        {
            #lexeme_callback_defs
            #error_callback_defs
            #rule_callback_defs
            lr_rust::Engine::from_raw(
                #parser,
                #regex,
                #trie,
                vec![#lexeme_callbacks],
                vec![#error_callbacks],
                vec![#rule_callbacks],
            )
        }
    })
}