pm-lexer 0.1.4

This is a sample lexer using the proc-macro crate, to be used in the pomelo testings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

use proc_macro2::{TokenTree, TokenStream, Punct, Spacing, Delimiter};

pub fn parse<E, F>(toks: TokenStream, mut f: F) -> Result<(), E>
where F: FnMut(&TokenTree) -> Result<(), E>
{
    parse2(toks, &mut f)
}

fn parse2<E>(toks: TokenStream, f: &mut impl FnMut(&TokenTree) -> Result<(), E>)  -> Result<(), E> {
    for tk in toks {
        match tk {
            TokenTree::Group(g) => {
                let (l, r) = match g.delimiter() {
                    Delimiter::Parenthesis => ('(', ')'),
                    Delimiter::Brace => ('{', '}'),
                    Delimiter::Bracket => ('[', ']'),
                    Delimiter::None => (' ', ' '),
                };
                if l != ' ' {
                    f(&TokenTree::Punct(Punct::new(l, Spacing::Alone)))?;
                }
                parse2(g.stream(), f)?;
                if r != ' ' {
                    f(&TokenTree::Punct(Punct::new(r, Spacing::Alone)))?;
                }
            }
            tt => { f(&tt)?; }
        }
    }
    Ok(())
}