lexer

plugin based lexical reader

extern crate lexer;

use lexer::{Input, Reader, ReaderResult, ReadersBuilder, State, Token, TokenError, TokenMeta};

#[derive(Debug)]
pub enum TokenValue {
    Whitespace(String),
    Identifier(String),
}

pub type MyToken = Token<TokenValue>;
pub type MyError = TokenError<&'static str>;

pub struct WhitespaceReader;

impl Reader<MyToken, MyError> for WhitespaceReader {
    #[inline(always)]
    fn priority(&self) -> usize {
        0usize
    }

    fn read(
        &self,
        input: &mut Input,
        current: &State,
        next: &mut State,
    ) -> ReaderResult<MyToken, MyError> {
        match input.read(next) {
            Some(ch) => if ch.is_whitespace() {
                let mut string = String::new();

                string.push(ch);

                while let Some(ch) = input.peek(next, 0) {
                    if ch.is_whitespace() {
                        input.read(next);
                        string.push(ch);
                    } else {
                        break;
                    }
                }

                ReaderResult::Some(Token::new(
                    TokenMeta::new_state_meta(current, next),
                    TokenValue::Whitespace(string),
                ))
            } else {
                ReaderResult::None
            },
            None => ReaderResult::None,
        }
    }
}

pub struct IdentifierReader;

impl Reader<MyToken, MyError> for IdentifierReader {
    #[inline(always)]
    fn priority(&self) -> usize {
        1usize
    }

    fn read(
        &self,
        input: &mut Input,
        current: &State,
        next: &mut State,
    ) -> ReaderResult<MyToken, MyError> {
        match input.read(next) {
            Some(ch) => if ch.is_alphabetic() {
                let mut string = String::new();

                string.push(ch);

                while let Some(ch) = input.peek(next, 0) {
                    if ch.is_alphanumeric() {
                        input.read(next);
                        string.push(ch);
                    } else {
                        break;
                    }
                }

                ReaderResult::Some(Token::new(
                    TokenMeta::new_state_meta(current, next),
                    TokenValue::Identifier(string),
                ))
            } else {
                ReaderResult::None
            },
            None => ReaderResult::None,
        }
    }
}

fn main() {
    let readers = ReadersBuilder::new()
        .add(WhitespaceReader)
        .add(IdentifierReader)
        .build();

    let lexer = readers.lexer("Hello world\n".chars());
    let tokens: Vec<MyToken> = lexer.map(|t| t.unwrap()).collect();

    println!("{:#?}", tokens);
}