cpp_rs 0.1.0

use super::build_string_from_tokens;
use crate::lexer::{OwningTokenizer, StringSource, Token, TokenType, Tokenizer};
use std::borrow::Cow;

macro_rules! gen_test {
    ($name:ident => $($body:tt)*) => {
        #[test]
        fn $name() {
            inner_gen_test!([]; $($body)*);
        }
    }
}

macro_rules! inner_gen_test {
    ([$($idents:ident),*]; $name:ident:$iter:ident, $($rest:tt)*) => {
        for $name in $iter {
            inner_gen_test!([$($idents,)* $name]; $($rest)*);
        }
    };

    ([$($idents:ident),*];) => {
        let mut toks = vec![$($idents.clone()),*, EOF.clone()];
        let input = build_string_from_tokens(&mut toks);
        eprintln!("Input: {:?}", input);

        let tokenizer : OwningTokenizer = Tokenizer::new(input);

        itertools::assert_equal(tokenizer, toks.into_iter().map(|x| Ok(x)));
    };

}

static IDENTS: &[Token<'static>] = &[
    Token {
        token_type: TokenType::Identifier,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("a"),
    },
    Token {
        token_type: TokenType::Identifier,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed("ae"),
    },
    Token {
        token_type: TokenType::Identifier,
        start_byte: 0,
        end_byte: 7,
        text: Cow::Borrowed("u83bb_e4"),
    },
    Token {
        token_type: TokenType::Identifier,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("_"),
    },
];

static OPERATORS: &[Token<'static>] = &[
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("&"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("/"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("*"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("+"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("-"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("="),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("<"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed(">"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed("<<"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed(">>"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("|"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed("&&"),
    },
    Token {
        token_type: TokenType::Punctuator,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed("||"),
    },
];

static NUMBERS: &[Token<'static>] = &[
    Token {
        token_type: TokenType::PPNumber,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("0"),
    },
    Token {
        token_type: TokenType::PPNumber,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed(".3"),
    },
    Token {
        token_type: TokenType::PPNumber,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed(".3"),
    },
    Token {
        token_type: TokenType::PPNumber,
        start_byte: 0,
        end_byte: 3,
        text: Cow::Borrowed(".3e+"),
    },
    Token {
        token_type: TokenType::PPNumber,
        start_byte: 0,
        end_byte: 4,
        text: Cow::Borrowed(".3e+5"),
    },
];

static WHITESPACE: &[Token<'static>] = &[
    Token {
        token_type: TokenType::Whitespace,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed(" "),
    },
    Token {
        token_type: TokenType::Whitespace,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("\t"),
    },
    Token {
        token_type: TokenType::Whitespace,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed("\t "),
    },
    Token {
        token_type: TokenType::Whitespace,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed(" \t"),
    },
    Token {
        token_type: TokenType::EOL,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("\n"),
    },
    Token {
        token_type: TokenType::EOL,
        start_byte: 0,
        end_byte: 0,
        text: Cow::Borrowed("\r"),
    },
    Token {
        token_type: TokenType::EOL,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed("\n\r"),
    },
    Token {
        token_type: TokenType::EOL,
        start_byte: 0,
        end_byte: 1,
        text: Cow::Borrowed("\r\n"),
    },
];

static EOF: Token<'static> = Token {
    token_type: TokenType::EOF,
    start_byte: 0,
    end_byte: 0,
    text: Cow::Borrowed(""),
};

gen_test!(ident_op_ident => i:IDENTS, o:OPERATORS, i2:IDENTS,);
gen_test!(ident_op_ppnum => i:IDENTS, o:OPERATORS, n:NUMBERS,);
gen_test!(ppnum_op_ident => n:NUMBERS, o:OPERATORS, i:IDENTS,);
gen_test!(ppnum_op_ppnum => n:NUMBERS, o:OPERATORS, n2:NUMBERS,);
gen_test!(ident_ws_ident => i:IDENTS, ws:WHITESPACE, i2:IDENTS,);
gen_test!(ident_ws_ppnum => i:IDENTS, ws:WHITESPACE, n:NUMBERS,);
gen_test!(ppnum_ws_ident => n:NUMBERS, ws:WHITESPACE, i:IDENTS,);
gen_test!(ppnum_ws_ppnum => n:NUMBERS, ws:WHITESPACE, n2:NUMBERS,);
gen_test!(ident_ws_op_ws_ppnum => i:IDENTS, ws:WHITESPACE, o:OPERATORS, ws2: WHITESPACE, n:NUMBERS,);