cpp_rs 0.1.0

use crate::lexer::{StrTokenizer, Token, TokenType, Tokenizer};
use std::borrow::Cow;

#[test]
fn dot_ident_rather_than_ppnum() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new(".foo");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned(".".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Identifier,
            start_byte: 1,
            end_byte: 3,
            text: Cow::Owned("foo".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 4,
            end_byte: 4,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn multi_dot_ppnum() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("..0");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned(".".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 1,
            end_byte: 2,
            text: Cow::Owned(".0".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn greedy_dots() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new(".....");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 0,
            end_byte: 2,
            text: Cow::Owned("...".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned(".".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 4,
            end_byte: 4,
            text: Cow::Owned(".".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 5,
            end_byte: 5,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn greedy_dots2() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("...0");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 0,
            end_byte: 2,
            text: Cow::Owned("...".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("0".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 4,
            end_byte: 4,
            text: Cow::Owned("".into()),
        }
    )
}

// Make sure that when we find a non-ident digit we can transition properly
#[test]
fn ident_to_num_1() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("Foo.3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Identifier,
            start_byte: 0,
            end_byte: 2,
            text: Cow::Owned("Foo".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 3,
            end_byte: 4,
            text: Cow::Owned(".3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 5,
            end_byte: 5,
            text: Cow::Owned("".into()),
        }
    )
}

// In this test the '0' should be part of the identifier not the number
#[test]
fn ident_to_num_2() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("Fo0.3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Identifier,
            start_byte: 0,
            end_byte: 2,
            text: Cow::Owned("Fo0".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 3,
            end_byte: 4,
            text: Cow::Owned(".3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 5,
            end_byte: 5,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn num_to_punct_to_num() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("3[3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 1,
            end_byte: 1,
            text: Cow::Owned("[".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 2,
            end_byte: 2,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn num_to_punct_to_num2() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("3>=3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 1,
            end_byte: 2,
            text: Cow::Owned(">=".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 4,
            end_byte: 4,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn num_to_punct_to_num3() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("3>3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 1,
            end_byte: 1,
            text: Cow::Owned(">".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 2,
            end_byte: 2,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn num_to_punct_to_num4() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("3<3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 1,
            end_byte: 1,
            text: Cow::Owned("<".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 2,
            end_byte: 2,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn num_to_punct_to_num5() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("3<<3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 1,
            end_byte: 2,
            text: Cow::Owned("<<".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 4,
            end_byte: 4,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn num_to_punct_to_num6() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("3>>3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 1,
            end_byte: 2,
            text: Cow::Owned(">>".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 4,
            end_byte: 4,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn num_to_punct_to_num7() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("3<<=3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 1,
            end_byte: 3,
            text: Cow::Owned("<<=".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 4,
            end_byte: 4,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 5,
            end_byte: 5,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn num_to_punct_to_num8() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("3>>=3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 1,
            end_byte: 3,
            text: Cow::Owned(">>=".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 4,
            end_byte: 4,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 5,
            end_byte: 5,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn num_to_punct_to_num9() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("3%3");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 0,
            end_byte: 0,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 1,
            end_byte: 1,
            text: Cow::Owned("%".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 2,
            end_byte: 2,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn ternary_operator() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("Foo?3:11e4");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Identifier,
            start_byte: 0,
            end_byte: 2,
            text: Cow::Owned("Foo".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("?".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 4,
            end_byte: 4,
            text: Cow::Owned("3".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 5,
            end_byte: 5,
            text: Cow::Owned(":".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::PPNumber,
            start_byte: 6,
            end_byte: 9,
            text: Cow::Owned("11e4".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 10,
            end_byte: 10,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn test_alternate_pound() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("%:%");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 0,
            end_byte: 1,
            text: Cow::Owned("%:".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Punctuator,
            start_byte: 2,
            end_byte: 2,
            text: Cow::Owned("%".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 3,
            end_byte: 3,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn test_line_comment_after_ident() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("Foo//bar");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Identifier,
            start_byte: 0,
            end_byte: 2,
            text: Cow::Owned("Foo".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Comment,
            start_byte: 3,
            end_byte: 7,
            text: Cow::Owned("//bar".into()),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 8,
            end_byte: 8,
            text: Cow::Owned("".into()),
        }
    )
}

#[test]
fn test_line_comment_with_EOL() {
    let _ = env_logger::try_init();
    let mut tokenizer: StrTokenizer<'static> = Tokenizer::new("//bar\nFoo");

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Comment,
            start_byte: 0,
            end_byte: 4,
            text: Cow::Borrowed("//bar"),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOL,
            start_byte: 5,
            end_byte: 5,
            text: Cow::Borrowed("\n"),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::Identifier,
            start_byte: 6,
            end_byte: 8,
            text: Cow::Borrowed("Foo"),
        }
    );

    assert_eq!(
        tokenizer.get_next_token().unwrap(),
        Token {
            token_type: TokenType::EOF,
            start_byte: 9,
            end_byte: 9,
            text: Cow::Owned("".into()),
        }
    )
}