unlab-gpu 0.1.0

//
// Copyright (c) 2025 Łukasz Szpakowski
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
//
use std::io::Cursor;
use super::*;

#[test]
fn test_lexer_next_returns_tokens()
{
    let s = "+";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::Plus, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 2), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_skips_comments()
{
    let s = "
# first text
+ % second text
# third text
";
    let s2 = &s[1..];
    let mut cursor = Cursor::new(s2.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 13), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Plus, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 2, 1), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 2, 16), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 3, 13), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_returns_newline_token_for_semicolon()
{
    let s = "
+; -
";
    let s2 = &s[1..];
    let mut cursor = Cursor::new(s2.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::Plus, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 2), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Minus, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 4), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 5), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_returns_interpuction_tokens()
{
    let s = "()[]{}.[.]?*/+-.*./.+.-<>=><== ==!='.: ::";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::LParen, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::RParen, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 2), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::LBracket, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 3), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::RBracket, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 4), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::LBrace, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 5), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::RBrace, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 6), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::DotLBracket, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 7), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::DotRBracket, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 9), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Ques, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 11), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Star, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 12), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Slash, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 13), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Plus, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 14), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Minus, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 15), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::DotStar, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 16), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::DotSlash, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 18), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::DotPlus, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 20), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::DotMinus, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 22), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Lt, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 24), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::GtEq, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 25), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Gt, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 27), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::LtEq, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 28), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Eq, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 30), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::EqEq, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 32), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::ExEq, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 34), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Apos, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 36), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Dot, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 37), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Colon, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 38), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::ColonColon, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 40), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 42), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_returns_keyboard_tokens()
{
    let s = "and break by continue else end false fill for function if in inf module nan none not or quit return root to true while";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::And, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Break, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 5), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::By, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 11), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Continue, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 14), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Else, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 23), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::End, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 28), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::False, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 32), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Fill, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 38), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::For, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 43), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Function, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 47), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::If, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 56), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::In, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 59), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Inf, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 62), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Module, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 66), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Nan, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 73), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::None, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 77), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Not, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 82), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Or, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 86), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Quit, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 89), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Return, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 94), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Root, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 101), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::To, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 106), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::True, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 109), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::While, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 114), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 119), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_returns_integer_tokens()
{
    let s = "
1234
0x1234abcf
0XABCF1234
01234
";
    let s2 = &s[1..];
    let mut cursor = Cursor::new(s2.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::Int(1234), pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 5), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Int(0x1234abcf), pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 2, 1), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 2, 11), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Int(0xabcf1234), pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 3, 1), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 3, 11), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Int(1234), pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 4, 1), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 4, 6), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_returns_float_tokens()
{
    let s = "
12.3456
1.5e12
2e10
3E10
4e+10
5e-10
";
    let s2 = &s[1..];
    let mut cursor = Cursor::new(s2.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::Float(n), pos))) => {
            assert_eq!(12.3456, n);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 8), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Float(n), pos))) => {
            assert_eq!(1.5e12, n);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 2, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 2, 7), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Float(n), pos))) => {
            assert_eq!(2e10, n);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 3, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 3, 5), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Float(n), pos))) => {
            assert_eq!(3e10, n);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 4, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 4, 5), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Float(n), pos))) => {
            assert_eq!(4e10, n);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 5, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 5, 6), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Float(n), pos))) => {
            assert_eq!(5e-10, n);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 6, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 6, 6), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_returns_string_token()
{
    let s = "\"abc def ghi jkl ąćę\"";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::String(t), pos))) => {
            assert_eq!(String::from("abc def ghi jkl ąćę"), t);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 22), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_returns_string_token_with_escapes()
{
    let s = "\"\\a\\b\\t\\n\\v\\f\\r\\7\\41\\177\\\\\\\"\"";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::String(t), pos))) => {
            assert_eq!(String::from("\x07\x08\t\n\x0b\x0c\r\x07!\x7f\\\""), t);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 30), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_returns_string_token_with_unicode_escapes()
{
    let s = "\"\\u0104\\U10ffff\"";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::String(t), pos))) => {
            assert_eq!(String::from("\u{0104}\u{10ffff}"), t);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 17), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_returns_identifier_tokens()
{
    let s = "
ABCdef
abcDEF
abc123
_abc123
_123abc
abc_def
ąćę
";
    let s2 = &s[1..];
    let mut cursor = Cursor::new(s2.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Ok((Token::Ident(ident), pos))) => {
            assert_eq!(String::from("ABCdef"), ident);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 7), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Ident(ident), pos))) => {
            assert_eq!(String::from("abcDEF"), ident);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 2, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 2, 7), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Ident(ident), pos))) => {
            assert_eq!(String::from("abc123"), ident);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 3, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 3, 7), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Ident(ident), pos))) => {
            assert_eq!(String::from("_abc123"), ident);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 4, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 4, 8), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Ident(ident), pos))) => {
            assert_eq!(String::from("_123abc"), ident);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 5, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 5, 8), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Ident(ident), pos))) => {
            assert_eq!(String::from("abc_def"), ident);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 6, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 6, 8), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Ident(ident), pos))) => {
            assert_eq!(String::from("ąćę"), ident);
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 7, 1), pos);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        Some(Ok((Token::Newline, pos))) => assert_eq!(Pos::new(Arc::new(String::from("test.un")), 7, 4), pos),
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_take_doc_returns_documentation()
{
    let s = "
## first text
##second text
%% third text
+
";
    let s2 = &s[1..];
    let mut cursor = Cursor::new(s2.as_bytes());
    let mut lexer = Lexer::new_with_doc_flag(Arc::new(String::from("test.un")), &mut cursor, true);
    lexer.next();
    lexer.next();
    lexer.next();
    lexer.next();
    match lexer.take_doc() {
        Some(t) => assert_eq!(String::from("first text\nsecond text\nthird text\n"), t),
        None => assert!(false),
    }
}

#[test]
fn test_lexer_take_doc_does_not_return_documentation()
{
    let s = "+";
    let s2 = &s[1..];
    let mut cursor = Cursor::new(s2.as_bytes());
    let mut lexer = Lexer::new_with_doc_flag(Arc::new(String::from("test.un")), &mut cursor, true);
    lexer.next();
    match lexer.take_doc() {
        None => assert!(true),
        Some(_) => assert!(false),
    }
}

#[test]
fn test_lexer_take_doc_returns_second_documentation()
{
    let s = "
## first text
##second text
%% third text
+
## fourth text
-
";
    let s2 = &s[1..];
    let mut cursor = Cursor::new(s2.as_bytes());
    let mut lexer = Lexer::new_with_doc_flag(Arc::new(String::from("test.un")), &mut cursor, true);
    lexer.next();
    lexer.next();
    lexer.next();
    lexer.next();
    match lexer.take_doc() {
        Some(t) => assert_eq!(String::from("first text\nsecond text\nthird text\n"), t),
        None => assert!(false),
    }
    lexer.next();
    lexer.next();
    lexer.next();
    match lexer.take_doc() {
        Some(t) => assert_eq!(String::from("fourth text\n"), t),
        None => assert!(false),
    }
}

#[test]
fn test_lexer_take_doc_does_not_return_second_documentation()
{
    let s = "
## first text
##second text
%% third text
+
-
";
    let s2 = &s[1..];
    let mut cursor = Cursor::new(s2.as_bytes());
    let mut lexer = Lexer::new_with_doc_flag(Arc::new(String::from("test.un")), &mut cursor, true);
    lexer.next();
    lexer.next();
    lexer.next();
    lexer.next();
    match lexer.take_doc() {
        Some(t) => assert_eq!(String::from("first text\nsecond text\nthird text\n"), t),
        None => assert!(false),
    }
    lexer.next();
    lexer.next();
    match lexer.take_doc() {
        None => assert!(true),
        Some(_) => assert!(false),
    }
}


#[test]
fn test_lexer_next_complains_on_unexpected_character()
{
    let s = "@";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
            assert_eq!(String::from("unexpected character"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_complains_on_unexpected_character_for_exclamation()
{
    let s = "!";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
            assert_eq!(String::from("unexpected character"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_complains_on_no_hexadecimal_digits()
{
    let s = "0xg";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 3), pos);
            assert_eq!(String::from("no hexadecimal digits"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_complains_on_no_decimal_digits()
{
    let s = "0.a";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 3), pos);
            assert_eq!(String::from("no decimal digits"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_complains_on_invalid_number_for_too_large_hexadecimal_number()
{
    let s = "0x8000000000000000";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
            assert_eq!(String::from("invalid number"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_complains_on_invalid_number_for_too_large_decimal_number()
{
    let s = "9223372036854775808";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
            assert_eq!(String::from("invalid number"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_complains_on_invalid_unicode_escape_for_small_unicode_escape_and_invalid_hexadecimal_digit()
{
    let s = "\"\\u123g\"";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 2), pos);
            assert_eq!(String::from("invalid unicode escape"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_complains_on_invalid_unicode_escape_for_big_unicode_escape_and_invalid_hexadecimal_digit()
{
    let s = "\"\\U12345g\"";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 2), pos);
            assert_eq!(String::from("invalid unicode escape"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_complains_on_invalid_unicode_escape_for_big_unicode_escape_and_too_large_number()
{
    let s = "\"\\Uffffff\"";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 2), pos);
            assert_eq!(String::from("invalid unicode escape"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}

#[test]
fn test_lexer_next_complains_on_unclosed_string()
{
    let s = "\"abc";
    let mut cursor = Cursor::new(s.as_bytes());
    let mut lexer = Lexer::new(Arc::new(String::from("test.un")), &mut cursor);
    match lexer.next() {
        Some(Err(Error::Parser(pos, msg))) => {
            assert_eq!(Pos::new(Arc::new(String::from("test.un")), 1, 1), pos);
            assert_eq!(String::from("unclosed string"), msg);
        },
        _ => assert!(false),
    }
    match lexer.next() {
        None => assert!(true),
        _ => assert!(false),
    }
}