luau-syntax 0.732.0

use luau_syntax::allocator::AstArena;
use luau_syntax::ast_names::AstNameTable;
use luau_syntax::lexer::{Lexer, QuoteStyle, ReservedWord, Token};

#[derive(Debug, PartialEq, Eq)]
struct TestLocation {
    begin_line: usize,
    begin_column: usize,
    end_line: usize,
    end_column: usize,
}

#[derive(Debug, PartialEq, Eq)]
enum TestToken {
    Eof,
    BrokenString,
    BrokenComment,
    BrokenUnicode {
        codepoint: u32,
    },
    BlockComment(usize),
    QuotedString {
        value: Vec<u8>,
        quote_style: QuoteStyle,
    },
    RawString {
        block_depth: usize,
    },
}

impl TestToken {
    fn from_token(token: Token<'_, '_>) -> Self {
        match token {
            Token::Eof => Self::Eof,
            Token::BrokenString => Self::BrokenString,
            Token::BrokenComment => Self::BrokenComment,
            Token::BrokenUnicode { codepoint } => Self::BrokenUnicode { codepoint },
            Token::BlockComment(block_depth) => Self::BlockComment(block_depth),
            Token::QuotedString { value, quote_style } => Self::QuotedString {
                value: value.to_vec(),
                quote_style,
            },
            Token::RawString { block_depth, .. } => Self::RawString { block_depth },
            other => panic!("unexpected token returned from owned test helper: {other:?}"),
        }
    }
}

macro_rules! with_lexer {
    ($source:expr, $lexer:ident, $body:block) => {{
        let arena = AstArena::new();
        let mut names = AstNameTable::new(&arena);
        let mut $lexer = Lexer::new($source, &mut names);
        $body
    }};
}

fn next(source: &str, skip_comments: bool) -> (TestToken, TestLocation) {
    with_lexer!(source.as_bytes(), lexer, {
        let (token, start, end) = lexer.next_token_span(skip_comments);
        (TestToken::from_token(token), location(source, start, end))
    })
}

fn byte_token_sequence(source: &[u8]) -> Vec<(TestToken, TestLocation)> {
    with_lexer!(source, lexer, {
        let mut tokens = Vec::new();

        loop {
            let (token, start, end) = lexer.next_token_span(false);
            if token == Token::Eof {
                break;
            }
            tokens.push((
                TestToken::from_token(token),
                byte_location(source, start, end),
            ));
        }

        tokens
    })
}

fn assert_ident_token(token: Token<'_, '_>, expected: &str) {
    match token {
        Token::Ident(name) => assert_eq!(name, expected),
        other => panic!("expected identifier {expected:?}, got {other:?}"),
    }
}

fn location(source: &str, start: usize, end: usize) -> TestLocation {
    let (begin_line, begin_column) = position(source, start);
    let (end_line, end_column) = position(source, end);

    TestLocation {
        begin_line,
        begin_column,
        end_line,
        end_column,
    }
}

fn position(source: &str, offset: usize) -> (usize, usize) {
    let before = &source[..offset];
    let line = before.bytes().filter(|byte| *byte == b'\n').count();
    let column = before
        .bytes()
        .rev()
        .take_while(|byte| *byte != b'\n')
        .count();
    (line, column)
}

fn byte_location(source: &[u8], start: usize, end: usize) -> TestLocation {
    let (begin_line, begin_column) = byte_position(source, start);
    let (end_line, end_column) = byte_position(source, end);

    TestLocation {
        begin_line,
        begin_column,
        end_line,
        end_column,
    }
}

fn byte_position(source: &[u8], offset: usize) -> (usize, usize) {
    let before = &source[..offset];
    let line = before.iter().filter(|byte| **byte == b'\n').count();
    let column = before
        .iter()
        .rev()
        .take_while(|byte| **byte != b'\n')
        .count();
    (line, column)
}

// Lexer.test.cpp: broken_string_works
#[test]
fn broken_string_works() {
    let (token, location) = next("[[", false);

    assert_eq!(token, TestToken::BrokenString);
    assert_eq!(
        location,
        TestLocation {
            begin_line: 0,
            begin_column: 0,
            end_line: 0,
            end_column: 2,
        }
    );
}

// Lexer.test.cpp: broken_comment
#[test]
fn broken_comment() {
    let (token, location) = next("--[[  ", false);

    assert_eq!(token, TestToken::BrokenComment);
    assert_eq!(
        location,
        TestLocation {
            begin_line: 0,
            begin_column: 0,
            end_line: 0,
            end_column: 6,
        }
    );
}

// Lexer.test.cpp: broken_comment_kept
#[test]
fn broken_comment_kept_when_comments_are_skipped() {
    let (token, _) = next("--[[  ", true);

    assert_eq!(token, TestToken::BrokenComment);
}

// Parser.test.cpp: lex_broken_unicode
#[test]
fn lex_broken_unicode() {
    let tokens = byte_token_sequence(b"\xFF\xFE\xE2\x98\x83\xE2\x80\xA4");

    assert_eq!(tokens.len(), 4);
    assert_eq!(
        tokens[0],
        (
            TestToken::BrokenUnicode { codepoint: 0 },
            TestLocation {
                begin_line: 0,
                begin_column: 0,
                end_line: 0,
                end_column: 1,
            },
        )
    );
    assert_eq!(
        tokens[1],
        (
            TestToken::BrokenUnicode { codepoint: 0 },
            TestLocation {
                begin_line: 0,
                begin_column: 1,
                end_line: 0,
                end_column: 2,
            },
        )
    );
    assert!(matches!(
        tokens[2].0,
        TestToken::BrokenUnicode { codepoint: 0x2603 }
    ));
    assert_eq!(
        tokens[2].1,
        TestLocation {
            begin_line: 0,
            begin_column: 2,
            end_line: 0,
            end_column: 5,
        }
    );
    assert!(matches!(
        tokens[3].0,
        TestToken::BrokenUnicode { codepoint: 0x2024 }
    ));
    assert_eq!(
        tokens[3].1,
        TestLocation {
            begin_line: 0,
            begin_column: 5,
            end_line: 0,
            end_column: 8,
        }
    );
}

// Lexer.test.cpp: comment_skipped
#[test]
fn comment_skipped() {
    let (token, _) = next("--  ", true);

    assert_eq!(token, TestToken::Eof);
}

// Lexer.test.cpp: multilineCommentWithLexemeInAndAfter
#[test]
fn multiline_comment_with_lexeme_in_and_after() {
    let source = "--[[ function \n]] end";
    with_lexer!(source.as_bytes(), lexer, {
        let (comment, start, end) = lexer.next_token_span(false);
        let (token_end, end_start, end_end) = lexer.next_token_span(false);

        assert_eq!(comment, Token::BlockComment(0));
        assert_eq!(
            location(source, start, end),
            TestLocation {
                begin_line: 0,
                begin_column: 0,
                end_line: 1,
                end_column: 2,
            }
        );
        assert_eq!(token_end, Token::Reserved(ReservedWord::End));
        assert_eq!(
            location(source, end_start, end_end),
            TestLocation {
                begin_line: 1,
                begin_column: 3,
                end_line: 1,
                end_column: 6,
            }
        );
    });
}

// Lexer.test.cpp: testBigDelimiters
#[test]
fn test_big_delimiters() {
    let source = "--[===[\n\n\n\n]===]";
    let (token, location) = next(source, false);

    assert!(matches!(token, TestToken::BlockComment(_)));
    assert_eq!(
        location,
        TestLocation {
            begin_line: 0,
            begin_column: 0,
            end_line: 4,
            end_column: 5,
        }
    );
}

// Lexer.test.cpp: testBrokenEscapeTolerant
#[test]
fn test_broken_escape_tolerant() {
    let source = "'\\3729472897292378'";
    let (token, location) = next(source, false);

    assert!(matches!(token, TestToken::QuotedString { .. }));
    assert_eq!(
        location,
        TestLocation {
            begin_line: 0,
            begin_column: 0,
            end_line: 0,
            end_column: source.len(),
        }
    );
}

// Lexer.test.cpp: lookahead
#[test]
fn lookahead() {
    let source = "foo --[[ comment ]] bar : nil end";
    with_lexer!(source.as_bytes(), lexer, {
        lexer.next_lexeme(true);
        assert_ident_token(lexer.current_lexeme().token, "foo");
        assert_ident_token(lexer.lookahead_lexeme(true).token, "bar");

        lexer.next_lexeme(true);
        assert_ident_token(lexer.current_lexeme().token, "bar");
        assert_eq!(lexer.lookahead_lexeme(true).token, Token::Colon);

        lexer.next_lexeme(true);
        assert_eq!(lexer.current_lexeme().token, Token::Colon);
        assert_eq!(
            lexer.lookahead_lexeme(true).token,
            Token::Reserved(ReservedWord::Nil)
        );

        lexer.next_lexeme(true);
        assert_eq!(
            lexer.current_lexeme().token,
            Token::Reserved(ReservedWord::Nil)
        );
        assert_eq!(
            lexer.lookahead_lexeme(true).token,
            Token::Reserved(ReservedWord::End)
        );

        lexer.next_lexeme(true);
        assert_eq!(
            lexer.current_lexeme().token,
            Token::Reserved(ReservedWord::End)
        );
        assert_eq!(lexer.lookahead_lexeme(true).token, Token::Eof);

        lexer.next_lexeme(true);
        assert_eq!(lexer.current_lexeme().token, Token::Eof);
        assert_eq!(lexer.lookahead_lexeme(true).token, Token::Eof);
    });
}

// Lexer.test.cpp: string_interpolation_basic
#[test]
fn string_interpolation_basic() {
    let source = r#"`foo {"bar"}`"#;
    with_lexer!(source.as_bytes(), lexer, {
        assert!(matches!(
            lexer.next_token_span(false).0,
            Token::InterpStringBegin(_)
        ));
        assert!(matches!(
            lexer.next_token_span(false).0,
            Token::QuotedString { .. }
        ));

        let (token, start, _) = lexer.next_token_span(false);
        assert_eq!(token, Token::InterpStringEnd(b""));
        assert_eq!(position(source, start).1, 11);
    });
}

// Lexer.test.cpp: string_interpolation_full
#[test]
fn string_interpolation_full() {
    let source = r#"`foo {"bar"} {"baz"} end`"#;
    with_lexer!(source.as_bytes(), lexer, {
        assert_eq!(
            lexer.next_token_span(false).0,
            Token::InterpStringBegin(b"foo ")
        );
        assert_eq!(
            lexer.next_token_span(false).0,
            Token::QuotedString {
                value: b"bar",
                quote_style: QuoteStyle::Double,
            }
        );

        let (token, start, _) = lexer.next_token_span(false);
        assert_eq!(token, Token::InterpStringMid(b" "));
        assert_eq!(position(source, start).1, 11);

        assert_eq!(
            lexer.next_token_span(false).0,
            Token::QuotedString {
                value: b"baz",
                quote_style: QuoteStyle::Double,
            }
        );

        let (token, start, _) = lexer.next_token_span(false);
        assert_eq!(token, Token::InterpStringEnd(b" end"));
        assert_eq!(position(source, start).1, 19);
    });
}

// Lexer.test.cpp: string_interpolation_double_brace
#[test]
fn string_interpolation_double_brace() {
    let source = r"`foo{{bad}}bar`";
    with_lexer!(source.as_bytes(), lexer, {
        assert_eq!(
            lexer.next_token_span(false).0,
            Token::BrokenInterpDoubleBrace(b"foo")
        );
        assert!(matches!(lexer.next_token_span(false).0, Token::Ident(_)));
        assert_eq!(
            lexer.next_token_span(false).0,
            Token::InterpStringEnd(b"}bar")
        );
    });
}

// Lexer.test.cpp: string_interpolation_double_but_unmatched_brace
#[test]
fn string_interpolation_double_but_unmatched_brace() {
    let source = r"`{{oops}`, 1";
    with_lexer!(source.as_bytes(), lexer, {
        assert!(matches!(
            lexer.next_token_span(false).0,
            Token::BrokenInterpDoubleBrace(_)
        ));
        assert!(matches!(lexer.next_token_span(false).0, Token::Ident(_)));
        assert!(matches!(
            lexer.next_token_span(false).0,
            Token::InterpStringEnd(_)
        ));
        assert_eq!(lexer.next_token_span(false).0, Token::Comma);
        assert!(matches!(lexer.next_token_span(false).0, Token::Number(_)));
    });
}

// Lexer.test.cpp: string_interpolation_unmatched_brace
#[test]
fn string_interpolation_unmatched_brace() {
    let source = r#"{
        `hello {"world"}
    } -- this might be incorrectly parsed as a string"#;
    with_lexer!(source.as_bytes(), lexer, {
        assert_eq!(lexer.next_token_span(false).0, Token::LeftBrace);
        assert_eq!(
            lexer.next_token_span(false).0,
            Token::InterpStringBegin(b"hello ")
        );
        assert_eq!(
            lexer.next_token_span(false).0,
            Token::QuotedString {
                value: b"world",
                quote_style: QuoteStyle::Double,
            }
        );
        assert_eq!(lexer.next_token_span(false).0, Token::BrokenString);
        assert_eq!(lexer.next_token_span(false).0, Token::RightBrace);
    });
}

// Lexer.test.cpp: string_interpolation_with_unicode_escape
#[test]
fn string_interpolation_with_unicode_escape() {
    with_lexer!(r"`\u{1F41B}`".as_bytes(), lexer, {
        assert!(matches!(
            lexer.next_token_span(false).0,
            Token::InterpStringSimple(_)
        ));
        assert_eq!(lexer.next_token_span(false).0, Token::Eof);
    });
}

// Lexer.test.cpp: single_quoted_string
#[test]
fn single_quoted_string() {
    let (token, _) = next("'test'", false);

    assert_eq!(
        token,
        TestToken::QuotedString {
            value: b"test".to_vec(),
            quote_style: QuoteStyle::Single,
        }
    );
}

// Lexer.test.cpp: double_quoted_string
#[test]
fn double_quoted_string() {
    let (token, _) = next(r#""test""#, false);

    assert_eq!(
        token,
        TestToken::QuotedString {
            value: b"test".to_vec(),
            quote_style: QuoteStyle::Double,
        }
    );
}

fn raw_string_block_depth(source: &str) -> usize {
    let (token, _) = next(source, false);
    assert!(
        matches!(token, TestToken::RawString { .. }),
        "expected raw string, got {:?}",
        token
    );
    let TestToken::RawString { block_depth, .. } = token else {
        unreachable!()
    };
    block_depth
}

fn comment_block_depth(source: &str) -> usize {
    let (token, _) = next(source, false);
    assert!(
        matches!(token, TestToken::BlockComment(_)),
        "expected block comment, got {:?}",
        token
    );
    let TestToken::BlockComment(block_depth) = token else {
        unreachable!()
    };
    block_depth
}

// Lexer.test.cpp: lexer_determines_string_block_depth_0
#[test]
fn lexer_determines_string_block_depth_0() {
    assert_eq!(raw_string_block_depth("[[ test ]]"), 0);
}

// Lexer.test.cpp: lexer_determines_string_block_depth_0_multiline_1
#[test]
fn lexer_determines_string_block_depth_0_multiline_1() {
    assert_eq!(
        raw_string_block_depth(
            r"[[ test
    ]]"
        ),
        0
    );
}

// Lexer.test.cpp: lexer_determines_string_block_depth_0_multiline_2
#[test]
fn lexer_determines_string_block_depth_0_multiline_2() {
    assert_eq!(
        raw_string_block_depth(
            r"[[
    test
    ]]"
        ),
        0
    );
}

// Lexer.test.cpp: lexer_determines_string_block_depth_0_multiline_3
#[test]
fn lexer_determines_string_block_depth_0_multiline_3() {
    assert_eq!(
        raw_string_block_depth(
            r"[[
    test ]]"
        ),
        0
    );
}

// Lexer.test.cpp: lexer_determines_string_block_depth_1
#[test]
fn lexer_determines_string_block_depth_1() {
    assert_eq!(raw_string_block_depth("[=[[%s]]=]"), 1);
}

// Lexer.test.cpp: lexer_determines_string_block_depth_2
#[test]
fn lexer_determines_string_block_depth_2() {
    assert_eq!(raw_string_block_depth("[==[ test ]==]"), 2);
}

// Lexer.test.cpp: lexer_determines_string_block_depth_2_multiline_1
#[test]
fn lexer_determines_string_block_depth_2_multiline_1() {
    assert_eq!(
        raw_string_block_depth(
            r"[==[ test
    ]==]"
        ),
        2
    );
}

// Lexer.test.cpp: lexer_determines_string_block_depth_2_multiline_2
#[test]
fn lexer_determines_string_block_depth_2_multiline_2() {
    assert_eq!(
        raw_string_block_depth(
            r"[==[
    test
    ]==]"
        ),
        2
    );
}

// Lexer.test.cpp: lexer_determines_string_block_depth_2_multiline_3
#[test]
fn lexer_determines_string_block_depth_2_multiline_3() {
    assert_eq!(
        raw_string_block_depth(
            r"[==[

    test ]==]"
        ),
        2
    );
}

// Lexer.test.cpp: lexer_determines_comment_block_depth_0
#[test]
fn lexer_determines_comment_block_depth_0() {
    assert_eq!(comment_block_depth("--[[ test ]]"), 0);
}

// Lexer.test.cpp: lexer_determines_comment_block_depth_1
#[test]
fn lexer_determines_comment_block_depth_1() {
    assert_eq!(comment_block_depth("--[=[ μέλλον ]=]"), 1);
}

// Lexer.test.cpp: lexer_determines_comment_block_depth_2
#[test]
fn lexer_determines_comment_block_depth_2() {
    assert_eq!(comment_block_depth("--[==[ test ]==]"), 2);
}