trivet 3.1.0

// Trivet
// Copyright (c) 2025 by Stacy Prowell.  All rights reserved.
// https://gitlab.com/binary-tools/trivet

//! Tests of the parser module.

use crate::decoder::Decode;
use crate::errors::ParseResult;
use crate::parse_from_bytes;
use crate::parse_from_path;
use crate::parse_from_string;
use crate::parsers::keyword::KeywordParser;
use crate::Parser;
use crate::ParserCore;
use std::env::temp_dir;
use std::fs::remove_file;
use std::fs::File;
use std::io::Write;
use std::vec;

#[test]
fn console() {
    // Check that the correct loc is returned for console.  File locs are
    // tested several times elsewhere.
    let decoder = Decode::new(vec![]);
    let parser = Parser::new("", decoder);
    assert_eq!(parser.loc().to_string(), "1:1");
}

#[test]
fn trivial() {
    // Test correct handling of an empty stream at start.
    let decoder = Decode::new(vec![]);
    let parser = Parser::new("<string>", decoder);
    let core = ParserCore::new("<null>", Decode::new(vec![]));
    assert!(parser.is_at_eof());
    assert!(core.is_at_eof());
}

#[test]
fn eof() {
    let mut parser = parse_from_string("G");
    #[cfg(not(feature = "no_tracking"))]
    assert_eq!(parser.loc().to_string(), "<string>:1:1");
    assert_eq!(parser.peek(), 'G');
    parser.consume();
    #[cfg(not(feature = "no_tracking"))]
    assert_eq!(parser.loc().to_string(), "<string>:1:2");
    // We're at the end of file, but don't know that until we
    // force the parser to look.
    assert!(parser.is_at_eof());
}

#[test]
fn expect_chars() {
    let mut parser = parse_from_string("Lawyers, Guns, and Money");
    assert!(parser
        .expect_chars(&['L', 'a', 'w', 'y', 'e', 'r', 's'])
        .is_ok());
    assert!(parser.expect(',').is_ok());
    assert!(parser.expect(' ').is_ok());
    assert!(parser.expect('H').is_err());
    assert!(parser.expect_chars(&['G', 'u', 'n', 's', ',', ' ']).is_ok());
    assert!(parser
        .expect_chars(&['a', 'n', 'd', ' ', 'H', 'o', 'n', 'e', 'y'])
        .is_err())
}

#[test]
fn take_while() {
    let mut parser = parse_from_string("Can we find the end?");
    assert_eq!(parser.take_until("C"), "");
    assert_eq!(parser.peek(), 'a');
    assert_eq!(
        parser.take_while(|ch| ch.is_alphanumeric() || ch == ' '),
        "an we find the end"
    );
}

#[test]
fn take_while_unless() {
    let mut parser = parse_from_string("Can we find the end?");
    assert_eq!(parser.take_until("C"), "");
    assert_eq!(parser.peek(), 'a');
    assert_eq!(
        parser.take_while_unless(|ch| ch.is_alphanumeric(), |ch| ch.is_whitespace()),
        "anwefindtheend"
    );
}

#[test]
fn check() {
    let mut parser = parse_from_string("Can we find the end?");
    assert!(!parser.peek_chars(&"Ban".chars().collect::<Vec<char>>()));
    assert!(parser.peek_chars(&"Can".chars().collect::<Vec<char>>()));
    assert!(parser.peek_str("Can"));
    assert!(!parser.peek_str("Flan"));
    assert!(parser.peek_chars(&"Can we".chars().collect::<Vec<char>>()));
}

#[test]
fn peek_n() {
    let mut parser = parse_from_string("Can we find the end?");
    assert_eq!(parser.peek_n(4), "Can ");
    parser.consume_n(4);
    assert_eq!(parser.peek_n(4), "we f");
    parser.consume_n(4);
    assert_eq!(parser.peek_n(4), "ind ");
    parser.consume_n(4);
    assert_eq!(parser.peek_n(4), "the ");
    parser.consume_n(4);
    assert_eq!(parser.peek_n(4), "end?");
    parser.consume_n(4);
    assert!(parser.is_at_eof());
}

#[cfg(not(feature = "no_stall_detection"))]
#[test]
#[should_panic]
fn catch_stall_peek() {
    let mut parser = parse_from_string("Can we find the end?");
    while parser.peek() == 'C' {
        // Do nothing.
    }
}

#[cfg(not(feature = "no_stall_detection"))]
#[test]
#[should_panic]
fn catch_stall_peek_2() {
    let mut parser = parse_from_string("Can we find the end?");
    loop {
        parser.peek_n(17);
    }
}

#[cfg(not(feature = "no_stall_detection"))]
#[test]
#[should_panic]
fn catch_stall_peek_3() {
    let mut parser = parse_from_string("Can we find the end?");
    loop {
        parser.peek_offset(17);
    }
}

#[cfg(not(feature = "no_stall_detection"))]
#[test]
#[should_panic]
fn catch_stall_peek_4() {
    let mut parser = parse_from_string("Can we find the end?");
    loop {
        parser.peek_n_vec(17);
    }
}

#[cfg(not(feature = "no_stall_detection"))]
#[test]
#[should_panic]
fn catch_stall_eof() {
    let mut parser = parse_from_string("C");
    loop {
        parser.consume();
    }
}

#[cfg(not(feature = "no_stall_detection"))]
#[test]
#[should_panic]
fn catch_stall_eof_2() {
    let mut parser = parse_from_string("C");
    loop {
        parser.consume_n(31);
    }
}

#[test]
fn peek_and_consume() {
    let mut parser =
        parse_from_string("the rain \n /*in*/ spain falls\t\t\tmainly/* */ on the plain");
    #[cfg(not(feature = "no_tracking"))]
    assert_eq!(parser.loc().to_string(), "<string>:1:1");
    parser.consume();
    #[cfg(not(feature = "no_tracking"))]
    assert_eq!(parser.loc().to_string(), "<string>:1:2");
    assert_eq!(parser.peek(), 'h');
    parser.consume_n(6);
    assert_eq!(parser.peek(), 'n');
    assert_eq!(parser.take_until("*/"), "n \n /*in");
    assert_eq!(parser.peek(), ' ');
    parser.consume();
    assert_eq!(parser.peek(), 's');
    assert_eq!(parser.peek_n(5), "spain");
    assert!(!parser.peek_str("spaim"));
    assert!(!parser.peek_and_consume_str("spain  "));
    assert!(parser.peek_and_consume_str("spain"));
    #[cfg(not(feature = "no_tracking"))]
    assert_eq!(parser.loc().to_string(), "<string>:2:14");
    #[cfg(not(feature = "no_tracking"))]
    assert_eq!(parser.borrow_core().get_column_number(), 14);
    #[cfg(not(feature = "no_tracking"))]
    assert_eq!(parser.borrow_core().get_line_number(), 2);
    assert!(!parser.peek_and_consume_str(" falz"));
    assert!(parser.peek_and_consume_str(" falls"));
    assert!(parser.consume_ws_only());
    assert!(!parser.consume_ws_only());
    assert!(!parser.consume_ws_only());
    assert!(!parser.peek_and_consume('n'));
    assert!(parser.peek_and_consume_str("mainly"));
    assert_eq!(parser.take_until("*/"), "/* ");
    assert!(parser.consume_ws_only());
    assert!(parser.peek_str("on the plain"));
}

#[test]
fn take() {
    let mut parser = parse_from_bytes(b"any given day:104_218,ab7ac-8ab9--4");
    assert_eq!(
        parser.take_while(|ch| ch.is_alphabetic() || ch.is_whitespace()),
        "any given day"
    );
    assert!(parser.peek_and_consume(':'));
    assert_eq!(
        parser.take_while_unless(|ch| ch.is_numeric(), |ch| ch == '_'),
        "104218"
    );
    assert!(parser.peek_and_consume(','));
    assert_eq!(parser.take_until("--"), "ab7ac-8ab9");
}

#[test]
fn consume() {
    let mut parser = parse_from_bytes(b"any given day:104_218,ab7ac-8ab9--4");
    assert!(parser.consume_while(|ch| ch.is_alphabetic() || ch.is_whitespace()));
    assert!(parser.peek_and_consume(':'));
    assert!(!parser.consume_while(|ch| ch.is_alphabetic()));
    assert!(parser.consume_while(|ch| { ch.is_numeric() || ch == '_' }));
    assert!(parser.peek_and_consume(','));
    assert!(parser.consume_until("--"));
    assert!(parser.peek_str("4"));
}

#[test]
fn file() {
    // Get a file path.
    let mut buf = temp_dir();
    buf.push("trivet-test.tmp");

    // Remove the file if it exists.
    let _ = remove_file(buf.clone());

    // Create a file.
    let mut file = File::create(buf.clone()).expect("Unable to create file");
    write!(
        file,
        r#"
        {{
            "Ḽơᶉëᶆ" : "ȋṕšᶙṁ" ,
            "ḍỡḽǭᵳ" : [ "ʂǐť", "ӓṁệẗ" ] ,
            "Lorem" : {{
                "ipsum":["dolor","sit","amet"]
            }}
        }}
        "#
    )
    .expect("Unable to write to file");
    let _ = file.flush();

    // Parse the file.
    let mut parser = parse_from_path(&buf).expect("Unable to open file.");
    let thing = parser.take_while_unless(|_| true, |ch| ch.is_whitespace());
    assert_eq!(
        thing,
        r#"{"Ḽơᶉëᶆ":"ȋṕšᶙṁ","ḍỡḽǭᵳ":["ʂǐť","ӓṁệẗ"],"Lorem":{"ipsum":["dolor","sit","amet"]}}"#
    );
}

#[test]
fn peek_edge() {
    // Edge case for peek_n.
    let mut parser = parse_from_bytes(b"any ");
    assert_eq!(parser.peek_n(0), "");
    assert!(parser.peek_str(""));
    assert!(parser.peek_chars(&Vec::<char>::new()));
    assert_eq!(parser.peek_n(10947), "any ");
    assert!(!parser.peek_chars(&['a', 'n', 'y', ' ', '\0']));
}

#[test]
fn takes() {
    // Edge case for peek_n.
    let mut parser = parse_from_bytes(b"xyzzy \t\n\r 92_8346_46trivet");
    assert_eq!(parser.take_while(|ch| ch.is_alphabetic()), "xyzzy");
    assert_eq!(parser.take_while(|ch| ch.is_whitespace()), " \t\n\r ");
    assert_eq!(
        parser.take_while_unless(|ch| ch.is_ascii_digit(), |ch| ch == '_'),
        "92834646"
    );
    assert_eq!(parser.take_until("<"), "trivet");
    assert!(parser.is_at_eof());
}

#[test]
fn consume_2() {
    // Edge case for peek_n.
    let mut parser = parse_from_bytes(b"xyzzy \t\n\r 92_8346_46trivet");
    assert!(parser.peek_and_consume_chars(&['x', 'y', 'z', 'z', 'y', ' ']));
    assert!(!parser.peek_and_consume_ws('#'));
    assert!(parser.peek_and_consume_ws('\t'));
    let mut parser = parse_from_bytes(b"xyzzy \t\n\r 92_8346_46trivet");
    assert!(!parser.peek_and_consume_str_ws("xyzzo"));
    assert!(parser.peek_and_consume_str_ws("xyzzy"));
    assert_eq!(parser.peek(), '9');
    assert!(parser.consume_until("ve"));
    assert_eq!(parser.peek(), 't');
}

#[test]
fn comments() {
    // Test a little comment processing.
    let text = r#"
    // This is a comment.
    # This is also a comment, and should be skipped.
    /* This is a longer comment.
       Second line of longer comment. */
    token
    # Final comment.
    "#;
    let mut parser = parse_from_string(text);
    assert!(parser.consume_ws());
    assert!(!parser.consume_ws());
    assert_eq!(parser.peek(), '#');
    parser.borrow_comment_parser().enable_python = true;
    assert!(parser.consume_ws());
    assert!(!parser.consume_ws());
    assert!(parser.peek_and_consume_chars_ws(&['t', 'o', 'k', 'e', 'n']));
    assert!(parser.is_at_eof());
    let mut parser = parse_from_string(text);
    parser.borrow_comment_parser().enable_python = false;
    parser.borrow_comment_parser().enable_c = false;
    parser.borrow_comment_parser().enable_cpp = false;
    assert!(parser.consume_ws());
    assert_eq!(parser.peek(), '/');
}

#[test]
fn greedy() {
    // Test greedy peek.
    let mut parser = parse_from_string("''''''''");
    while !parser.is_at_eof() && !parser.peek_str_greedy("'''") {
        parser.consume();
    }
    // Consume the end marker.
    parser.consume_n(3);
    assert!(parser.is_at_eof());
    // Failing peeks.
    let mut parser = parse_from_string("abbbaaa");
    assert!(parser.peek_str_greedy("a"));
    parser.consume();
    assert!(!parser.peek_str_greedy("bb"));
    parser.consume();
    assert!(parser.peek_str_greedy("bb"));
    parser.consume();
    parser.consume();
    assert!(!parser.peek_str_greedy("aaaa"));
    assert!(parser.peek_str_greedy("aaa"));
    assert!(!parser.peek_str_greedy("aa"));
}

#[test]
fn embedded_parsers() -> ParseResult<()> {
    let mut parser = parse_from_string("\"This is a string\"0x17.4");
    let numpar = parser.borrow_number_parser();
    numpar.settings.permit_binary = false;
    numpar.settings.permit_octal = false;
    let strpar = parser.borrow_string_parser();
    strpar.set(crate::strings::StringStandard::Trivet);
    parser.consume();
    let string = parser.parse_string_until_delimiter('"')?;
    assert_eq!(string, "This is a string");
    let number = parser.parse_f64()?;
    assert_eq!(number, 23.25);
    let mut parser = parse_from_string("17_21  0x21  -12  0b101 48.5 1.1e-7");
    assert_eq!(parser.parse_u128()?, 1721);
    parser.consume_ws_only();
    assert_eq!(parser.parse_u128_ws()?, 0x21);
    assert_eq!(parser.parse_i128()?, -12);
    parser.consume_ws_only();
    assert_eq!(parser.parse_i128_ws()?, 5);
    assert_eq!(parser.parse_f64()?, 48.5);
    parser.consume_ws_only();
    assert_eq!(parser.parse_f64_ws()?, 1.1e-7);
    Ok(())
}

#[test]
fn parse_items() -> ParseResult<()> {
    let mut parser = parse_from_string(r#""Down in Ohio" 2112 -2112 -21.12"#);
    parser.parse_comments = false;
    assert_eq!(parser.parse_string_match_delimiter_ws()?, "Down in Ohio");
    assert_eq!(parser.parse_u128_ws()?, 2112);
    assert_eq!(parser.parse_i64_ws()?, -2112);
    assert_eq!(parser.parse_f64_ws()?, -21.12);

    let mut parser = parse_from_string(r#""#);
    assert_eq!(parser.parse_string_match_delimiter()?, "");
    assert_eq!(parser.parse_string_match_delimiter_ws()?, "");

    let mut parser = parse_from_string(r#" This is unusual « 776"#);
    assert_eq!(
        parser.parse_string_until_delimiter_ws('«')?,
        " This is unusual "
    );
    assert_eq!(parser.parse_u128()?, 776);
    assert_eq!(parser.parse_string(r#"\r\n\t\t"#)?, "\r\n\t\t");
    Ok(())
}

#[test]
fn parse_numbers_test() {
    let mut parser = parse_from_string("982735");
    assert_eq!(parser.parse_u64().unwrap(), 982735);
    let mut parser = parse_from_string("-982735");
    assert_eq!(parser.parse_i64().unwrap(), -982735);
    let mut parser = parse_from_string("-982735.1423e1");
    assert_eq!(parser.parse_f64_decimal().unwrap(), -982735.1423e1);
    let mut parser = parse_from_string("982735 587432 628596");
    assert_eq!(parser.parse_u64_ws().unwrap(), 982735);
    assert_eq!(parser.parse_u64_ws().unwrap(), 587432);
    assert_eq!(parser.parse_u64_ws().unwrap(), 628596);
    let mut parser = parse_from_string("-982735 587432 -628596");
    assert_eq!(parser.parse_i64_ws().unwrap(), -982735);
    assert_eq!(parser.parse_i64_ws().unwrap(), 587432);
    assert_eq!(parser.parse_i64_ws().unwrap(), -628596);
    let mut parser = parse_from_string("-982735.1423e1 5874.e32 -62.859e6");
    assert_eq!(parser.parse_f64_decimal_ws().unwrap(), -982735.1423e1);
    assert_eq!(parser.parse_f64_decimal_ws().unwrap(), 5874.0e32);
    assert_eq!(parser.parse_f64_decimal_ws().unwrap(), -62.859e6);
}

#[test]
fn keywords_test_1() {
    let mut parser = parse_from_string("proper haberdashery _nom_nom Fritz_1");
    assert_eq!(parser.parse_keyword().unwrap(), "proper");
    parser.consume();
    assert_eq!(parser.parse_keyword_ws().unwrap(), "haberdashery");
    assert_eq!(parser.parse_keyword_ws().unwrap(), "_nom_nom");
    assert_eq!(parser.parse_keyword().unwrap(), "Fritz_1");
}

#[test]
fn keywords_test_2() {
    let mut parser = parse_from_string("PROPER Haber-Dashery -nom-nom");
    parser.borrow_keyword_parser().transclude_hyphens = true;
    assert_eq!(parser.parse_keyword().unwrap(), "PROPER");
    parser.consume();
    assert_eq!(parser.parse_keyword_ws().unwrap(), "Haber_Dashery");
    assert_eq!(parser.parse_keyword_ws().unwrap(), "_nom_nom");
    let mut kwp = KeywordParser::new();
    kwp.transform = Box::new(|ch: char| if !ch.is_ascii() { '*' } else { ch });
    kwp.use_transform = true;
    kwp.permit =
        Box::new(|state: i32, ch: char| (ch.is_alphanumeric() || ch == '_' || ch == '*', state));
    kwp.use_permit = true;
    let mut parser = parse_from_string("_first_выб move_либо быв_line");
    let _ = parser.replace_keyword_parser(kwp);
    assert_eq!(parser.parse_keyword_ws().unwrap(), "_first_***");
    assert_eq!(parser.parse_keyword_ws().unwrap(), "move_****");
    assert_eq!(parser.parse_keyword_ws().unwrap(), "***_line");
}

#[test]
fn peek_offset_test() {
    let mut parser = parse_from_string(r#""Down in Ohio" 2112 -2112 -21.12f"#);
    assert_eq!(parser.peek_offset(0), '"');
    assert_eq!(parser.peek_offset(17), '1');
    assert_eq!(parser.peek_offset(16), '1');
    assert_eq!(parser.peek_offset(15), '2');
    assert_eq!(parser.peek_offset(0), '"');
    assert_eq!(parser.peek_offset(17), '1');
    assert_eq!(parser.peek_offset(32), 'f');
    assert_eq!(parser.peek_offset(33), '\0');
    assert!(parser.peek_and_consume_str(r#""Down in Ohio""#));
    assert_eq!(parser.peek_offset(1000000000000), '\0');
}

#[test]
fn take_tests() {
    let mut parser = parse_from_string(
        r#"
        This is a test.
        This is a DIFFERENT test, but largely the same.
        We should parse this 1 word-by-word, right?
        (OIYGOU^ER$I&^%^(&*{POJ:NL *^CROGIOILUI})
        12_134.231_22e1_234
    "#,
    );

    parser.consume_ws();
    assert_eq!(
        parser.take(|_| false, |ch| ch == '.'),
        ("This is a test".chars().collect(), Some('.'))
    );
    parser.peek_and_consume('.');

    parser.consume_ws();
    assert_eq!(
        parser.take(|_| false, |ch| ch == '.'),
        (
            "This is a DIFFERENT test, but largely the same"
                .chars()
                .collect(),
            Some('.')
        )
    );
    parser.peek_and_consume('.');

    parser.consume_ws();
    assert_eq!(
        parser.take(|ch| ch == ' ' || ch.is_ascii_punctuation(), |ch| ch == '\n'),
        (
            "Weshouldparsethis1wordbywordright".chars().collect(),
            Some('\n')
        )
    );
    parser.peek_and_consume('?');

    parser.consume_ws();
    assert_eq!(
        parser.take(|ch| ch.is_alphabetic(), |ch| ch == ')'),
        ("(^$&^%^(&*{: *^}".chars().collect(), Some(')'))
    );
    parser.peek_and_consume(')');

    parser.consume_ws();
    assert_eq!(
        parser.take(
            |ch| ch == '_',
            |ch| ch == '.' || ch == 'e' || !ch.is_alphanumeric()
        ),
        ("12134".chars().collect(), Some('.'))
    );
    parser.peek_and_consume('.');

    parser.consume_ws();
    assert_eq!(
        parser.take(
            |ch| ch == '_',
            |ch| ch == '.' || ch == 'e' || !ch.is_alphanumeric()
        ),
        ("23122".chars().collect(), Some('e'))
    );
    parser.peek_and_consume('e');

    parser.consume_ws();
    assert_eq!(
        parser.take(
            |ch| ch == '_',
            |ch| ch == '.' || ch == 'e' || !ch.is_alphanumeric()
        ),
        ("1234".chars().collect(), Some('\n'))
    );
    parser.peek_and_consume('\n');

    dbg!(parser.peek_n(10));
    assert_eq!(parser.take(|ch| ch == ' ', |ch| ch == '$'), (vec![], None))
}

#[test]
fn take_until_greedy_test() -> ParseResult<()> {
    let mut parser = parse_from_string(
        r#"
        """This is a string in quotes."""
        """"This is a string in quotes.""""
        """""This is a string in quotes."""""
        """This is a string.""
    "#,
    );
    parser.consume_ws();
    parser.peek_and_consume_chars(&['"', '"', '"']);
    assert_eq!(
        parser.take_until_greedy(&['"', '"', '"'], false)?,
        "This is a string in quotes.".to_string()
    );
    parser.consume_ws();
    parser.peek_and_consume_chars(&['"', '"', '"']);
    assert_eq!(
        parser.take_until_greedy(&['"', '"', '"'], true)?,
        r#""This is a string in quotes.""#.to_string()
    );
    parser.consume_ws();
    parser.peek_and_consume_chars(&['"', '"', '"']);
    assert_eq!(
        parser.take_until_greedy(&['"', '"', '"'], true)?,
        r#"""This is a string in quotes."""#.to_string()
    );
    parser.consume_ws();
    parser.peek_and_consume_chars(&['"', '"', '"']);
    assert!(parser.take_until_greedy(&['"', '"', '"'], true).is_err());
    assert!(parser.take_until_greedy(&['K'], false).is_ok());
    Ok(())
}

#[test]
fn peek_n_vec_test() {
    let mut parser = parse_from_string(r#"abbracadabra"#);
    assert_eq!(parser.peek_n_vec(0), vec![]);
    assert_eq!(parser.peek_n_vec(1), vec!['a']);
    assert_eq!(parser.peek_n_vec(2), vec!['a', 'b']);
    assert_eq!(parser.peek_n_vec(3), vec!['a', 'b', 'b']);
    assert_eq!(parser.peek_n_vec(4), vec!['a', 'b', 'b', 'r']);
    assert_eq!(parser.peek_n_vec(5), vec!['a', 'b', 'b', 'r', 'a']);
    assert_eq!(
        parser.peek_n_vec(16000),
        "abbracadabra".chars().collect::<Vec<char>>()
    );
}

#[test]
fn parse_string_match_delimiter_ws_test() -> ParseResult<()> {
    let mut parser = parse_from_string(
        r#"
        "This is a string to parse."
        'This is a string to parse.'
        $This is a string to parse.$
    "#,
    );
    parser.consume_ws();
    assert_eq!(
        parser.parse_string_match_delimiter_ws()?,
        "This is a string to parse.".to_string()
    );
    assert_eq!(
        parser.parse_string_match_delimiter_ws()?,
        "This is a string to parse.".to_string()
    );
    assert_eq!(
        parser.parse_string_match_delimiter_ws()?,
        "This is a string to parse.".to_string()
    );
    Ok(())
}

#[test]
fn peek_chars_greedy_test() {
    let mut parser = parse_from_string("$$$");
    assert!(parser.peek_chars_greedy(&[]));
    assert!(!parser.peek_chars_greedy(&['$', '$']));
    parser.consume();
    assert!(parser.peek_chars_greedy(&['$', '$']));
}