pos-tagger 0.2.0

//! Tokenizer conformance tests.
//!
//! The table reproduces the documented tokenizer cases. Each row is an input and
//! its expected token list. The edge cases at the end cover empty input, a single
//! bare word, hyphenated words, and a dotted abbreviation.

use pos_tagger::Lexer;

fn lex(input: &str) -> Vec<String> {
    Lexer::new().lex(input)
}

#[test]
fn tokenizes_documented_cases() {
    let cases: &[(&str, &[&str])] = &[
        // Dollar sign splits off, the number stays whole.
        ("I made $5.42 today.", &["I", "made", "$", "5.42", "today", "."]),
        // Dollar sign with a word following.
        ("I made $some today.", &["I", "made", "$", "some", "today", "."]),
        // Parentheses are individual tokens, trailing ").":  ")" then ".".
        (
            "I made $5.42 today (but it should have been more).",
            &[
                "I", "made", "$", "5.42", "today", "(", "but", "it", "should", "have", "been",
                "more", ")", ".",
            ],
        ),
        // Apostrophe in "I'm" splits into I, ', m.
        ("I'm fixing issue #6.", &["I", "'", "m", "fixing", "issue", "#", "6", "."]),
        // Hash with a word following.
        ("I'm fixing issue #tags.", &["I", "'", "m", "fixing", "issue", "#", "tags", "."]),
        // Times stay whole, with and without the inner space, any case.
        ("It is now 10:00am", &["It", "is", "now", "10:00am"]),
        ("It is now 10:00 am", &["It", "is", "now", "10:00 am"]),
        ("It is now 10:00pm", &["It", "is", "now", "10:00pm"]),
        ("It is now 10:00 pm", &["It", "is", "now", "10:00 pm"]),
        ("It is now 10:00AM", &["It", "is", "now", "10:00AM"]),
        ("It is now 10:00PM", &["It", "is", "now", "10:00PM"]),
        // Email, URL, and a long ID stay whole.
        (
            "my email is me@exampleabc.com, http://example.com and 19bdnznUXdHEOlp0Pnp9JY0rug6VuA2R3zK4AACdFzhE",
            &[
                "my",
                "email",
                "is",
                "me@exampleabc.com",
                ",",
                "http://example.com",
                "and",
                "19bdnznUXdHEOlp0Pnp9JY0rug6VuA2R3zK4AACdFzhE",
            ],
        ),
        // A full URL with a path and query stays whole.
        (
            "my share url is https://docs.google.com/spreadsheets/d/19bdnznUXdHEOlp0Pnp9JY0rug6VuA2R3zK4AACdFzhE/edit?usp=sharing",
            &[
                "my",
                "share",
                "url",
                "is",
                "https://docs.google.com/spreadsheets/d/19bdnznUXdHEOlp0Pnp9JY0rug6VuA2R3zK4AACdFzhE/edit?usp=sharing",
            ],
        ),
    ];

    for (input, expected) in cases {
        assert_eq!(lex(input), *expected, "input: {input}");
    }
}

#[test]
fn empty_and_whitespace_yield_no_tokens() {
    assert_eq!(lex(""), Vec::<String>::new());
    assert_eq!(lex("   "), Vec::<String>::new());
    assert_eq!(lex("\t\n  "), Vec::<String>::new());
}

#[test]
fn single_word_stays_whole() {
    assert_eq!(lex("Hello"), vec!["Hello"]);
}

#[test]
fn hyphenated_word_stays_whole() {
    // The ids regex needs length 8 or more, so short hyphenated words are not
    // captured. No other regex splits on the hyphen.
    assert_eq!(lex("so-called"), vec!["so-called"]);
}

#[test]
fn eight_char_token_stays_whole() {
    assert_eq!(lex("abcdefgh"), vec!["abcdefgh"]);
}

#[test]
fn email_with_digit_local_part_stays_whole() {
    assert_eq!(lex("me2@exampleabc.com"), vec!["me2@exampleabc.com"]);
}

#[test]
fn url_query_ampersand_stays_whole() {
    assert_eq!(
        lex("https://example.com/search?a=1&b=2"),
        vec!["https://example.com/search?a=1&b=2"]
    );
}

#[test]
fn url_percent_path_stays_whole() {
    assert_eq!(
        lex("https://example.com/a%20b"),
        vec!["https://example.com/a%20b"]
    );
}

#[test]
fn long_id_before_comma_keeps_id_then_comma() {
    assert_eq!(
        lex("19bdnznUXdHEOlp0Pnp9JY0rug6VuA2R3zK4AACdFzhE,"),
        vec!["19bdnznUXdHEOlp0Pnp9JY0rug6VuA2R3zK4AACdFzhE", ","]
    );
}

#[test]
fn email_with_long_local_part_stays_whole() {
    assert_eq!(lex("johnsmith@example.com"), vec!["johnsmith@example.com"]);
}

#[test]
fn long_decimal_stays_whole() {
    assert_eq!(lex("12345678.5"), vec!["12345678.5"]);
}

#[test]
fn id_ending_with_hyphen_before_comma_stays_whole() {
    assert_eq!(lex("abcdefgh-,"), vec!["abcdefgh-", ","]);
}

#[test]
fn long_hyphenated_token_does_not_split_at_prefix() {
    assert_eq!(
        lex("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS-TAIL"),
        vec!["abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS-TAIL"]
    );
}

#[test]
fn dotted_abbreviation_splits_into_letters() {
    assert_eq!(
        lex("The E.M.T.'s were on time"),
        vec!["The", "E", ".", "M", ".", "T", ".", "'", "s", "were", "on", "time"]
    );
}

#[test]
fn whitespace_class_matches_js_on_nel_and_bom() {
    // U+0085 (NEL) is not whitespace under the JS rules, so a NEL between
    // letters keeps the run as one token.
    assert_eq!(lex("alpha\u{0085}beta"), vec!["alpha\u{0085}beta"]);
    // U+FEFF (BOM) is whitespace under the JS rules, so a BOM between letters
    // splits the run into two tokens.
    assert_eq!(lex("alpha\u{FEFF}beta"), vec!["alpha", "beta"]);
}

#[test]
fn mixed_sentence_recursive_split() {
    assert_eq!(
        lex("I made $5.60 today in 1 hour of work.  The E.M.T.'s were on time, but only barely."),
        vec![
            "I", "made", "$", "5.60", "today", "in", "1", "hour", "of", "work", ".", "The", "E",
            ".", "M", ".", "T", ".", "'", "s", "were", "on", "time", ",", "but", "only", "barely",
            ".",
        ]
    );
}