pos-tagger 0.2.0

//! Tagger conformance tests.
//!
//! Covers lexicon lookup, the lowercase fallback, the `NN` default, lexicon
//! extension, `word_in_lexicon`, and each of the eight rules in isolation. Inputs
//! use out-of-lexicon stems so exactly one rule fires where intended.

use pos_tagger::Tagger;
use std::collections::HashMap;

fn pairs(input: &[(&str, &str)]) -> Vec<(String, String)> {
    input
        .iter()
        .map(|(w, t)| (w.to_string(), t.to_string()))
        .collect()
}

fn tag(words: &[&str]) -> Vec<(String, String)> {
    Tagger::new().tag(words)
}

#[test]
fn extends_lexicon_with_new_word() {
    let mut tagger = Tagger::new();
    let mut entries = HashMap::new();
    entries.insert("Obama".to_string(), vec!["NNP".to_string()]);
    tagger.extend_lexicon(entries);

    assert_eq!(
        tagger.tag(&["Mr", "Obama"]),
        pairs(&[("Mr", "NNP"), ("Obama", "NNP")])
    );
}

#[test]
fn tag_accepts_owned_strings_from_lexer() {
    // tag borrows its input as `str`, so the lexer's Vec<String> passes with no
    // intermediate Vec<&str>.
    use pos_tagger::Lexer;
    let tokens = Lexer::new().lex("the dog");
    let tagged = Tagger::new().tag(&tokens);
    assert_eq!(tagged, pairs(&[("the", "DT"), ("dog", "NN")]));
}

#[test]
fn extend_lexicon_accepts_an_array_of_pairs() {
    // extend_lexicon takes any iterable of pairs, so an array works without a
    // HashMap.
    let mut tagger = Tagger::new();
    tagger.extend_lexicon([("Obama".to_string(), vec!["NNP".to_string()])]);
    assert_eq!(tagger.tag(&["Obama"]), pairs(&[("Obama", "NNP")]));
}

#[test]
fn fresh_tagger_does_not_know_extended_word() {
    // Each tagger owns its lexicon. An extension on one instance does not leak to
    // another. A fresh tagger has never seen "Obama", so it falls back to NN.
    assert_eq!(tag(&["Obama"]), pairs(&[("Obama", "NN")]));
}

#[test]
fn extend_lexicon_never_overrides_existing() {
    let mut tagger = Tagger::new();
    let mut entries = HashMap::new();
    entries.insert("Mr".to_string(), vec!["XX".to_string()]);
    tagger.extend_lexicon(entries);

    // "Mr" was already NNP in the base lexicon and stays NNP.
    assert_eq!(tagger.tag(&["Mr"]), pairs(&[("Mr", "NNP")]));
}

#[test]
fn unknown_word_defaults_to_nn() {
    assert_eq!(tag(&["Zzqxxw"]), pairs(&[("Zzqxxw", "NN")]));
}

#[test]
fn single_unknown_char_defaults_to_nn() {
    // A single unknown character has no lexicon entry and no rule fires on it, so
    // it falls through to the NN default.
    assert_eq!(tag(&["~"]), pairs(&[("~", "NN")]));
    assert_eq!(tag(&["z"]), pairs(&[("z", "NN")]));
}

#[test]
fn at_sign_is_a_lexicon_entry() {
    // "@" is in the lexicon as IN, not the NN default.
    assert_eq!(tag(&["@"]), pairs(&[("@", "IN")]));
}

#[test]
fn object_property_names_are_not_lexicon_hits() {
    // Object member names are not lexicon entries, so they default to NN.
    assert_eq!(tag(&["constructor"]), pairs(&[("constructor", "NN")]));
    assert_eq!(tag(&["hasOwnProperty"]), pairs(&[("hasOwnProperty", "NN")]));
    // "toString" is also absent, so it starts at NN. It ends in "ing" with no
    // earlier "ing", so rule 8 rewrites it to VBG.
    assert_eq!(tag(&["toString"]), pairs(&[("toString", "VBG")]));
}

#[test]
fn word_in_lexicon_is_false_for_object_member_names() {
    // This crate stores the lexicon in a HashMap, which has no prototype chain.
    // Object member names are absent from the data, so word_in_lexicon returns
    // false. A lookup over a plain object would inherit these names and report
    // true. The cleaner HashMap result is the deliberate choice here.
    let tagger = Tagger::new();
    for name in [
        "constructor",
        "valueOf",
        "hasOwnProperty",
        "toString",
        "__proto__",
    ] {
        assert!(
            !tagger.word_in_lexicon(name),
            "{name:?} should not be a lexicon hit"
        );
    }
}

#[test]
fn mixed_case_falls_back_to_lowercase() {
    assert_eq!(tag(&["THE"]), pairs(&[("THE", "DT")]));
    assert_eq!(tag(&["the"]), pairs(&[("the", "DT")]));
}

#[test]
fn empty_input_returns_empty() {
    assert_eq!(tag(&[]), Vec::<(String, String)>::new());
}

#[test]
fn empty_token_tags_as_nn() {
    // The empty string is a lexicon key with an empty tag list. The lexer never
    // emits an empty token, so this only arises from a direct tag call. The
    // empty tag list falls back to NN rather than panicking.
    assert_eq!(tag(&[""]), pairs(&[("", "NN")]));
}

// One test per Brill rule, isolated so a single rule fires.

#[test]
fn rule1_determiner_then_verb_becomes_noun() {
    assert_eq!(tag(&["the", "run"]), pairs(&[("the", "DT"), ("run", "NN")]));
}

#[test]
fn rule2_noun_with_dot_and_letters_becomes_url() {
    assert_eq!(tag(&["example.com"]), pairs(&[("example.com", "URL")]));
    assert_eq!(tag(&["google.com"]), pairs(&[("google.com", "URL")]));
}

#[test]
fn rule2_numeric_becomes_cd() {
    assert_eq!(tag(&["5.42"]), pairs(&[("5.42", "CD")]));
    assert_eq!(tag(&["3.14"]), pairs(&[("3.14", "CD")]));
}

#[test]
fn rule2_parsefloat_prefix_becomes_cd() {
    assert_eq!(tag(&["5abc"]), pairs(&[("5abc", "CD")]));
}

#[test]
fn rule2_dotted_non_numeric_becomes_cd() {
    // "v1.2.3" has a dot but no two contiguous letters, so the first sub-branch
    // sets CD. parseFloat is NaN, so it stays CD.
    assert_eq!(tag(&["v1.2.3"]), pairs(&[("v1.2.3", "CD")]));
}

#[test]
fn rule3_noun_ending_ed_becomes_past_participle() {
    assert_eq!(tag(&["Foobared"]), pairs(&[("Foobared", "VBN")]));
}

#[test]
fn rule4_ly_becomes_adverb() {
    assert_eq!(tag(&["quickly"]), pairs(&[("quickly", "RB")]));
}

#[test]
fn rule5_noun_ending_al_becomes_adjective() {
    assert_eq!(tag(&["Zzqxxwal"]), pairs(&[("Zzqxxwal", "JJ")]));
}

#[test]
fn rule6_noun_after_would_becomes_verb() {
    assert_eq!(
        tag(&["would", "Zzqxxw"]),
        pairs(&[("would", "MD"), ("Zzqxxw", "VB")])
    );
}

#[test]
fn rule7_noun_ending_s_becomes_plural() {
    assert_eq!(tag(&["Zzqxxws"]), pairs(&[("Zzqxxws", "NNS")]));
}

#[test]
fn rule8_noun_ending_ing_becomes_gerund() {
    assert_eq!(tag(&["Zzqxxwing"]), pairs(&[("Zzqxxwing", "VBG")]));
}

#[test]
fn buggy_ends_with_keeps_boss_and_class_singular() {
    // The first-occurrence suffix check means words with an earlier "s" are not
    // pluralized.
    assert_eq!(tag(&["boss"]), pairs(&[("boss", "NN")]));
    assert_eq!(tag(&["class"]), pairs(&[("class", "NN")]));
    // A word with a single trailing "s" still pluralizes.
    assert_eq!(tag(&["dogs"]), pairs(&[("dogs", "NNS")]));
}

#[test]
fn first_occurrence_ends_with_across_all_suffixes() {
    // The suffix check uses the first match position, not the last. A word that
    // repeats its suffix earlier returns false, so the rule does not fire and the
    // word stays NN. Each input below is absent from the lexicon, so it starts at
    // NN and probes one rule. The s-suffix case is pinned separately above.
    let non_firing: &[(&str, &str)] = &[
        ("Zzedzzed", "NN"),  // rule 3, "ed" repeats
        ("Zzlyzzly", "NN"),  // rule 4, "ly" repeats
        ("lyrically", "NN"), // rule 4, "ly" appears early
        ("Zzalzzal", "NN"),  // rule 5, "al" repeats
        ("Zzinging", "NN"),  // rule 8, "ing" repeats
    ];
    for (word, expected) in non_firing {
        assert_eq!(tag(&[*word]), pairs(&[(word, expected)]));
    }

    // When the suffix sits only at the end, the rule fires.
    let firing: &[(&str, &str)] = &[
        ("Zzaling", "VBG"), // rule 8 fires
        ("Zzlying", "VBG"), // rule 8 fires; rule 4 skipped, "ly" is not at the end
    ];
    for (word, expected) in firing {
        assert_eq!(tag(&[*word]), pairs(&[(word, expected)]));
    }
}

#[test]
fn word_in_lexicon() {
    let tagger = Tagger::new();
    assert!(tagger.word_in_lexicon("Mr"));
    assert!(tagger.word_in_lexicon("THE")); // lowercase fallback
    assert!(tagger.word_in_lexicon("")); // empty key exists
    assert!(!tagger.word_in_lexicon("Zzqxxw"));
}