pos-tagger 0.2.0

//! The eight contextual rewrite rules applied after initial tag assignment.
//!
//! Each rule inspects one position in the tagged sentence and may overwrite that
//! position's tag in place. Rules run in fixed order 1 through 8 for each word,
//! left to right across the sentence. A rule may read the already-finalized tag
//! of the position to its left.
//!
//! The rule set derives from Brill transformation rules by Hugo W.L. ter Doest
//! (Copyright (C) 2015), released under the GNU GPL version 3 or later. This
//! crate carries that license. See the LICENSE and NOTICE files.

/// One tagged token: the original word and its current tag.
pub(crate) type Tagged = (String, String);

/// Apply all eight rules in order to the token at `index`.
///
/// The slice is shared and mutated in place so later rules and later positions
/// observe earlier changes.
pub(crate) fn apply_all(sentence: &mut [Tagged], index: usize) {
    rule1(sentence, index);
    rule2(sentence, index);
    rule3(sentence, index);
    rule4(sentence, index);
    rule5(sentence, index);
    rule6(sentence, index);
    rule7(sentence, index);
    rule8(sentence, index);
}

/// True if `s` begins with `prefix`. Empty `prefix` is never a match.
fn starts_with(s: &str, prefix: &str) -> bool {
    if prefix.is_empty() {
        return false;
    }
    s.find(prefix) == Some(0)
}

/// True if `s` ends with `suffix`, using first-occurrence search.
///
/// This deliberately uses the first match position, not the last. A word that
/// contains the suffix earlier returns false even when it does end with it. For
/// example `ends_with("boss", "s")` is false because the first `s` sits at index
/// 2 while the tail starts at index 3. This quirk shapes which words rules 3
/// through 8 rewrite, so it stays.
///
/// Lengths and indices are counted in UTF-16 code units so suffix tests match
/// JavaScript string indexing exactly, which the lexicon and rules assume.
fn ends_with(s: &str, suffix: &str) -> bool {
    if suffix.is_empty() {
        return false;
    }
    let s_len = utf16_len(s);
    let suffix_len = utf16_len(suffix);
    if suffix_len > s_len {
        return false;
    }
    match utf16_find(s, suffix) {
        Some(idx) => idx == s_len - suffix_len,
        None => false,
    }
}

/// Length of `s` in UTF-16 code units.
fn utf16_len(s: &str) -> usize {
    s.chars().map(char::len_utf16).sum()
}

/// First index of `needle` in `haystack`, counted in UTF-16 code units.
///
/// Returns `None` when `needle` does not occur. Empty `needle` returns `Some(0)`,
/// matching JavaScript `indexOf`, though callers guard against the empty case.
fn utf16_find(haystack: &str, needle: &str) -> Option<usize> {
    let byte_idx = haystack.find(needle)?;
    Some(utf16_len(&haystack[..byte_idx]))
}

/// Rule 1: a determiner followed by a finite or base verb becomes a noun.
///
/// When the previous tag is exactly `DT` and the current tag is `VBD`, `VBP`, or
/// `VB`, set the current tag to `NN`.
fn rule1(sentence: &mut [Tagged], index: usize) {
    if index > 0 && sentence[index - 1].1 == "DT" {
        let tag = sentence[index].1.as_str();
        if tag == "VBD" || tag == "VBP" || tag == "VB" {
            sentence[index].1 = "NN".to_string();
        }
    }
}

/// Rule 2: a noun containing a dot becomes a URL or a number.
///
/// Fires only when the current tag starts with `N`. If the word contains a dot
/// and has two contiguous ASCII letters, the tag becomes `URL`, otherwise `CD`.
/// Then, independently, if the word parses as a leading number it becomes `CD`,
/// which can override the first decision.
fn rule2(sentence: &mut [Tagged], index: usize) {
    if !starts_with(&sentence[index].1, "N") {
        return;
    }
    let (word, tag) = &mut sentence[index];
    if word.contains('.') {
        if has_two_contiguous_letters(word) {
            *tag = "URL".to_string();
        } else {
            *tag = "CD".to_string();
        }
    }
    if parses_as_float(word) {
        *tag = "CD".to_string();
    }
}

/// True if `word` contains two consecutive ASCII letters, matching `/[a-zA-Z]{2}/`.
fn has_two_contiguous_letters(word: &str) -> bool {
    let bytes = word.as_bytes();
    bytes
        .windows(2)
        .any(|w| w[0].is_ascii_alphabetic() && w[1].is_ascii_alphabetic())
}

/// Rule 3: a noun ending in "ed" becomes a past participle.
fn rule3(sentence: &mut [Tagged], index: usize) {
    if starts_with(&sentence[index].1, "N") && ends_with(&sentence[index].0, "ed") {
        sentence[index].1 = "VBN".to_string();
    }
}

/// Rule 4: any word ending in "ly" becomes an adverb, regardless of current tag.
fn rule4(sentence: &mut [Tagged], index: usize) {
    if ends_with(&sentence[index].0, "ly") {
        sentence[index].1 = "RB".to_string();
    }
}

/// Rule 5: a common noun ending in "al" becomes an adjective.
fn rule5(sentence: &mut [Tagged], index: usize) {
    if starts_with(&sentence[index].1, "NN") && ends_with(&sentence[index].0, "al") {
        sentence[index].1 = "JJ".to_string();
    }
}

/// Rule 6: a common noun after the word "would" becomes a base verb.
fn rule6(sentence: &mut [Tagged], index: usize) {
    if index > 0
        && starts_with(&sentence[index].1, "NN")
        && sentence[index - 1].0.eq_ignore_ascii_case("would")
    {
        sentence[index].1 = "VB".to_string();
    }
}

/// Rule 7: a common noun ending in "s" becomes a plural common noun.
///
/// The current tag must be exactly `NN`. The first-occurrence `ends_with` means
/// words with an earlier `s`, such as "boss" and "class", are not pluralized.
fn rule7(sentence: &mut [Tagged], index: usize) {
    if sentence[index].1 == "NN" && ends_with(&sentence[index].0, "s") {
        sentence[index].1 = "NNS".to_string();
    }
}

/// Rule 8: a common noun ending in "ing" becomes a gerund.
fn rule8(sentence: &mut [Tagged], index: usize) {
    if starts_with(&sentence[index].1, "NN") && ends_with(&sentence[index].0, "ing") {
        sentence[index].1 = "VBG".to_string();
    }
}

/// True if `s` begins with a numeric token that JavaScript `parseFloat` accepts.
///
/// This mirrors `!isNaN(parseFloat(s))`. It skips leading whitespace, accepts an
/// optional sign, then either the literal `Infinity` or a decimal mantissa with
/// an optional exponent. Trailing characters are ignored. The function only
/// reports whether a value would be produced, not the value itself.
fn parses_as_float(s: &str) -> bool {
    let bytes = s.as_bytes();
    let mut i = 0;

    // Skip leading whitespace. parseFloat strips the same set as String.trim,
    // but for the numeric prefix only ASCII whitespace and a few Unicode spaces
    // matter. ASCII covers all realistic tagger input.
    while i < bytes.len() && matches!(bytes[i], b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c) {
        i += 1;
    }

    // Optional sign.
    if i < bytes.len() && (bytes[i] == b'+' || bytes[i] == b'-') {
        i += 1;
    }

    // Infinity is a valid parseFloat result.
    if s[i..].starts_with("Infinity") {
        return true;
    }

    let mut saw_digit = false;
    while i < bytes.len() && bytes[i].is_ascii_digit() {
        i += 1;
        saw_digit = true;
    }
    if i < bytes.len() && bytes[i] == b'.' {
        i += 1;
        while i < bytes.len() && bytes[i].is_ascii_digit() {
            i += 1;
            saw_digit = true;
        }
    }

    // Without at least one mantissa digit there is no number.
    if !saw_digit {
        return false;
    }

    // An exponent is optional and does not change whether a number exists,
    // because a lone trailing 'e' is ignored by parseFloat. No further work
    // needed for the NaN test.
    true
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ends_with_uses_first_occurrence() {
        assert!(!ends_with("boss", "s"));
        assert!(!ends_with("class", "s"));
        assert!(ends_with("dogs", "s"));
        assert!(ends_with("passed", "ed"));
        assert!(!ends_with("word", ""));
    }

    #[test]
    fn starts_with_basic() {
        assert!(starts_with("NNP", "N"));
        assert!(starts_with("NNS", "NN"));
        assert!(!starts_with("VBD", "N"));
        assert!(!starts_with("anything", ""));
    }

    #[test]
    fn parse_float_matches_js() {
        assert!(parses_as_float("5.2"));
        assert!(parses_as_float("5.2abc"));
        assert!(parses_as_float("5abc"));
        assert!(parses_as_float("3.14"));
        assert!(parses_as_float(".5"));
        assert!(parses_as_float("5."));
        assert!(parses_as_float("-5.2"));
        assert!(parses_as_float("+5.2"));
        assert!(parses_as_float("1e3"));
        assert!(parses_as_float("  5.2"));
        assert!(parses_as_float("Infinity"));
        assert!(parses_as_float("-Infinity"));
        assert!(parses_as_float("0x10"));
        assert!(!parses_as_float("v1.2"));
        assert!(!parses_as_float("v1.2.3"));
        assert!(!parses_as_float("."));
        assert!(!parses_as_float(".e5"));
    }

    #[test]
    fn two_contiguous_letters() {
        assert!(has_two_contiguous_letters("google.com"));
        assert!(!has_two_contiguous_letters("v1.2.3"));
        assert!(!has_two_contiguous_letters("5.42"));
    }
}