vader-parity 0.1.0

//! Input preparation: emoji translation, tokenization, and case analysis,
//! matching the reference's `SentiText` and the emoji walk in
//! `polarity_scores` (vaderSentiment 3.3.2, `vaderSentiment.py`). The two
//! casing calls, `str.lower()` and `str.isupper()`, go through
//! [`crate::casing`], which answers from tables recorded from the pinned
//! CPython. The Rust standard library's casing data tracks a different
//! Unicode version, and a CI grep keeps it out of this path.

use crate::casing::{py_isupper, py_lower};
use std::collections::HashMap;

/// One token of the reference's words-and-emoticons list, carrying the two
/// derived forms the scoring pipeline reads: the Python `str.lower()` form and
/// the Python `str.isupper()` fact.
pub(crate) struct Token {
    /// `token.lower()`.
    pub(crate) lower: String,
    /// `token.isupper()`.
    pub(crate) is_upper: bool,
}

/// Python's `str.split()` / `str.strip()` whitespace class: every character
/// where `str.isspace()` is true. That is the Unicode `White_Space` set plus
/// the four ASCII separator controls U+001C through U+001F, which carry
/// bidirectional class B or S and count as space in CPython but not in the
/// Unicode `White_Space` property Rust's `char::is_whitespace` implements.
pub(crate) fn is_py_space(c: char) -> bool {
    c.is_whitespace() || matches!(c, '\u{1c}'..='\u{1f}')
}

/// Python `str.strip()` with no argument: trim `is_py_space` from both ends.
pub(crate) fn py_strip(s: &str) -> &str {
    s.trim_matches(is_py_space)
}

/// The emoji walk at the top of the reference's `polarity_scores`: each input
/// character present in the emoji lexicon is replaced by its description, with
/// one space inserted when the previously emitted character was not a space.
///
/// ```python
/// text_no_emoji = ""
/// prev_space = True
/// for chr in text:
///     if chr in self.emojis:
///         description = self.emojis[chr]
///         if not prev_space:
///             text_no_emoji += ' '
///         text_no_emoji += description
///         prev_space = False
///     else:
///         text_no_emoji += chr
///         prev_space = chr == ' '
/// text = text_no_emoji.strip()
/// ```
///
/// `prev_space` tracks the ASCII space character only, exactly as above, and
/// the final strip is Python's whitespace strip.
pub(crate) fn translate_emojis(text: &str, emojis: &HashMap<char, String>) -> String {
    let mut out = String::with_capacity(text.len());
    let mut prev_space = true;
    for c in text.chars() {
        if let Some(description) = emojis.get(&c) {
            if !prev_space {
                out.push(' ');
            }
            out.push_str(description);
            prev_space = false;
        } else {
            out.push(c);
            prev_space = c == ' ';
        }
    }
    py_strip(&out).to_string()
}

/// The reference's `_strip_punc_if_word`: strip leading and trailing ASCII
/// punctuation (`string.punctuation`, exactly Rust's `is_ascii_punctuation`
/// set), and keep the stripped form only when it is longer than two
/// characters, so ":)" and "8-)" survive as themselves.
fn strip_punc_if_word(token: &str) -> &str {
    let stripped = token.trim_matches(|c: char| c.is_ascii_punctuation());
    if stripped.chars().count() <= 2 {
        token
    } else {
        stripped
    }
}

/// The reference's words-and-emoticons list: whitespace split, then the
/// punctuation strip, with the two derived forms computed once per token.
pub(crate) fn words_and_emoticons(text: &str) -> Vec<Token> {
    text.split(is_py_space)
        .filter(|w| !w.is_empty())
        .map(|w| {
            let stripped = strip_punc_if_word(w);
            Token {
                lower: py_lower(stripped),
                is_upper: py_isupper(stripped),
            }
        })
        .collect()
}

/// The reference's `allcap_differential`: true when some but not all tokens
/// are all-caps.
///
/// ```python
/// cap_differential = len(words) - allcap_words
/// if 0 < cap_differential < len(words): is_different = True
/// ```
///
/// `0 < len - upper < len` holds exactly when at least one token is not
/// all-caps and at least one is, which is how it is spelled here.
pub(crate) fn allcap_differential(tokens: &[Token]) -> bool {
    tokens.iter().any(|t| !t.is_upper) && tokens.iter().any(|t| t.is_upper)
}

#[cfg(test)]
#[allow(clippy::indexing_slicing)]
mod tests {
    use super::{
        allcap_differential, is_py_space, py_strip, strip_punc_if_word, translate_emojis,
        words_and_emoticons,
    };
    use std::collections::HashMap;

    #[test]
    fn python_space_class_covers_separator_controls() {
        assert!(is_py_space(' '));
        assert!(is_py_space('\t'));
        assert!(is_py_space('\n'));
        assert!(is_py_space('\u{1c}'));
        assert!(is_py_space('\u{1f}'));
        assert!(!is_py_space('a'));
        assert!(!is_py_space('!'));
    }

    #[test]
    fn strip_trims_both_ends_only() {
        assert_eq!(py_strip("  a b  "), "a b");
        assert_eq!(py_strip("\t\nx\u{1c}"), "x");
        assert_eq!(py_strip(""), "");
    }

    #[test]
    fn punctuation_strip_keeps_short_forms() {
        assert_eq!(strip_punc_if_word("happy,"), "happy");
        assert_eq!(strip_punc_if_word("--good--"), "good");
        assert_eq!(strip_punc_if_word(":)"), ":)");
        assert_eq!(strip_punc_if_word("8-)"), "8-)");
        // "is?" strips to "is", two characters, so the original survives.
        assert_eq!(strip_punc_if_word("is?"), "is?");
        assert_eq!(strip_punc_if_word("so!"), "so!");
        assert_eq!(strip_punc_if_word("don't"), "don't");
        assert_eq!(strip_punc_if_word("!!!"), "!!!");
    }

    #[test]
    fn tokenizer_splits_on_whitespace_runs() {
        let toks = words_and_emoticons("  GOOD \t bad,  ");
        let lowers: Vec<&str> = toks.iter().map(|t| t.lower.as_str()).collect();
        assert_eq!(lowers, ["good", "bad"]);
        assert!(toks[0].is_upper);
        assert!(!toks[1].is_upper);
        assert!(words_and_emoticons("").is_empty());
        assert!(words_and_emoticons("   ").is_empty());
    }

    #[test]
    fn emoji_walk_inserts_spaces_like_the_reference() {
        let mut emojis = HashMap::new();
        emojis.insert('\u{1F498}', String::from("heart with arrow"));
        emojis.insert('\u{1F48B}', String::from("kiss mark"));
        // After a non-space character: one space inserted.
        assert_eq!(
            translate_emojis("x\u{1F498}", &emojis),
            "x heart with arrow"
        );
        // After a space: no extra space.
        assert_eq!(
            translate_emojis("x \u{1F498}", &emojis),
            "x heart with arrow"
        );
        // Adjacent emoji: prev_space is false after a description, so a space
        // separates them even though none was typed.
        assert_eq!(
            translate_emojis("\u{1F498}\u{1F48B}", &emojis),
            "heart with arrow kiss mark"
        );
        // Text after an emoji gets no inserted space: the walk only inserts
        // before descriptions.
        assert_eq!(translate_emojis("\u{1F498}y", &emojis), "heart with arrowy");
        // Leading and trailing whitespace stripped at the end.
        assert_eq!(translate_emojis("  plain  ", &emojis), "plain");
    }

    #[test]
    fn cap_differential_needs_a_mix() {
        let mixed = words_and_emoticons("GOOD bad");
        assert!(allcap_differential(&mixed));
        let all_caps = words_and_emoticons("GOOD BAD");
        assert!(!allcap_differential(&all_caps));
        let no_caps = words_and_emoticons("good bad");
        assert!(!allcap_differential(&no_caps));
        assert!(!allcap_differential(&[]));
    }
}