vader-parity 0.1.0

//! The vendored VADER 3.3.2 data tables and their loaders.
//!
//! The two lexicon files are byte-identical copies from the vaderSentiment
//! 3.3.2 sdist, and `rules_tables.tsv` transcribes the idiom and booster
//! tables that the reference embeds in its source file. `data/PROVENANCE`
//! records the sha256 digests; `tests/provenance.rs` enforces them. The
//! loaders reproduce the reference's `make_lex_dict` and `make_emoji_dict`
//! semantics over the reference's bytes, which is the definition of the
//! tables: no edits, no re-sorting, no de-duplication.

use crate::text::py_strip;
use std::collections::HashMap;

/// `data/vader_lexicon.txt`, byte-identical to the 3.3.2 sdist.
pub(crate) const VADER_LEXICON: &str = include_str!("../data/vader_lexicon.txt");
/// `data/emoji_utf8_lexicon.txt`, byte-identical to the 3.3.2 sdist.
pub(crate) const EMOJI_LEXICON: &str = include_str!("../data/emoji_utf8_lexicon.txt");
/// `data/rules_tables.tsv`, the transcribed `SPECIAL_CASES` and `BOOSTER_DICT`.
pub(crate) const RULES_TABLES: &str = include_str!("../data/rules_tables.tsv");

/// One `SPECIAL_CASES` entry. Python's table mixes int values ("the shit": 3)
/// with float values ("bad ass": 1.5), and CPython's builtin `sum` adds int
/// list items outside its float compensation step, so whether a replacement
/// valence was an int is observable in the pre-rounding compound and must
/// ride along with the value.
#[derive(Clone, Copy)]
pub(crate) struct SpecialCase {
    /// The replacement valence.
    pub(crate) value: f64,
    /// True when the Python literal is an int.
    pub(crate) is_int: bool,
}

/// The loaded tables one analyzer holds.
pub(crate) struct Tables {
    /// Token to valence, from `vader_lexicon.txt`.
    pub(crate) lexicon: HashMap<String, f64>,
    /// Single code point to description, from `emoji_utf8_lexicon.txt`.
    pub(crate) emojis: HashMap<char, String>,
    /// Exact idiom sequence to replacement valence (`SPECIAL_CASES`).
    pub(crate) special_cases: HashMap<String, SpecialCase>,
    /// Booster or dampener key to increment (`BOOSTER_DICT`), including the
    /// multi-token keys "just enough", "kind of", and "sort of".
    pub(crate) boosters: HashMap<String, f64>,
}

impl Tables {
    /// Parse all vendored tables. Infallible over the vendored bytes, which
    /// the provenance test pins; the count tests below pin the parsed sizes.
    pub(crate) fn load() -> Tables {
        let (special_cases, boosters) = parse_rules_tables(RULES_TABLES);
        Tables {
            lexicon: parse_lexicon(VADER_LEXICON),
            emojis: parse_emojis(EMOJI_LEXICON),
            special_cases,
            boosters,
        }
    }
}

/// The reference's `make_lex_dict`:
///
/// ```python
/// for line in self.lexicon_full_filepath.rstrip('\n').split('\n'):
///     if not line: continue
///     (word, measure) = line.strip().split('\t')[0:2]
///     lex_dict[word] = float(measure)
/// ```
///
/// Later duplicate keys overwrite earlier ones, dict-style. A line without a
/// tab or with an unparseable measure would raise in Python; the vendored
/// file has none, and here such a line is skipped so the loader stays total.
fn parse_lexicon(raw: &str) -> HashMap<String, f64> {
    let mut map = HashMap::new();
    for line in raw.trim_end_matches('\n').split('\n') {
        if line.is_empty() {
            continue;
        }
        let mut fields = py_strip(line).split('\t');
        if let (Some(word), Some(measure)) = (fields.next(), fields.next()) {
            if let Ok(value) = measure.parse::<f64>() {
                map.insert(word.to_string(), value);
            }
        }
    }
    map
}

/// The reference's `make_emoji_dict`, same line shape without the empty-line
/// skip. The reference looks emoji up one character at a time, so a key of
/// two or more code points can never match; only single code point keys are
/// kept, later duplicates overwriting earlier ones.
fn parse_emojis(raw: &str) -> HashMap<char, String> {
    let mut map = HashMap::new();
    for line in raw.trim_end_matches('\n').split('\n') {
        let mut fields = py_strip(line).split('\t');
        if let (Some(key), Some(description)) = (fields.next(), fields.next()) {
            let mut chars = key.chars();
            if let (Some(c), None) = (chars.next(), chars.next()) {
                map.insert(c, description.to_string());
            }
        }
    }
    map
}

/// The transcribed `SPECIAL_CASES` and `BOOSTER_DICT` tables: three
/// tab-separated columns, `kind` then `key` then `value`, with `#` comment
/// lines carrying the provenance header. The value column holds Python
/// literal reprs, so a value with no decimal point is a Python int, which
/// [`SpecialCase::is_int`] records.
fn parse_rules_tables(raw: &str) -> (HashMap<String, SpecialCase>, HashMap<String, f64>) {
    let mut special_cases = HashMap::new();
    let mut boosters = HashMap::new();
    for line in raw.lines() {
        if line.is_empty() || line.as_bytes().first() == Some(&b'#') {
            continue;
        }
        let mut fields = line.split('\t');
        if let (Some(kind), Some(key), Some(literal)) =
            (fields.next(), fields.next(), fields.next())
        {
            if let Ok(value) = literal.parse::<f64>() {
                match kind {
                    "special" => {
                        let is_int = !literal.bytes().any(|b| b == b'.');
                        special_cases.insert(key.to_string(), SpecialCase { value, is_int });
                    }
                    "booster" => {
                        boosters.insert(key.to_string(), value);
                    }
                    _ => {}
                }
            }
        }
    }
    (special_cases, boosters)
}

#[cfg(test)]
mod tests {
    use super::{parse_emojis, parse_lexicon, parse_rules_tables, Tables};

    #[test]
    fn table_sizes_match_the_reference() {
        // Counts observed on the loaded Python 3.3.2 reference: 7506 lexicon
        // entries (7520 lines, 14 later-wins duplicates), 1212 single code
        // point emoji keys of the dict's 3570, 9 special cases, 84 boosters.
        let tables = Tables::load();
        assert_eq!(tables.lexicon.len(), 7506);
        assert_eq!(tables.emojis.len(), 1212);
        assert_eq!(tables.special_cases.len(), 9);
        assert_eq!(tables.boosters.len(), 84);
    }

    #[test]
    fn known_entries_parse_to_reference_values() {
        let tables = Tables::load();
        assert_eq!(tables.lexicon.get("good").copied(), Some(1.9));
        assert_eq!(tables.lexicon.get("no").copied(), Some(-1.2));
        assert_eq!(tables.lexicon.get(":)").copied(), Some(2.0));
        let the_shit = tables.special_cases.get("the shit").copied();
        assert_eq!(the_shit.map(|c| c.value), Some(3.0));
        // Python spells it as the int literal 3.
        assert_eq!(the_shit.map(|c| c.is_int), Some(true));
        let beating_heart = tables.special_cases.get("beating heart").copied();
        assert_eq!(beating_heart.map(|c| c.value), Some(3.5));
        assert_eq!(beating_heart.map(|c| c.is_int), Some(false));
        let bus_stop = tables.special_cases.get("bus stop").copied();
        // "bus stop": 0.0 is a float literal in Python, unlike the int 3s.
        assert_eq!(bus_stop.map(|c| c.value), Some(0.0));
        assert_eq!(bus_stop.map(|c| c.is_int), Some(false));
        assert_eq!(tables.boosters.get("so").copied(), Some(0.293));
        assert_eq!(tables.boosters.get("sort of").copied(), Some(-0.293));
        assert_eq!(
            tables.emojis.get(&'\u{1F496}').map(String::as_str),
            Some("sparkling heart")
        );
    }

    #[test]
    fn lexicon_loader_skips_what_python_would_reject() {
        let map = parse_lexicon("good\t1.9\n\nbad\tnot-a-number\nlonely-field\nok\t0.5\n");
        assert_eq!(map.len(), 2);
        assert_eq!(map.get("good").copied(), Some(1.9));
        assert_eq!(map.get("ok").copied(), Some(0.5));
    }

    #[test]
    fn lexicon_loader_lets_later_duplicates_win() {
        let map = parse_lexicon("word\t1.0\nword\t2.0\n");
        assert_eq!(map.get("word").copied(), Some(2.0));
    }

    #[test]
    fn emoji_loader_keeps_single_code_point_keys_only() {
        let map = parse_emojis("\u{1F600}\tgrinning face\n\u{1F468}\u{200D}\u{1F469}\tfamily\nx\tearly\nx\tlate\nbroken-line\n");
        assert_eq!(map.len(), 2);
        assert_eq!(map.get(&'x').map(String::as_str), Some("late"));
        assert_eq!(
            map.get(&'\u{1F600}').map(String::as_str),
            Some("grinning face")
        );
    }

    #[test]
    fn rules_tables_loader_splits_kinds_and_skips_junk() {
        let (special, boost) = parse_rules_tables(
            "# header\nspecial\tthe shit\t3\nspecial\tbad ass\t1.5\nbooster\tso\t0.293\nmystery\tx\t1\nbooster\tbad\tNaN?no\nshort-line\n\n",
        );
        assert_eq!(special.len(), 2);
        assert_eq!(boost.len(), 1);
        assert_eq!(
            special.get("the shit").map(|c| (c.value, c.is_int)),
            Some((3.0, true))
        );
        assert_eq!(
            special.get("bad ass").map(|c| (c.value, c.is_int)),
            Some((1.5, false))
        );
        assert_eq!(boost.get("so").copied(), Some(0.293));
    }
}