vader_parity/

lib.rs

//! VADER sentiment scoring with exact output parity against Python
//! [vaderSentiment] 3.3.2.
//!
//! VADER (Valence Aware Dictionary and sEntiment Reasoner) is the rule-based
//! sentiment model of Hutto & Gilbert (ICWSM-14). This crate reimplements
//! that package's whole rule set over that package's own data files,
//! vendored byte-identical from the 3.3.2 sdist: lexicon scoring, booster
//! and dampener modifiers with the casing increment, negation with the
//! "least" special case, contrastive "but" reweighting, exclamation and
//! question mark emphasis, the special-case idioms, emoji translation, and
//! the alpha-15 normalization to compound. For every input, all four scores
//! equal what the Python package returns at its published precision: three
//! decimals for the proportions, four for the compound. The frozen oracle
//! vectors in `tests/vectors/frozen.tsv` replay on every `cargo test`, and a
//! release-time differential corpus against the pinned package holds that
//! parity with zero tolerated divergence. Casing is pinned with the rest:
//! `str.isupper()` and `str.lower()` answer from tables vendored out of
//! CPython 3.14.6 with Unicode data 16.0.0, checked across every codepoint,
//! so the scores hold still when the Rust standard library adopts a newer
//! Unicode version than the reference interpreter's.
//!
//! ```
//! use vader_parity::SentimentIntensityAnalyzer;
//!
//! let analyzer = SentimentIntensityAnalyzer::new();
//! let scores = analyzer.polarity_scores("VADER is smart, handsome, and funny.");
//! assert_eq!(scores.neg, 0.0);
//! assert_eq!(scores.neu, 0.254);
//! assert_eq!(scores.pos, 0.746);
//! assert_eq!(scores.compound, 0.8316);
//! ```
//!
//! # Scoring many texts
//!
//! [`SentimentIntensityAnalyzer::new`] parses the vendored lexicons, about
//! 11,000 data lines, once. Build one analyzer and score many texts. Each
//! call is a bounded linear pass, and the example below is the crate's own
//! regression tripwire for that claim.
//!
//! ```
//! use vader_parity::SentimentIntensityAnalyzer;
//!
//! let analyzer = SentimentIntensityAnalyzer::new();
//! let started = std::time::Instant::now();
//! for _ in 0..10_000 {
//!     analyzer.polarity_scores("Today only kinda sux! But I'll get by, lol");
//! }
//! assert!(started.elapsed() < std::time::Duration::from_secs(30));
//! ```
//!
//! # Scope
//!
//! The reference is English-only and so is its lexicon. This crate is the
//! scoring engine only, pinned to reference version 3.3.2, `std`-only, with
//! zero runtime dependencies. Sentence splitting, custom lexicons, and other
//! reference versions are out of scope.
//!
//! # Attribution
//!
//! The VADER algorithm design, the sentiment lexicon, and the emoji lexicon
//! are the MIT-licensed work of C.J. Hutto and E.E. Gilbert. The vendored
//! files in `data/` ship with the authors' license text and their recorded
//! sha256 digests, and the Rust implementation and its tests are this
//! crate's. If you use VADER, cite: Hutto, C.J. & Gilbert, E.E. (2014).
//! VADER: A Parsimonious Rule-based Model for Sentiment Analysis of Social
//! Media Text. Eighth International Conference on Weblogs and Social Media
//! (ICWSM-14). Ann Arbor, MI.
//!
//! [vaderSentiment]: https://pypi.org/project/vaderSentiment/3.3.2/

#![forbid(unsafe_code)]
#![deny(missing_docs)]
#![deny(
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::panic,
    clippy::indexing_slicing
)]

mod casing;
mod data;
mod round;
mod rules;
mod text;

/// The four scores for one text. Field values match Python vaderSentiment
/// 3.3.2, at that package's published precision.
///
/// The proportions describe how the text's tokens split across the three
/// categories and sum to 1 up to rounding residue. The compound is the
/// normalized, weighted composite in [-1, 1], the single number most callers
/// read (its sign is the sentiment label). `tests/properties.rs` holds the
/// distribution invariant, and the frozen vectors pin all four fields.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct SentimentScores {
    /// Negative proportion, rounded to 3 decimals. In [0, 1].
    pub neg: f64,
    /// Neutral proportion, rounded to 3 decimals. In [0, 1].
    pub neu: f64,
    /// Positive proportion, rounded to 3 decimals. In [0, 1].
    pub pos: f64,
    /// Normalized compound score, rounded to 4 decimals. In [-1, 1].
    pub compound: f64,
}

/// The analyzer, holding the loaded lexicons.
///
/// Construction parses the vendored VADER 3.3.2 data once, and scoring
/// reuses the parsed tables. Two analyzers are interchangeable: same input,
/// same bits, which `tests/properties.rs` asserts.
pub struct SentimentIntensityAnalyzer {
    tables: data::Tables,
}

impl SentimentIntensityAnalyzer {
    /// Build an analyzer over the vendored VADER 3.3.2 lexicons.
    ///
    /// Infallible: the data is vendored and its digests are enforced by
    /// `tests/provenance.rs` before any caller exists.
    #[must_use]
    pub fn new() -> SentimentIntensityAnalyzer {
        SentimentIntensityAnalyzer {
            tables: data::Tables::load(),
        }
    }

    /// Score one text. Total: every `&str` input returns scores.
    ///
    /// Text with nothing scoreable is neutral, not an error, exactly as the
    /// reference scores it:
    ///
    /// ```
    /// use vader_parity::SentimentIntensityAnalyzer;
    ///
    /// let analyzer = SentimentIntensityAnalyzer::new();
    /// assert_eq!(analyzer.polarity_scores("is?").compound, 0.0);
    /// assert_eq!(analyzer.polarity_scores("is?").neu, 1.0);
    /// assert_eq!(analyzer.polarity_scores("").compound, 0.0);
    /// ```
    #[must_use]
    pub fn polarity_scores(&self, text: &str) -> SentimentScores {
        rules::polarity_scores(&self.tables, text).0
    }

    /// Score one text and also return the compound before rounding. The
    /// differential harness compares this bit-for-bit against the oracle
    /// (invariant 9). Only test builds carry it.
    #[cfg(test)]
    pub(crate) fn polarity_scores_with_raw(&self, text: &str) -> (SentimentScores, f64) {
        rules::polarity_scores(&self.tables, text)
    }
}

impl Default for SentimentIntensityAnalyzer {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod invariant9;