vader-parity 0.1.0

//! The VADER scoring rules, transcribed from vaderSentiment 3.3.2's
//! `vaderSentiment.py`. Every compound condition carries the Python it
//! transcribes, fully parenthesized (M4). Conditions over the token window
//! bind the named locals `prev1`, `prev2`, and `prev3` first (M1). Every
//! idiom and booster n-gram lookup routes through [`joined_lower`] and a map
//! equality (M2). The punctuation emphasis is a three-arm match with no
//! default arm (M3).
//!
//! Token-window arithmetic lives in this module and nowhere else, all of it
//! through checked operations, which is what the module-wide
//! `arithmetic_side_effects` deny enforces; float arithmetic cannot panic and
//! is exempt from that lint.
#![deny(clippy::arithmetic_side_effects)]

use crate::data::Tables;
use crate::round::python_round;
use crate::text::{allcap_differential, translate_emojis, words_and_emoticons, Token};
use crate::SentimentScores;
use std::cmp::Ordering;
use std::collections::{BTreeSet, HashMap};

/// Booster increment. Python: `B_INCR = 0.293`. The scoring code reads the
/// increments from the vendored booster table; this constant exists so the
/// constant-pin test can assert the table agrees with it.
#[allow(dead_code)]
pub(crate) const B_INCR: f64 = 0.293;
/// Dampener decrement. Python: `B_DECR = -0.293`. Pinned like `B_INCR`.
#[allow(dead_code)]
pub(crate) const B_DECR: f64 = -0.293;
/// All-caps emphasis increment. Python: `C_INCR = 0.733`.
pub(crate) const C_INCR: f64 = 0.733;
/// Negation scalar. Python: `N_SCALAR = -0.74`.
pub(crate) const N_SCALAR: f64 = -0.74;
/// Per-"!" emphasis step, at most four counted.
pub(crate) const EP_STEP: f64 = 0.292;
/// Per-"?" emphasis step when the count is 2 or 3.
pub(crate) const QM_STEP: f64 = 0.18;
/// "?" emphasis cap when the count exceeds 3.
pub(crate) const QM_CAP: f64 = 0.96;
/// Normalization denominator term. Python: `normalize(score, alpha=15)`.
pub(crate) const ALPHA: f64 = 15.0;
/// Valence weight before the first "but".
pub(crate) const BUT_BEFORE: f64 = 0.5;
/// Valence weight after the first "but".
pub(crate) const BUT_AFTER: f64 = 1.5;
/// Booster damping two tokens back.
pub(crate) const DAMP_2: f64 = 0.95;
/// Booster damping three tokens back.
pub(crate) const DAMP_3: f64 = 0.9;
/// The "never so" / "never this" intensifier.
pub(crate) const NEVER_SCALAR: f64 = 1.25;

/// The reference's `NEGATE` list, in its order.
pub(crate) const NEGATE: [&str; 59] = [
    "aint",
    "arent",
    "cannot",
    "cant",
    "couldnt",
    "darent",
    "didnt",
    "doesnt",
    "ain't",
    "aren't",
    "can't",
    "couldn't",
    "daren't",
    "didn't",
    "doesn't",
    "dont",
    "hadnt",
    "hasnt",
    "havent",
    "isnt",
    "mightnt",
    "mustnt",
    "neither",
    "don't",
    "hadn't",
    "hasn't",
    "haven't",
    "isn't",
    "mightn't",
    "mustn't",
    "neednt",
    "needn't",
    "never",
    "none",
    "nope",
    "nor",
    "not",
    "nothing",
    "nowhere",
    "oughtnt",
    "shant",
    "shouldnt",
    "uhuh",
    "wasnt",
    "werent",
    "oughtn't",
    "shan't",
    "shouldn't",
    "uh-uh",
    "wasn't",
    "weren't",
    "without",
    "wont",
    "wouldnt",
    "won't",
    "wouldn't",
    "rarely",
    "seldom",
    "despite",
];

/// One entry of the sentiments list, carrying the valence and whether Python
/// would hold it as an int at this point. The reference's list mixes Python
/// ints (the literal `0` appended for skipped and non-lexicon tokens, and an
/// idiom replacement whose table value is an int literal that no later
/// operation touched) with floats (everything a float operation produced),
/// and CPython's builtin `sum` treats the two differently, observably in the
/// pre-rounding compound. Every arithmetic operation clears the flag; only
/// the two Python int sources set it.
#[derive(Clone, Copy, Debug, PartialEq)]
pub(crate) struct Valence {
    /// The numeric value.
    pub(crate) value: f64,
    /// True when Python's value at this point is an int.
    pub(crate) from_int: bool,
}

impl Valence {
    /// Python's literal int `0`.
    const INT_ZERO: Valence = Valence {
        value: 0.0,
        from_int: true,
    };
}

/// Score one text: the reference's `polarity_scores`. Returns the rounded
/// scores plus the compound before rounding, which the differential harness
/// and the invariant-9 test compare bit-for-bit against the oracle.
pub(crate) fn polarity_scores(tables: &Tables, text: &str) -> (SentimentScores, f64) {
    let translated = translate_emojis(text, &tables.emojis);
    let tokens = words_and_emoticons(&translated);
    let is_cap_diff = allcap_differential(&tokens);

    let mut sentiments: Vec<Valence> = Vec::with_capacity(tokens.len());
    for (i, item) in tokens.iter().enumerate() {
        // Python: if item.lower() in BOOSTER_DICT:
        //             sentiments.append(valence); continue    # int 0
        if tables.boosters.contains_key(item.lower.as_str()) {
            sentiments.push(Valence::INT_ZERO);
            continue;
        }
        // Python: if (i < len(words_and_emoticons) - 1 and item.lower() == "kind"
        //             and words_and_emoticons[i + 1].lower() == "of"):
        //             sentiments.append(valence); continue    # int 0
        if (item.lower == "kind")
            && i.checked_add(1)
                .and_then(|j| tokens.get(j))
                .is_some_and(|next| next.lower == "of")
        {
            sentiments.push(Valence::INT_ZERO);
            continue;
        }
        sentiment_valence(tables, is_cap_diff, &tokens, i, item, &mut sentiments);
    }

    but_check(&tokens, &mut sentiments);
    score_valence(&sentiments, &translated)
}

/// The reference's `sentiment_valence`: one valence for the token at `i`,
/// pushed onto `sentiments`.
fn sentiment_valence(
    tables: &Tables,
    is_cap_diff: bool,
    tokens: &[Token],
    i: usize,
    item: &Token,
    sentiments: &mut Vec<Valence>,
) {
    // Python: valence = 0 — the int, which is what a non-lexicon token
    // appends. A lexicon hit reassigns it to the lexicon float.
    let mut valence = Valence::INT_ZERO;
    if let Some(&lex_valence) = tables.lexicon.get(item.lower.as_str()) {
        let mut valence_value = lex_valence;

        // The token window, one named local per position (M1): prev1 is one
        // token back, prev2 two, prev3 three, next1 one ahead.
        let prev1 = i.checked_sub(1).and_then(|j| tokens.get(j));
        let prev2 = i.checked_sub(2).and_then(|j| tokens.get(j));
        let prev3 = i.checked_sub(3).and_then(|j| tokens.get(j));
        let next1 = i.checked_add(1).and_then(|j| tokens.get(j));

        // Python: if item_lowercase == "no" and i != len(words_and_emoticons) - 1
        //            and words_and_emoticons[i + 1].lower() in self.lexicon:
        //             valence = 0.0
        if (item.lower == "no")
            && next1.is_some_and(|t| tables.lexicon.contains_key(t.lower.as_str()))
        {
            valence_value = 0.0;
        }
        // Python: if (i > 0 and words_and_emoticons[i - 1].lower() == "no") \
        //            or (i > 1 and words_and_emoticons[i - 2].lower() == "no") \
        //            or (i > 2 and words_and_emoticons[i - 3].lower() == "no"
        //                and words_and_emoticons[i - 1].lower() in ["or", "nor"]):
        //             valence = self.lexicon[item_lowercase] * N_SCALAR
        if (prev1.is_some_and(|t| t.lower == "no"))
            || (prev2.is_some_and(|t| t.lower == "no"))
            || ((prev3.is_some_and(|t| t.lower == "no"))
                && (prev1.is_some_and(|t| (t.lower == "or") || (t.lower == "nor"))))
        {
            valence_value = lex_valence * N_SCALAR;
        }

        // Python: if item.isupper() and is_cap_diff:
        //             if valence > 0: valence += C_INCR
        //             else: valence -= C_INCR
        if item.is_upper && is_cap_diff {
            if valence_value > 0.0 {
                valence_value += C_INCR;
            } else {
                valence_value -= C_INCR;
            }
        }

        // Everything above is float arithmetic in Python; only an idiom
        // replacement in the distance-3 block below can hand back an int.
        let mut valence_from_int = false;

        // The reference's booster scan, `for start_i in range(0, 3)`, unrolled
        // onto the named window locals. Each arm mirrors:
        //   if i > start_i and words_and_emoticons[i - (start_i + 1)].lower()
        //           not in self.lexicon:
        //       s = scalar_inc_dec(...)
        //       if start_i == 1 and s != 0: s = s * 0.95
        //       if start_i == 2 and s != 0: s = s * 0.9
        //       valence = valence + s
        //       valence = self._negation_check(valence, words_and_emoticons, start_i, i)
        //       if start_i == 2:
        //           valence = self._special_idioms_check(valence, words_and_emoticons, i)
        // Note the negation and idiom checks only run when the token at that
        // distance is absent from the lexicon.

        // start_i == 0
        if let Some(p1) = prev1 {
            if !tables.lexicon.contains_key(p1.lower.as_str()) {
                let s = scalar_inc_dec(p1, valence_value, is_cap_diff, tables);
                valence_value += s;
                valence_value = negation_check_distance_1(valence_value, p1);
            }
        }
        // start_i == 1
        if let (Some(p1), Some(p2)) = (prev1, prev2) {
            if !tables.lexicon.contains_key(p2.lower.as_str()) {
                let mut s = scalar_inc_dec(p2, valence_value, is_cap_diff, tables);
                if s != 0.0 {
                    s *= DAMP_2;
                }
                valence_value += s;
                valence_value = negation_check_distance_2(valence_value, p1, p2);
            }
        }
        // start_i == 2
        if let (Some(p1), Some(p2), Some(p3)) = (prev1, prev2, prev3) {
            if !tables.lexicon.contains_key(p3.lower.as_str()) {
                let mut s = scalar_inc_dec(p3, valence_value, is_cap_diff, tables);
                if s != 0.0 {
                    s *= DAMP_3;
                }
                valence_value += s;
                valence_value = negation_check_distance_3(valence_value, p1, p2, p3);
                (valence_value, valence_from_int) =
                    special_idioms_check(valence_value, tokens, i, tables);
            }
        }

        valence = least_check(valence_value, valence_from_int, prev1, prev2, tables);
    }
    sentiments.push(valence);
}

/// The reference's `negated` for a single window token: membership in the
/// `NEGATE` list, or containment of "n't". The containment is the one
/// substring the reference itself performs; it is spelled through `find`
/// because M2 grep-bans the substring predicate methods across the crate,
/// and the same ban is why the list membership stays `iter().any` over
/// clippy's suggested slice-membership method.
#[allow(clippy::search_is_some, clippy::manual_contains)]
fn negated(word_lower: &str) -> bool {
    (NEGATE.iter().any(|neg| *neg == word_lower)) || (word_lower.find("n't").is_some())
}

/// The reference's `scalar_inc_dec`: the booster contribution of one
/// preceding token, sign-aligned with the valence, with its own all-caps
/// adjustment under the cap-differential condition.
fn scalar_inc_dec(word: &Token, valence: f64, is_cap_diff: bool, tables: &Tables) -> f64 {
    let mut scalar = 0.0_f64;
    if let Some(&increment) = tables.boosters.get(word.lower.as_str()) {
        scalar = increment;
        if valence < 0.0 {
            scalar = -scalar;
        }
        // Python: if word.isupper() and is_cap_diff:
        if word.is_upper && is_cap_diff {
            if valence > 0.0 {
                scalar += C_INCR;
            } else {
                scalar -= C_INCR;
            }
        }
    }
    scalar
}

/// `_negation_check`, `start_i == 0`:
///
/// ```python
/// if negated([words_and_emoticons_lower[i - (start_i + 1)]]):
///     valence = valence * N_SCALAR
/// ```
fn negation_check_distance_1(valence: f64, prev1: &Token) -> f64 {
    if negated(&prev1.lower) {
        valence * N_SCALAR
    } else {
        valence
    }
}

/// `_negation_check`, `start_i == 1`:
///
/// ```python
/// if words_and_emoticons_lower[i - 2] == "never" and \
///         (words_and_emoticons_lower[i - 1] == "so" or
///          words_and_emoticons_lower[i - 1] == "this"):
///     valence = valence * 1.25
/// elif words_and_emoticons_lower[i - 2] == "without" and \
///         words_and_emoticons_lower[i - 1] == "doubt":
///     valence = valence
/// elif negated([words_and_emoticons_lower[i - (start_i + 1)]]):
///     valence = valence * N_SCALAR
/// ```
fn negation_check_distance_2(valence: f64, prev1: &Token, prev2: &Token) -> f64 {
    if (prev2.lower == "never") && ((prev1.lower == "so") || (prev1.lower == "this")) {
        valence * NEVER_SCALAR
    } else if (prev2.lower == "without") && (prev1.lower == "doubt") {
        valence
    } else if negated(&prev2.lower) {
        valence * N_SCALAR
    } else {
        valence
    }
}

/// `_negation_check`, `start_i == 2`:
///
/// ```python
/// if words_and_emoticons_lower[i - 3] == "never" and \
///         (words_and_emoticons_lower[i - 2] == "so" or words_and_emoticons_lower[i - 2] == "this") or \
///         (words_and_emoticons_lower[i - 1] == "so" or words_and_emoticons_lower[i - 1] == "this"):
///     valence = valence * 1.25
/// elif words_and_emoticons_lower[i - 3] == "without" and \
///         (words_and_emoticons_lower[i - 2] == "doubt" or words_and_emoticons_lower[i - 1] == "doubt"):
///     valence = valence
/// elif negated([words_and_emoticons_lower[i - (start_i + 1)]]):
///     valence = valence * N_SCALAR
/// ```
///
/// Python's `and` binds tighter than `or`, so the first condition groups as
/// `(never3 and (so2 or this2)) or (so1 or this1)`: a bare "so" or "this"
/// directly before the scored token takes this arm with no "never" anywhere.
/// The parenthesization below spells that grouping out (M4).
fn negation_check_distance_3(valence: f64, prev1: &Token, prev2: &Token, prev3: &Token) -> f64 {
    if ((prev3.lower == "never") && ((prev2.lower == "so") || (prev2.lower == "this")))
        || ((prev1.lower == "so") || (prev1.lower == "this"))
    {
        valence * NEVER_SCALAR
    } else if (prev3.lower == "without") && ((prev2.lower == "doubt") || (prev1.lower == "doubt")) {
        valence
    } else if negated(&prev3.lower) {
        valence * N_SCALAR
    } else {
        valence
    }
}

/// The reference's `_least_check`:
///
/// ```python
/// if i > 1 and words_and_emoticons[i - 1].lower() not in self.lexicon \
///         and words_and_emoticons[i - 1].lower() == "least":
///     if words_and_emoticons[i - 2].lower() != "at" and words_and_emoticons[i - 2].lower() != "very":
///         valence = valence * N_SCALAR
/// elif i > 0 and words_and_emoticons[i - 1].lower() not in self.lexicon \
///         and words_and_emoticons[i - 1].lower() == "least":
///     valence = valence * N_SCALAR
/// ```
///
/// The `elif` can only fire at `i == 1`, because a "least" at `i - 1` with
/// `i > 1` is claimed by the first branch whether or not the exemption then
/// applies. The match below carries that structure: the two-predecessor arm
/// holds both the damped and the exempted paths, the one-predecessor arm
/// damps unconditionally.
///
/// A damping multiplication is a float operation in Python and clears the
/// int provenance an idiom replacement may carry; the untouched paths keep
/// it, which is why the flag rides through here.
fn least_check(
    value: f64,
    from_int: bool,
    prev1: Option<&Token>,
    prev2: Option<&Token>,
    tables: &Tables,
) -> Valence {
    let kept = Valence { value, from_int };
    let damped = Valence {
        value: value * N_SCALAR,
        from_int: false,
    };
    match (prev1, prev2) {
        (Some(p1), Some(p2)) => {
            if (!tables.lexicon.contains_key(p1.lower.as_str())) && (p1.lower == "least") {
                if (p2.lower != "at") && (p2.lower != "very") {
                    damped
                } else {
                    kept
                }
            } else {
                kept
            }
        }
        (Some(p1), None) => {
            if (!tables.lexicon.contains_key(p1.lower.as_str())) && (p1.lower == "least") {
                damped
            } else {
                kept
            }
        }
        (None, _) => kept,
    }
}

/// The one builder every idiom and booster n-gram lookup routes through (M2):
/// a contiguous token slice at fixed offsets, already-lowercased forms,
/// joined with single spaces. An out-of-range window yields `None`, which
/// the callers treat as "no candidate", matching the reference's guards. The
/// saturating end keeps the range valid at the far edge of `usize` and lands
/// in the same `None`.
fn joined_lower(tokens: &[Token], start: usize, len: usize) -> Option<String> {
    let slice = tokens.get(start..start.saturating_add(len))?;
    let mut joined = String::new();
    for (k, token) in slice.iter().enumerate() {
        if k > 0 {
            joined.push(' ');
        }
        joined.push_str(&token.lower);
    }
    Some(joined)
}

/// The reference's `_special_idioms_check`. The candidate set is fixed: five
/// backward sequences (the bigram ending at the scored word, the trigram
/// ending at it, the bigram ending one before it, the trigram ending one
/// before it, and the two-token sequence starting three back), two forward
/// sequences (the bigram and trigram starting at the scored word), then the
/// three backward n-grams against the booster table. The production caller
/// guarantees three predecessors; shorter windows only arise in direct unit
/// tests and skip cleanly.
///
/// Returns the valence plus whether it is a Python int at this point: an
/// idiom replacement adopts the int-ness of its table literal, and a booster
/// n-gram addition, being float arithmetic, clears it.
fn special_idioms_check(
    valence_in: f64,
    tokens: &[Token],
    i: usize,
    tables: &Tables,
) -> (f64, bool) {
    let mut valence = valence_in;
    let mut from_int = false;
    // Python: onezero = "{0} {1}".format(w[i - 1], w[i])
    let onezero = i.checked_sub(1).and_then(|s| joined_lower(tokens, s, 2));
    // Python: twoonezero = "{0} {1} {2}".format(w[i - 2], w[i - 1], w[i])
    let twoonezero = i.checked_sub(2).and_then(|s| joined_lower(tokens, s, 3));
    // Python: twoone = "{0} {1}".format(w[i - 2], w[i - 1])
    let twoone = i.checked_sub(2).and_then(|s| joined_lower(tokens, s, 2));
    // Python: threetwoone = "{0} {1} {2}".format(w[i - 3], w[i - 2], w[i - 1])
    let threetwoone = i.checked_sub(3).and_then(|s| joined_lower(tokens, s, 3));
    // Python: threetwo = "{0} {1}".format(w[i - 3], w[i - 2])
    let threetwo = i.checked_sub(3).and_then(|s| joined_lower(tokens, s, 2));

    // Python: sequences = [onezero, twoonezero, twoone, threetwoone, threetwo]
    //         for seq in sequences:
    //             if seq in SPECIAL_CASES: valence = SPECIAL_CASES[seq]; break
    for seq in [&onezero, &twoonezero, &twoone, &threetwoone, &threetwo] {
        if let Some(&hit) = seq
            .as_ref()
            .and_then(|s| tables.special_cases.get(s.as_str()))
        {
            valence = hit.value;
            from_int = hit.is_int;
            break;
        }
    }

    // Python: if len(words_and_emoticons_lower) - 1 > i:
    //             zeroone = "{0} {1}".format(w[i], w[i + 1])
    //             if zeroone in SPECIAL_CASES: valence = SPECIAL_CASES[zeroone]
    if let Some(&hit) = joined_lower(tokens, i, 2)
        .as_ref()
        .and_then(|s| tables.special_cases.get(s.as_str()))
    {
        valence = hit.value;
        from_int = hit.is_int;
    }
    // Python: if len(words_and_emoticons_lower) - 1 > i + 1:
    //             zeroonetwo = "{0} {1} {2}".format(w[i], w[i + 1], w[i + 2])
    //             if zeroonetwo in SPECIAL_CASES: valence = SPECIAL_CASES[zeroonetwo]
    if let Some(&hit) = joined_lower(tokens, i, 3)
        .as_ref()
        .and_then(|s| tables.special_cases.get(s.as_str()))
    {
        valence = hit.value;
        from_int = hit.is_int;
    }

    // Python: n_grams = [threetwoone, threetwo, twoone]
    //         for n_gram in n_grams:
    //             if n_gram in BOOSTER_DICT: valence = valence + BOOSTER_DICT[n_gram]
    for n_gram in [&threetwoone, &threetwo, &twoone] {
        if let Some(&increment) = n_gram
            .as_ref()
            .and_then(|s| tables.boosters.get(s.as_str()))
        {
            valence += increment;
            from_int = false;
        }
    }
    (valence, from_int)
}

/// The reference's `_but_check`, quirk included: Python iterates the live
/// list and re-finds each value with `sentiments.index(sentiment)`, which
/// returns the FIRST position holding an equal value, not the iteration
/// position. With duplicate valences across the pivot the reweighting lands
/// on the first equal slot, and this transcription reproduces that exactly
/// rather than the cleaner per-position rule.
///
/// ```python
/// if 'but' in words_and_emoticons_lower:
///     bi = words_and_emoticons_lower.index('but')
///     for sentiment in sentiments:
///         si = sentiments.index(sentiment)
///         if si < bi:
///             sentiments.pop(si); sentiments.insert(si, sentiment * 0.5)
///         elif si > bi:
///             sentiments.pop(si); sentiments.insert(si, sentiment * 1.5)
/// ```
///
/// Python's linear `index` scan per element made the transcription quadratic
/// on long many-valued inputs, so the first-equal-slot search reads a
/// maintained index instead: value bits to the ordered set of positions
/// currently holding that value, first position first. Each of the n
/// iterations does a bounded number of map and set operations, so the pass
/// is O(n log n) with O(n) extra memory, and the pathological-input test
/// below holds the bound.
///
/// Python's `index` compares by numeric equality (`3 == 3.0`, and `0.0`
/// equals `-0.0`), so the index is keyed on value bits with the two zeros
/// folded together, and equal values with differing int flags share one
/// key. A reweighting write is a float multiplication and clears the
/// written slot's int flag. A NaN value never equals anything under `==`,
/// so it maps to its own slot, matching the linear scan's no-match
/// fallback. No reachable valence is NaN.
fn but_check(tokens: &[Token], sentiments: &mut [Valence]) {
    /// Python `==` equality classes as bits: both zeros map to +0.0.
    fn key(value: f64) -> u64 {
        if value == 0.0 {
            0.0_f64.to_bits()
        } else {
            value.to_bits()
        }
    }
    if let Some(bi) = tokens.iter().position(|t| t.lower == "but") {
        let mut first_equal: HashMap<u64, BTreeSet<usize>> = HashMap::new();
        for (pos, sentiment) in sentiments.iter().enumerate() {
            let _ = first_equal
                .entry(key(sentiment.value))
                .or_default()
                .insert(pos);
        }
        for pos in 0..sentiments.len() {
            // `pos` is always in bounds, so the fallback value is inert.
            let sentiment = sentiments.get(pos).copied().unwrap_or(Valence::INT_ZERO);
            // The first slot holding this value: for a NaN the equality
            // search finds nothing and falls back to `pos` itself, and for
            // every other value the index holds at least `pos`, so the
            // fallback is inert.
            let si = if sentiment.value.is_nan() {
                pos
            } else {
                first_equal
                    .get(&key(sentiment.value))
                    .and_then(|set| set.first().copied())
                    .unwrap_or(pos)
            };
            // Python: if si < bi: replace with sentiment * 0.5
            //         elif si > bi: replace with sentiment * 1.5
            // The pivot's own slot (si == bi) is untouched. `get_mut` cannot
            // miss for a weighted slot, so its miss folds into the same
            // no-write path the pivot takes.
            let weight = match si.cmp(&bi) {
                Ordering::Less => Some(BUT_BEFORE),
                Ordering::Greater => Some(BUT_AFTER),
                Ordering::Equal => None,
            };
            if let (Some(weight), Some(slot)) = (weight, sentiments.get_mut(si)) {
                let written = Valence {
                    value: sentiment.value * weight,
                    from_int: false,
                };
                // The overwritten slot is always indexed, so the map hit is
                // total, and `map` folds the impossible miss into a no-op.
                let _ = first_equal
                    .get_mut(&key(slot.value))
                    .map(|set| set.remove(&si));
                let _ = first_equal
                    .entry(key(written.value))
                    .or_default()
                    .insert(si);
                *slot = written;
            }
        }
    }
}

/// The reference's `_amplify_ep`: 0.292 per "!", counting at most four.
fn amplify_ep(text: &str) -> f64 {
    // Python: ep_count = text.count("!"); if ep_count > 4: ep_count = 4
    let ep_count = text.chars().filter(|&c| c == '!').count().min(4);
    (ep_count as f64) * EP_STEP
}

/// The reference's `_amplify_qm`: nothing for one "?", 0.18 per "?" for two
/// or three, 0.96 past three.
fn amplify_qm(text: &str) -> f64 {
    let qm_count = text.chars().filter(|&c| c == '?').count();
    let mut qm_amplifier = 0.0_f64;
    // Python: if qm_count > 1:
    //             if qm_count <= 3: qm_amplifier = qm_count * 0.18
    //             else: qm_amplifier = 0.96
    if qm_count > 1 {
        if qm_count <= 3 {
            qm_amplifier = (qm_count as f64) * QM_STEP;
        } else {
            qm_amplifier = QM_CAP;
        }
    }
    qm_amplifier
}

/// The reference's `_punctuation_emphasis`.
fn punctuation_emphasis(text: &str) -> f64 {
    amplify_ep(text) + amplify_qm(text)
}

/// The reference's `normalize`: `score / sqrt(score * score + alpha)`,
/// clamped to [-1, 1]. The clamp is the reference's trailing bounds check;
/// the quotient is strictly inside (-1, 1) for every reachable sum, so
/// `f64::clamp` spells it without a dead branch of our own.
fn normalize(score: f64) -> f64 {
    (score / ((score * score) + ALPHA).sqrt()).clamp(-1.0, 1.0)
}

/// The reference's `_sift_sentiment_scores`: positive valences contribute
/// valence plus one, negatives valence minus one, zeros count as neutral.
/// The neutral count is held as f64; it is an exact integer well inside the
/// exact range, and Python promotes it to float at the same divisions.
fn sift_sentiment_scores(sentiments: &[Valence]) -> (f64, f64, f64) {
    let mut pos_sum = 0.0_f64;
    let mut neg_sum = 0.0_f64;
    let mut neu_count = 0.0_f64;
    for sentiment in sentiments {
        // Python: pos_sum += (float(sentiment_score) + 1) — explicit float
        // conversion, so int provenance is irrelevant here.
        let sentiment_score = sentiment.value;
        if sentiment_score > 0.0 {
            pos_sum += sentiment_score + 1.0;
        }
        if sentiment_score < 0.0 {
            neg_sum += sentiment_score - 1.0;
        }
        if sentiment_score == 0.0 {
            neu_count += 1.0;
        }
    }
    (pos_sum, neg_sum, neu_count)
}

/// CPython's builtin `sum` over the sentiments list, matched against probes
/// of the pinned oracle interpreter, because two of its behaviors are
/// observable in the pre-rounding compound:
///
/// - The int fast path: `sum` starts from int 0 and accumulates a leading
///   run of int items exactly, then adds the first non-int item to that
///   integer total in one plain, uncompensated addition (`PyNumber_Add`)
///   and enters the float loop. `sum([9, -2.6, -1.6])` differs from
///   `sum([9.0, -2.6, -1.6])` by exactly that missing compensation, and the
///   stage-1 corpus reached the same difference through int-valued idiom
///   replacements ("the bomb" is Python's int 3).
/// - The float loop (Neumaier-compensated summation since CPython 3.12):
///   every later item, int or float alike, takes the compensated step —
///   `sum([1e100, 1, -1e100])` is 1.0, so mid-run ints are compensated too.
///   Frozen row B65 caught the difference between this and a plain fold.
///
/// ```c
/// double t = f_result + x;
/// if (fabs(f_result) >= fabs(x)) { c += (f_result - t) + x; }
/// else                           { c += (x - t) + f_result; }
/// f_result = t;
/// ...
/// if (c && isfinite(c)) { f_result += c; }
/// ```
///
/// Every int in the sentiments list is a whole number of single-digit
/// magnitude, so the f64 leading-run accumulator is exact, matching
/// CPython's C-long arithmetic. The final result can never be negative zero
/// (the leading accumulator starts at `+0.0`, and every later operation
/// preserves that sign at an exact zero), which is what makes the zero arm
/// of the emphasis match below airtight.
fn python_sum(values: &[Valence]) -> f64 {
    let mut iter = values.iter();
    // Int fast path: exact whole-number accumulation in f64.
    let mut i_result = 0.0_f64;
    let (mut f_result, mut c) = loop {
        match iter.next() {
            None => return i_result,
            Some(v) if v.from_int => i_result += v.value,
            // First non-int item: one uncompensated add, then the float loop.
            Some(v) => break (i_result + v.value, 0.0_f64),
        }
    };
    for v in iter {
        let x = v.value;
        let t = f_result + x;
        if f_result.abs() >= x.abs() {
            c += (f_result - t) + x;
        } else {
            c += (x - t) + f_result;
        }
        f_result = t;
    }
    if (c != 0.0) && c.is_finite() {
        f_result += c;
    }
    f_result
}

/// The reference's `score_valence`, returning the rounded scores plus the
/// compound before rounding.
fn score_valence(sentiments: &[Valence], text: &str) -> (SentimentScores, f64) {
    let (compound_raw, neg, neu, pos);
    if !sentiments.is_empty() {
        // Python: sum_s = float(sum(sentiments))
        let mut sum_s = python_sum(sentiments);
        let punct_emph_amplifier = punctuation_emphasis(text);
        // Python: if sum_s > 0: sum_s += punct_emph_amplifier
        //         elif sum_s < 0: sum_s -= punct_emph_amplifier
        // Three arms, no default (M3): a zero sum takes no emphasis at all.
        match sum_s.total_cmp(&0.0) {
            Ordering::Greater => sum_s += punct_emph_amplifier,
            Ordering::Less => sum_s -= punct_emph_amplifier,
            Ordering::Equal => {}
        }
        compound_raw = normalize(sum_s);

        let (mut pos_sum, mut neg_sum, neu_count) = sift_sentiment_scores(sentiments);
        // Python: if pos_sum > math.fabs(neg_sum): pos_sum += punct_emph_amplifier
        //         elif pos_sum < math.fabs(neg_sum): neg_sum -= punct_emph_amplifier
        if pos_sum > neg_sum.abs() {
            pos_sum += punct_emph_amplifier;
        } else if pos_sum < neg_sum.abs() {
            neg_sum -= punct_emph_amplifier;
        }
        let total = pos_sum + neg_sum.abs() + neu_count;
        pos = (pos_sum / total).abs();
        neg = (neg_sum / total).abs();
        neu = (neu_count / total).abs();
    } else {
        compound_raw = 0.0;
        pos = 0.0;
        neg = 0.0;
        neu = 0.0;
    }
    let scores = SentimentScores {
        neg: python_round(neg, 3),
        neu: python_round(neu, 3),
        pos: python_round(pos, 3),
        compound: python_round(compound_raw, 4),
    };
    (scores, compound_raw)
}

#[cfg(test)]
#[allow(clippy::indexing_slicing)]
mod tests {
    use super::*;
    use crate::text::words_and_emoticons;

    fn tables() -> Tables {
        Tables::load()
    }

    /// Invariant 8: every constant of the constants table, pinned to its
    /// exact value, and the data-table entries that must agree with them.
    #[test]
    fn constants_pinned() {
        assert_eq!(B_INCR, 0.293);
        assert_eq!(B_DECR, -0.293);
        assert_eq!(C_INCR, 0.733);
        assert_eq!(N_SCALAR, -0.74);
        assert_eq!(EP_STEP, 0.292);
        assert_eq!(QM_STEP, 0.18);
        assert_eq!(QM_CAP, 0.96);
        assert_eq!(ALPHA, 15.0);
        assert_eq!(BUT_BEFORE, 0.5);
        assert_eq!(BUT_AFTER, 1.5);
        assert_eq!(DAMP_2, 0.95);
        assert_eq!(DAMP_3, 0.9);
        assert_eq!(NEVER_SCALAR, 1.25);
        assert_eq!(NEGATE.len(), 59);
        let t = tables();
        assert_eq!(t.boosters.get("so").copied(), Some(B_INCR));
        assert_eq!(t.boosters.get("sort of").copied(), Some(B_DECR));
    }

    #[test]
    fn negated_matches_the_reference_set() {
        assert!(negated("not"));
        assert!(negated("never"));
        assert!(negated("without"));
        assert!(negated("uh-uh"));
        assert!(negated("wasn't"));
        // "n't" containment, not list membership.
        assert!(negated("shan't"));
        assert!(negated("zzzn'tzzz"));
        // "no" is deliberately absent: it has its own rule.
        assert!(!negated("no"));
        assert!(!negated("least"));
        assert!(!negated("good"));
        assert!(!negated(""));
    }

    #[test]
    fn scalar_inc_dec_signs_and_caps() {
        let t = tables();
        let very = &words_and_emoticons("very")[0];
        let very_caps = &words_and_emoticons("VERY")[0];
        let plain = &words_and_emoticons("word")[0];
        assert_eq!(scalar_inc_dec(very, 1.9, false, &t), 0.293);
        assert_eq!(scalar_inc_dec(very, -1.9, false, &t), -0.293);
        assert_eq!(scalar_inc_dec(plain, 1.9, false, &t), 0.0);
        // All-caps booster under cap differential gains C_INCR by sign.
        assert_eq!(scalar_inc_dec(very_caps, 1.9, true, &t), 0.293 + C_INCR);
        assert_eq!(scalar_inc_dec(very_caps, -1.9, true, &t), -0.293 - C_INCR);
        // No cap differential: no C_INCR.
        assert_eq!(scalar_inc_dec(very_caps, 1.9, false, &t), 0.293);
        // Zero valence takes the negative arm of the caps branch, as in Python.
        assert_eq!(scalar_inc_dec(very_caps, 0.0, true, &t), 0.293 - C_INCR);
    }

    #[test]
    fn negation_check_distance_2_arms() {
        let tok = |s: &str| words_and_emoticons(s).remove(0);
        let (never, so, this, without, doubt, not, word) = (
            tok("never"),
            tok("so"),
            tok("this"),
            tok("without"),
            tok("doubt"),
            tok("not"),
            tok("word"),
        );
        assert_eq!(negation_check_distance_2(2.0, &so, &never), 2.5);
        assert_eq!(negation_check_distance_2(2.0, &this, &never), 2.5);
        // "never word": the never arm misses, NEGATE membership fires.
        assert_eq!(
            negation_check_distance_2(2.0, &word, &never),
            2.0 * N_SCALAR
        );
        // "without doubt": pass-through.
        assert_eq!(negation_check_distance_2(2.0, &doubt, &without), 2.0);
        // "without word": NEGATE membership of "without" fires.
        assert_eq!(
            negation_check_distance_2(2.0, &word, &without),
            2.0 * N_SCALAR
        );
        assert_eq!(negation_check_distance_2(2.0, &word, &not), 2.0 * N_SCALAR);
        assert_eq!(negation_check_distance_2(2.0, &word, &word), 2.0);
    }

    #[test]
    fn negation_check_distance_3_precedence() {
        let tok = |s: &str| words_and_emoticons(s).remove(0);
        let (never, so, this, without, doubt, not, word) = (
            tok("never"),
            tok("so"),
            tok("this"),
            tok("without"),
            tok("doubt"),
            tok("not"),
            tok("word"),
        );
        // never3 with so/this at distance 2.
        assert_eq!(negation_check_distance_3(2.0, &word, &so, &never), 2.5);
        assert_eq!(negation_check_distance_3(2.0, &word, &this, &never), 2.5);
        // The precedence quirk: bare "so"/"this" at distance 1, no "never".
        assert_eq!(negation_check_distance_3(2.0, &so, &word, &word), 2.5);
        assert_eq!(negation_check_distance_3(2.0, &this, &word, &word), 2.5);
        // without3 with doubt at either inner position.
        assert_eq!(negation_check_distance_3(2.0, &word, &doubt, &without), 2.0);
        assert_eq!(negation_check_distance_3(2.0, &doubt, &word, &without), 2.0);
        // NEGATE membership of the distance-3 token.
        assert_eq!(
            negation_check_distance_3(2.0, &word, &word, &not),
            2.0 * N_SCALAR
        );
        // "never" at distance 3 with no so/this anywhere: "never" is in
        // NEGATE, so the membership arm fires, exactly as in Python.
        assert_eq!(
            negation_check_distance_3(2.0, &word, &word, &never),
            2.0 * N_SCALAR
        );
        assert_eq!(negation_check_distance_3(2.0, &word, &word, &word), 2.0);
    }

    #[test]
    fn least_check_arms() {
        let t = tables();
        let tok = |s: &str| words_and_emoticons(s).remove(0);
        let (least, at, very, word, good) = (
            tok("least"),
            tok("at"),
            tok("very"),
            tok("word"),
            tok("good"),
        );
        let value = |v: Valence| v.value;
        // Two predecessors, damped: "word least X".
        let damped = least_check(1.9, false, Some(&least), Some(&word), &t);
        assert_eq!(damped.value, 1.9 * N_SCALAR);
        assert!(!damped.from_int);
        // Two predecessors, exempted: "at least X" and "very least X".
        assert_eq!(
            value(least_check(1.9, false, Some(&least), Some(&at), &t)),
            1.9
        );
        assert_eq!(
            value(least_check(1.9, false, Some(&least), Some(&very), &t)),
            1.9
        );
        // prev1 is not "least": untouched.
        assert_eq!(
            value(least_check(1.9, false, Some(&word), Some(&at), &t)),
            1.9
        );
        // prev1 in the lexicon: the guard blocks the branch.
        assert_eq!(
            value(least_check(1.9, false, Some(&good), Some(&word), &t)),
            1.9
        );
        // One predecessor: unconditional damping at i == 1.
        assert_eq!(
            value(least_check(1.9, false, Some(&least), None, &t)),
            1.9 * N_SCALAR
        );
        assert_eq!(value(least_check(1.9, false, Some(&word), None, &t)), 1.9);
        // No predecessors.
        assert_eq!(value(least_check(1.9, false, None, None, &t)), 1.9);
        // The untouched paths keep int provenance; damping clears it.
        assert!(least_check(3.0, true, None, None, &t).from_int);
        assert!(least_check(3.0, true, Some(&least), Some(&at), &t).from_int);
        assert!(!least_check(3.0, true, Some(&least), Some(&word), &t).from_int);
    }

    #[test]
    fn joined_lower_windows() {
        let toks = words_and_emoticons("The Quick brown");
        assert_eq!(joined_lower(&toks, 0, 2).as_deref(), Some("the quick"));
        assert_eq!(
            joined_lower(&toks, 0, 3).as_deref(),
            Some("the quick brown")
        );
        assert_eq!(joined_lower(&toks, 2, 1).as_deref(), Some("brown"));
        // Out of range and saturating far edge both yield None.
        assert_eq!(joined_lower(&toks, 2, 2), None);
        assert_eq!(joined_lower(&toks, 3, 1), None);
        assert_eq!(joined_lower(&toks, usize::MAX, 2), None);
    }

    #[test]
    fn special_idioms_direct_windows() {
        let t = tables();
        // Forward bigram from i == 0: "bad ass" replaces the valence with the
        // float literal 1.5.
        let toks = words_and_emoticons("bad ass");
        assert_eq!(special_idioms_check(-2.5, &toks, 0, &t), (1.5, false));
        // Backward bigram at i == 1: "the shit" replaces the valence with the
        // int literal 3.
        let toks = words_and_emoticons("the shit");
        assert_eq!(special_idioms_check(-2.6, &toks, 1, &t), (3.0, true));
        // Forward trigram: "kiss of death" from the scored token.
        let toks = words_and_emoticons("kiss of death now");
        assert_eq!(special_idioms_check(1.8, &toks, 0, &t), (-1.5, false));
        // Booster n-gram adds without replacing: "sort of good".
        let toks = words_and_emoticons("x sort of good");
        assert_eq!(
            special_idioms_check(1.9, &toks, 3, &t),
            (1.9 + B_DECR, false)
        );
        // An int replacement followed by a booster n-gram add goes float:
        // "the shit" via twoone, then "sort of"... there is no such overlap
        // in the real tables, so pin the flag transitions separately: the
        // int-valued idiom "to die for" at threetwoone with a booster bigram
        // would clear the flag; here the n-gram miss keeps it.
        let toks = words_and_emoticons("was to die for good");
        assert_eq!(special_idioms_check(1.9, &toks, 4, &t), (3.0, true));
        // No candidate hits: valence unchanged, no int provenance.
        let toks = words_and_emoticons("plain words only here");
        assert_eq!(special_idioms_check(1.1, &toks, 3, &t), (1.1, false));
    }

    /// The `_but_check` first-equal-slot quirk, traced against CPython: with
    /// `[2.0, 1.0, 0.0, 1.0]` and the pivot at 2, Python lands both halvings
    /// on the leading duplicates and never reweights the trailing 1.0.
    #[test]
    fn but_check_replicates_the_index_quirk() {
        let run = |text: &str, values: Vec<f64>| {
            let toks = words_and_emoticons(text);
            let mut s: Vec<Valence> = values
                .into_iter()
                .map(|value| Valence {
                    value,
                    from_int: false,
                })
                .collect();
            but_check(&toks, &mut s);
            s.iter().map(|v| v.value).collect::<Vec<f64>>()
        };
        assert_eq!(
            run("w x but y", vec![2.0, 1.0, 0.0, 1.0]),
            vec![0.5, 0.5, 0.0, 1.0]
        );
        assert_eq!(run("w but y", vec![2.0, 0.0, 2.0]), vec![1.0, 0.0, 3.0]);
        assert_eq!(
            run("w x but y", vec![1.0, 1.0, 0.0, 1.0]),
            vec![0.5, 0.5, 0.0, 1.5]
        );
        assert_eq!(run("but y z", vec![0.0, 0.5, 0.5]), vec![0.0, 0.75, 0.75]);
        // No "but": untouched.
        assert_eq!(run("w x y", vec![1.0, 2.0, 3.0]), vec![1.0, 2.0, 3.0]);
        // A reweighted slot loses int provenance (`3 * 1.5` is Python float);
        // an untouched slot keeps it, as does a slot whose first-equal search
        // lands on the pivot itself.
        let toks = words_and_emoticons("but y");
        let mut s = vec![
            Valence::INT_ZERO,
            Valence {
                value: 3.0,
                from_int: true,
            },
        ];
        but_check(&toks, &mut s);
        assert!(s[0].from_int);
        assert_eq!(s[1].value, 4.5);
        assert!(!s[1].from_int);
    }

    /// The maintained index must land every write exactly where Python's
    /// linear first-equal scan lands it. The reference here is a literal
    /// transcription of that scan, compared over every rotation of a
    /// duplicate-heavy palette (both zero signs, values whose 0.5 and 1.5
    /// products collide with other palette members) at every length and
    /// every pivot position.
    #[test]
    fn but_check_matches_the_linear_scan_reference() {
        fn linear_scan(bi: usize, sentiments: &mut [Valence]) {
            for pos in 0..sentiments.len() {
                let sentiment = sentiments.get(pos).copied().unwrap_or(Valence::INT_ZERO);
                let si = sentiments
                    .iter()
                    .position(|v| v.value == sentiment.value)
                    .unwrap_or(pos);
                let weight = match si.cmp(&bi) {
                    Ordering::Less => Some(BUT_BEFORE),
                    Ordering::Greater => Some(BUT_AFTER),
                    Ordering::Equal => None,
                };
                if let (Some(weight), Some(slot)) = (weight, sentiments.get_mut(si)) {
                    *slot = Valence {
                        value: sentiment.value * weight,
                        from_int: false,
                    };
                }
            }
        }
        let palette = [
            2.0, 1.0, -0.0, 0.0, 1.0, 3.0, -1.5, 2.0, 0.5, 1.5, -1.5, 4.5,
        ];
        for n in 1..=palette.len() {
            for shift in 0..palette.len() {
                let values: Vec<Valence> = palette
                    .iter()
                    .cycle()
                    .skip(shift)
                    .take(n)
                    .map(|&value| Valence {
                        value,
                        from_int: false,
                    })
                    .collect();
                for bi in 0..n {
                    let tokens: Vec<Token> = (0..n)
                        .map(|k| Token {
                            lower: String::from(if k == bi { "but" } else { "w" }),
                            is_upper: false,
                        })
                        .collect();
                    let mut expected = values.clone();
                    linear_scan(bi, &mut expected);
                    let mut actual = values.clone();
                    but_check(&tokens, &mut actual);
                    let bits =
                        |s: &[Valence]| s.iter().map(|v| v.value.to_bits()).collect::<Vec<u64>>();
                    assert_eq!(
                        bits(&actual),
                        bits(&expected),
                        "n={n} shift={shift} bi={bi}"
                    );
                }
            }
        }
    }

    /// A NaN valence never equals anything under Python `==`, so the
    /// first-equal search falls back to the slot itself, exactly as the
    /// linear scan's no-match fallback behaved. No real scoring reaches a
    /// NaN valence. The pin keeps the arm covered and equivalent.
    #[test]
    fn but_check_nan_falls_back_to_its_own_slot() {
        let toks = words_and_emoticons("x but y");
        let mut s = floats(&[f64::NAN, 0.0, 2.0]);
        but_check(&toks, &mut s);
        assert!(s[0].value.is_nan());
        assert_eq!(s[1].value.to_bits(), 0.0_f64.to_bits());
        assert_eq!(s[2].value, 3.0);
    }

    /// Python's `==` folds the two zeros together, so a `-0.0` finds a
    /// `0.0` slot and the reverse. Traced against CPython: with values
    /// `[0.0, -0.0, 5.0]` and the pivot at 2, both zero visits land on slot
    /// 0 and slot 1 is never rewritten.
    #[test]
    fn but_check_folds_zero_signs_like_python_eq() {
        let toks = words_and_emoticons("w x but");
        let mut s = floats(&[0.0, -0.0, 5.0]);
        but_check(&toks, &mut s);
        assert_eq!(s[0].value.to_bits(), (-0.0_f64).to_bits());
        assert_eq!(s[1].value.to_bits(), (-0.0_f64).to_bits());
        assert_eq!(s[2].value, 5.0);
    }

    /// Regression hold on the quadratic blowup: 60,000 distinct valences
    /// with the pivot at the far end put every first-equal search at its
    /// worst. The linear-scan transcription needs about 1.8e9 equality
    /// comparisons here, minutes in a debug run. The indexed pass finishes
    /// in milliseconds, and the budget below is two orders looser.
    #[test]
    fn but_check_pathological_input_stays_fast() {
        let n = 60_000_usize;
        let mut tokens: Vec<Token> = Vec::with_capacity(n);
        for _ in 1..n {
            tokens.push(Token {
                lower: String::from("w"),
                is_upper: false,
            });
        }
        tokens.push(Token {
            lower: String::from("but"),
            is_upper: false,
        });
        // Distinct values whose halves collide with nothing: fractions are
        // .25 before the pass and .125 or .625 after.
        let mut s = floats(&(0..n).map(|i| i as f64 + 0.25).collect::<Vec<f64>>());
        let started = std::time::Instant::now();
        but_check(&tokens, &mut s);
        let elapsed = started.elapsed();
        for (i, v) in s.iter().enumerate() {
            let original = i as f64 + 0.25;
            let expected = if i.checked_add(1) == Some(n) {
                original
            } else {
                original * BUT_BEFORE
            };
            assert_eq!(v.value, expected, "slot {i}");
        }
        assert!(
            elapsed < std::time::Duration::from_secs(10),
            "but_check took {elapsed:?} on the pathological input"
        );
    }

    #[test]
    fn punctuation_amplifiers_ladder() {
        assert_eq!(amplify_ep(""), 0.0);
        assert_eq!(amplify_ep("!"), EP_STEP);
        assert_eq!(amplify_ep("!!"), 2.0 * EP_STEP);
        assert_eq!(amplify_ep("!!!!"), 4.0 * EP_STEP);
        // Counting caps at four.
        assert_eq!(amplify_ep("!!!!!!"), 4.0 * EP_STEP);
        assert_eq!(amplify_qm(""), 0.0);
        assert_eq!(amplify_qm("?"), 0.0);
        assert_eq!(amplify_qm("??"), 2.0 * QM_STEP);
        assert_eq!(amplify_qm("???"), 3.0 * QM_STEP);
        assert_eq!(amplify_qm("????"), QM_CAP);
        assert_eq!(punctuation_emphasis("!?!?"), 2.0 * EP_STEP + 2.0 * QM_STEP);
    }

    #[test]
    fn normalize_shape() {
        assert_eq!(normalize(0.0), 0.0);
        assert_eq!(normalize(1.0), 1.0 / 16.0_f64.sqrt());
        assert_eq!(normalize(-1.0), -(1.0 / 16.0_f64.sqrt()));
        // Far outside the usual range the quotient rounds to exactly 1.
        assert_eq!(normalize(1.0e10), 1.0);
        assert_eq!(normalize(-1.0e10), -1.0);
    }

    fn floats(values: &[f64]) -> Vec<Valence> {
        values
            .iter()
            .map(|&value| Valence {
                value,
                from_int: false,
            })
            .collect()
    }

    #[test]
    fn sift_masses_and_neutral_count() {
        let (pos, neg, neu) = sift_sentiment_scores(&floats(&[1.9, -2.6, 0.0, 0.0, 0.5]));
        assert_eq!(pos, (1.9 + 1.0) + (0.5 + 1.0));
        assert_eq!(neg, -2.6 - 1.0);
        assert_eq!(neu, 2.0);
        assert_eq!(sift_sentiment_scores(&[]), (0.0, 0.0, 0.0));
    }

    /// The CPython 3.12+ compensated `sum`: the plain left fold of frozen row
    /// B65's valences lands one ulp low, and the compensation recovers the
    /// bit CPython returns.
    #[test]
    fn python_sum_is_neumaier_compensated() {
        let sum = python_sum(&floats(&[1.1, 1.2, 0.0, 1.9]));
        assert_eq!(sum.to_bits(), 4.2_f64.to_bits());
        // The uncompensated fold is one ulp lower; this is the divergence the
        // compensation exists to close.
        let fold = [1.1_f64, 1.2, 0.0, 1.9]
            .iter()
            .fold(0.0_f64, |acc, &v| acc + v);
        assert!(fold < sum);
        assert_eq!(python_sum(&[]), 0.0);
        // Negative zero entries cannot make the accumulator negative zero.
        assert_eq!(python_sum(&floats(&[-0.0])).to_bits(), 0.0_f64.to_bits());
        assert_eq!(
            python_sum(&floats(&[-0.0, -0.0])).to_bits(),
            0.0_f64.to_bits()
        );
        assert_eq!(
            python_sum(&floats(&[1.9, -1.9])).to_bits(),
            0.0_f64.to_bits()
        );
    }

    /// CPython's int handling in `sum`, pinned against an oracle trace: the
    /// stage-1 sentence "(coffee emoji) the bomb gracefullest delectably the
    /// shitty noon :-[ villainess infuriated noon intensely" yields the mixed
    /// list below (int-valued idiom replacements among floats), and CPython
    /// returns -1.0999999999999996 (bits 0xbff1_9999_9999_9998) where an
    /// all-float Neumaier run lands elsewhere. Ints accumulate exactly up
    /// front and are added without compensation once floats begin.
    #[test]
    fn python_sum_mixes_int_and_float_paths_like_cpython() {
        let iz = || Valence::INT_ZERO;
        let ii = |value: f64| Valence {
            value,
            from_int: true,
        };
        let ff = |value: f64| Valence {
            value,
            from_int: false,
        };
        let mixed = [
            iz(),
            iz(),
            iz(),
            ii(3.0),
            ii(3.0),
            ii(3.0),
            iz(),
            ff(-2.6),
            iz(),
            ff(-1.6),
            ff(-2.9),
            ff(-3.0),
            iz(),
            iz(),
        ];
        assert_eq!(python_sum(&mixed).to_bits(), 0xbff1_9999_9999_9998);
        // All-int list: the int fast path runs to the end.
        assert_eq!(python_sum(&[ii(3.0), iz(), ii(-2.0)]), 1.0);
        // An int after floats takes the compensated step exactly as a float
        // would: CPython's sum([1e100, 1, -1e100]) is 1.0.
        assert_eq!(
            python_sum(&[ff(0.1), ff(0.2), ii(3.0)]).to_bits(),
            python_sum(&[ff(0.1), ff(0.2), ff(3.0)]).to_bits()
        );
        assert_eq!(python_sum(&[ff(1.0e100), ii(1.0), ff(-1.0e100)]), 1.0);
        // The int-to-float transition add is the one uncompensated addition:
        // CPython's sum([9, -2.6, -1.6]) is 0x4013333333333334 while
        // sum([9.0, -2.6, -1.6]) is 0x4013333333333333.
        assert_eq!(
            python_sum(&[ii(9.0), ff(-2.6), ff(-1.6)]).to_bits(),
            0x4013_3333_3333_3334
        );
        assert_eq!(
            python_sum(&[ff(9.0), ff(-2.6), ff(-1.6)]).to_bits(),
            0x4013_3333_3333_3333
        );
    }

    #[test]
    fn score_valence_empty_is_all_zero() {
        let (scores, raw) = score_valence(&[], "!!");
        assert_eq!(raw, 0.0);
        assert_eq!(scores.neg, 0.0);
        assert_eq!(scores.neu, 0.0);
        assert_eq!(scores.pos, 0.0);
        assert_eq!(scores.compound, 0.0);
    }
}