vader-parity 0.1.0

//! Invariant 9: reference-order arithmetic. The pre-rounding compound for the
//! frozen corpus is bit-identical to the oracle's unrounded value on at least
//! 99 percent of rows, every non-identical row is listed in
//! `tests/vectors/ulp-exceptions.tsv` with its ulp distance, and no listed
//! distance exceeds 4. This module also carries the differential driver the
//! harness runs over the stage-1 and stage-2 corpora.

use crate::SentimentIntensityAnalyzer;
use std::fmt::Write as _;
use std::path::PathBuf;

fn vectors_dir() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
        .join("tests")
        .join("vectors")
}

/// Undo the frozen.tsv text escaping: `\\`, `\t`, `\n`, `\r`.
fn unescape(field: &str) -> String {
    let mut out = String::with_capacity(field.len());
    let mut chars = field.chars();
    while let Some(c) = chars.next() {
        if c == '\\' {
            match chars.next() {
                Some('t') => out.push('\t'),
                Some('n') => out.push('\n'),
                Some('r') => out.push('\r'),
                Some(other) => out.push(other),
                None => out.push('\\'),
            }
        } else {
            out.push(c);
        }
    }
    out
}

/// Map a float to an integer ordinal where adjacent floats differ by one, so
/// ulp distance is ordinal difference.
fn ordinal(x: f64) -> i64 {
    let bits = x.to_bits();
    if bits & 0x8000_0000_0000_0000 != 0 {
        -((bits & 0x7fff_ffff_ffff_ffff) as i64)
    } else {
        bits as i64
    }
}

fn ulp_distance(a: f64, b: f64) -> u64 {
    ordinal(a).abs_diff(ordinal(b))
}

struct FrozenRow {
    id: String,
    text: String,
    compound_raw: f64,
}

fn frozen_rows() -> Vec<FrozenRow> {
    let path = vectors_dir().join("frozen.tsv");
    let raw = std::fs::read_to_string(&path).unwrap_or_default();
    assert!(!raw.is_empty(), "missing {}", path.display());
    let mut rows = Vec::new();
    for line in raw.lines() {
        if line.is_empty() || line.as_bytes().first() == Some(&b'#') {
            continue;
        }
        let fields: Vec<&str> = line.split('\t').collect();
        assert_eq!(fields.len(), 7, "malformed frozen row: {line}");
        let (Some(id), Some(text), Some(raw_repr)) = (fields.first(), fields.get(1), fields.get(6))
        else {
            continue;
        };
        let parsed = raw_repr.parse::<f64>();
        assert!(parsed.is_ok(), "bad compound_raw in row {id}: {raw_repr}");
        rows.push(FrozenRow {
            id: (*id).to_string(),
            text: unescape(text),
            compound_raw: parsed.unwrap_or_default(),
        });
    }
    assert_eq!(rows.len(), 177, "frozen corpus row count drifted");
    rows
}

#[test]
fn frozen_corpus_pre_rounding_compound_is_reference_ordered() {
    let analyzer = SentimentIntensityAnalyzer::new();
    let rows = frozen_rows();
    let mut mismatched: Vec<(String, u64)> = Vec::new();
    for row in &rows {
        let (_, ours) = analyzer.polarity_scores_with_raw(&row.text);
        if ours.to_bits() != row.compound_raw.to_bits() {
            mismatched.push((row.id.clone(), ulp_distance(ours, row.compound_raw)));
        }
    }
    // At least 99 percent bit-identical, and nothing beyond 4 ulp.
    let allowed = rows.len() / 100;
    assert!(
        mismatched.len() <= allowed,
        "{} of {} rows not bit-identical: {mismatched:?}",
        mismatched.len(),
        rows.len()
    );
    for (id, ulps) in &mismatched {
        assert!(*ulps <= 4, "row {id} diverges by {ulps} ulp");
    }

    // The checked-in exception list must state exactly the observed set, so a
    // stale waiver cannot linger and a new divergence cannot hide.
    let listed =
        std::fs::read_to_string(vectors_dir().join("ulp-exceptions.tsv")).unwrap_or_default();
    let mut expected = String::from("# rows whose pre-rounding compound is not bit-identical to the oracle's repr\n# columns: id, ulp distance (max 4). Empty below this header means none.\n");
    for (id, ulps) in &mismatched {
        let _ = writeln!(expected, "{id}\t{ulps}");
    }
    assert_eq!(
        listed, expected,
        "ulp-exceptions.tsv does not match the observed divergence set"
    );
}

/// Compares `python_round` against CPython `round` output over a captured
/// sweep file (one line per case: `bits_hex TAB digits TAB rounded_repr`).
/// The harness generates the file (10^6 random doubles at 3 and 4 digits);
/// without it the test is a no-op skip.
#[test]
fn round_sweep_against_cpython_when_present() {
    use crate::round::python_round;
    let Ok(path) = std::env::var("VADER_ROUND_SWEEP") else {
        return;
    };
    let raw = std::fs::read_to_string(&path);
    assert!(raw.is_ok(), "sweep file {path} unreadable: {raw:?}");
    let raw = raw.unwrap_or_default();
    let mut checked = 0_u64;
    for line in raw.lines() {
        let mut parts = line.split('\t');
        let (Some(bits), Some(digits), Some(expected)) = (parts.next(), parts.next(), parts.next())
        else {
            continue;
        };
        let (Ok(bits), Ok(digits), Ok(expected)) = (
            u64::from_str_radix(bits, 16),
            digits.parse::<usize>(),
            expected.parse::<f64>(),
        ) else {
            continue;
        };
        let value = f64::from_bits(bits);
        let ours = python_round(value, digits);
        assert_eq!(
            ours.to_bits(),
            expected.to_bits(),
            "python_round({value:?}, {digits}) = {ours:?}, CPython says {expected:?}"
        );
        checked = checked.saturating_add(1);
    }
    assert!(checked > 0, "sweep file {path} held no cases");
}

/// The differential driver. Ignored in normal runs; the harness invokes it
/// with `VADER_DIFF_IN` pointing at a file of one text per line and
/// `VADER_DIFF_OUT` naming where to write
/// `index TAB neg TAB neu TAB pos TAB compound TAB raw_bits_hex` rows.
#[test]
#[ignore = "driver for the external differential harness"]
fn diff_driver() {
    let input = std::env::var("VADER_DIFF_IN").unwrap_or_default();
    let output = std::env::var("VADER_DIFF_OUT").unwrap_or_default();
    assert!(
        !input.is_empty() && !output.is_empty(),
        "set VADER_DIFF_IN and VADER_DIFF_OUT"
    );
    let texts = std::fs::read_to_string(&input);
    assert!(texts.is_ok(), "cannot read {input}: {texts:?}");
    let texts = texts.unwrap_or_default();
    let analyzer = SentimentIntensityAnalyzer::new();
    let mut out = String::new();
    for (idx, line) in texts.lines().enumerate() {
        let (scores, raw) = analyzer.polarity_scores_with_raw(line);
        let _ = writeln!(
            out,
            "{idx}\t{:?}\t{:?}\t{:?}\t{:?}\t{:016x}",
            scores.neg,
            scores.neu,
            scores.pos,
            scores.compound,
            raw.to_bits()
        );
    }
    let written = std::fs::write(&output, out);
    assert!(written.is_ok(), "cannot write {output}: {written:?}");
}