vader-parity 0.1.0

//! Casing predicates with CPython semantics, read from vendored tables.
//!
//! `str.isupper()` and `str.lower()` answer from the interpreter's own
//! Unicode tables, and those tables move between Unicode versions. Rust's
//! `char::is_uppercase` and `str::to_lowercase` move on their own schedule,
//! so a codepoint the pinned CPython leaves uncased can be uppercase under
//! the Rust standard library (U+A7CE is one), which flips the all-caps
//! booster and diverges the scores. The predicates here therefore read only
//! `data/casing_tables.tsv`, recorded from the pinned interpreter by probing
//! every codepoint through the same str methods the pipeline calls. The
//! header of that file names the CPython and Unicode versions, its digest is
//! pinned in `data/PROVENANCE`, and the sweep test below replays the
//! recorded probe serialization over all codepoints against these
//! predicates, so a mismatch on any single codepoint fails the build.

use std::cmp::Ordering;
use std::sync::OnceLock;

/// `data/casing_tables.tsv`, recorded from the pinned oracle interpreter.
const CASING_TABLES: &str = include_str!("../data/casing_tables.tsv");

const CAPITAL_SIGMA: char = '\u{03A3}';
const SMALL_SIGMA: char = '\u{03C3}';
const FINAL_SIGMA: char = '\u{03C2}';

/// The parsed casing tables: sorted inclusive codepoint ranges for the four
/// class sets, and the sorted non-identity `str.lower()` mapping.
pub(crate) struct CasingTables {
    /// Chars that make `str.isupper()` False on sight: CPython's per-char
    /// ISLOWER or ISTITLE.
    isupper_false: Vec<(u32, u32)>,
    /// Chars that count as the cased uppercase witness `str.isupper()`
    /// requires at least one of.
    isupper_cased: Vec<(u32, u32)>,
    /// Chars the final-sigma context scan skips (Case_Ignorable).
    case_ignorable: Vec<(u32, u32)>,
    /// Chars the final-sigma context scan accepts as cased once the
    /// ignorable skip stops on them.
    cased: Vec<(u32, u32)>,
    /// Codepoint to its full `str.lower()` expansion, where not identity.
    lower: Vec<(u32, String)>,
}

impl CasingTables {
    /// Parse the vendored table text. Rows are `kind TAB start TAB end` for
    /// the range sets and `lower TAB cp TAB mapped...` for the mapping, all
    /// hex. Comment and malformed lines are skipped so the parser stays
    /// total. The digest recorded in `data/PROVENANCE` pins the vendored
    /// bytes, and the sweep test fails on any behavioral gap a skipped line
    /// would leave.
    pub(crate) fn parse(raw: &str) -> CasingTables {
        let mut tables = CasingTables {
            isupper_false: Vec::new(),
            isupper_cased: Vec::new(),
            case_ignorable: Vec::new(),
            cased: Vec::new(),
            lower: Vec::new(),
        };
        for line in raw.lines() {
            if line.is_empty() || line.as_bytes().first() == Some(&b'#') {
                continue;
            }
            let mut fields = line.split('\t');
            let (Some(kind), Some(a), Some(b)) = (fields.next(), fields.next(), fields.next())
            else {
                continue;
            };
            let Ok(a) = u32::from_str_radix(a, 16) else {
                continue;
            };
            if kind == "lower" {
                let mapped: String = b
                    .split(' ')
                    .filter_map(|cp| u32::from_str_radix(cp, 16).ok())
                    .filter_map(char::from_u32)
                    .collect();
                tables.lower.push((a, mapped));
                continue;
            }
            let Ok(b) = u32::from_str_radix(b, 16) else {
                continue;
            };
            match kind {
                "isupper_false" => tables.isupper_false.push((a, b)),
                "isupper_cased" => tables.isupper_cased.push((a, b)),
                "case_ignorable" => tables.case_ignorable.push((a, b)),
                "cased" => tables.cased.push((a, b)),
                _ => {}
            }
        }
        tables.isupper_false.sort_unstable();
        tables.isupper_cased.sort_unstable();
        tables.case_ignorable.sort_unstable();
        tables.cased.sort_unstable();
        tables.lower.sort_unstable_by_key(|&(cp, _)| cp);
        tables
    }

    fn is_isupper_false(&self, c: char) -> bool {
        in_ranges(&self.isupper_false, c)
    }

    fn is_isupper_cased(&self, c: char) -> bool {
        in_ranges(&self.isupper_cased, c)
    }

    fn is_case_ignorable(&self, c: char) -> bool {
        in_ranges(&self.case_ignorable, c)
    }

    fn is_cased(&self, c: char) -> bool {
        in_ranges(&self.cased, c)
    }

    fn lower_of(&self, c: char) -> Option<&str> {
        let idx = self
            .lower
            .binary_search_by_key(&(c as u32), |&(cp, _)| cp)
            .ok()?;
        self.lower.get(idx).map(|(_, mapped)| mapped.as_str())
    }
}

/// Membership in a sorted list of inclusive ranges.
fn in_ranges(ranges: &[(u32, u32)], c: char) -> bool {
    let cp = c as u32;
    ranges
        .binary_search_by(|&(start, end)| {
            if end < cp {
                Ordering::Less
            } else if start > cp {
                Ordering::Greater
            } else {
                Ordering::Equal
            }
        })
        .is_ok()
}

/// The vendored tables, parsed once.
fn tables() -> &'static CasingTables {
    static TABLES: OnceLock<CasingTables> = OnceLock::new();
    TABLES.get_or_init(|| CasingTables::parse(CASING_TABLES))
}

/// Python `str.isupper()`: False on the first lowercase or titlecase char,
/// else True exactly when some char is cased uppercase. Every answer comes
/// from the vendored tables, so it is the pinned interpreter's.
pub(crate) fn py_isupper(s: &str) -> bool {
    let t = tables();
    let mut has_cased = false;
    for c in s.chars() {
        if t.is_isupper_false(c) {
            return false;
        }
        if t.is_isupper_cased(c) {
            has_cased = true;
        }
    }
    has_cased
}

/// Python `str.lower()`: the per-char full lowercase mapping from the
/// vendored tables, with CPython's one contextual case, capital sigma, which
/// lowercases to U+03C2 in final position and U+03C3 elsewhere.
pub(crate) fn py_lower(s: &str) -> String {
    let t = tables();
    let chars: Vec<char> = s.chars().collect();
    let mut out = String::with_capacity(s.len());
    for (i, &c) in chars.iter().enumerate() {
        if c == CAPITAL_SIGMA {
            let sigma = if final_sigma(t, &chars, i) {
                FINAL_SIGMA
            } else {
                SMALL_SIGMA
            };
            out.push(sigma);
        } else if let Some(mapped) = t.lower_of(c) {
            out.push_str(mapped);
        } else {
            out.push(c);
        }
    }
    out
}

/// CPython's final-sigma test at position `i`: skipping case-ignorable
/// chars, a cased char stands before the sigma and none stands after it.
fn final_sigma(t: &CasingTables, chars: &[char], i: usize) -> bool {
    let before = chars
        .iter()
        .take(i)
        .rev()
        .find(|&&c| !t.is_case_ignorable(c));
    if !before.is_some_and(|&c| t.is_cased(c)) {
        return false;
    }
    let after = chars
        .iter()
        .skip(i.saturating_add(1))
        .find(|&&c| !t.is_case_ignorable(c));
    !after.is_some_and(|&c| t.is_cased(c))
}

#[cfg(test)]
mod tests {
    use super::{py_isupper, py_lower, tables, CasingTables, CASING_TABLES};
    use crate::text::is_py_space;
    use sha2::{Digest, Sha256};
    use std::fmt::Write as _;

    #[test]
    fn isupper_matches_cpython() {
        assert!(py_isupper("VADER"));
        assert!(py_isupper("V1"));
        assert!(!py_isupper("Vader"));
        assert!(!py_isupper("vader"));
        assert!(!py_isupper("123"));
        assert!(!py_isupper(":)"));
        assert!(!py_isupper(""));
        // Titlecase Dz-digraph is cased but not uppercase in CPython.
        assert!(!py_isupper("A\u{01C5}"));
        assert!(!py_isupper("\u{01C5}"));
        // U+A7CE: uppercase under current Rust std, unassigned and uncased
        // in the pinned CPython. The vendored tables side with CPython.
        assert!(!py_isupper("\u{A7CE}"));
        assert!(py_isupper("GOOD\u{A7CE}"));
        // Other_Uppercase: circled letters count as the cased witness.
        assert!(py_isupper("\u{24B6}"));
    }

    #[test]
    fn lower_matches_cpython() {
        assert_eq!(py_lower("GOOD"), "good");
        assert_eq!(py_lower("don't"), "don't");
        assert_eq!(py_lower(""), "");
        // Unmapped and uncased codepoints pass through.
        assert_eq!(py_lower("\u{A7CE}1!"), "\u{A7CE}1!");
        // Full mappings expand: İ is "i" plus a combining dot above, and ẞ
        // lowers to ß while ß and fi stay themselves.
        assert_eq!(py_lower("\u{0130}"), "i\u{0307}");
        assert_eq!(py_lower("\u{1E9E}"), "\u{00DF}");
        assert_eq!(py_lower("\u{00DF}\u{FB01}"), "\u{00DF}\u{FB01}");
    }

    /// Every sigma-context arm, values traced against the oracle: the
    /// final form needs a cased char before (skipping case-ignorables like
    /// U+0345) and no cased char after.
    #[test]
    fn sigma_context_matches_cpython() {
        assert_eq!(py_lower("\u{03A3}"), "\u{03C3}");
        assert_eq!(py_lower("\u{03A3}\u{03A3}"), "\u{03C3}\u{03C2}");
        assert_eq!(py_lower("A\u{03A3}B"), "a\u{03C3}b");
        assert_eq!(py_lower("A\u{03A3}"), "a\u{03C2}");
        assert_eq!(py_lower("A\u{03A3}!"), "a\u{03C2}!");
        assert_eq!(py_lower("!\u{03A3}"), "!\u{03C3}");
        assert_eq!(py_lower("A\u{0345}\u{03A3}"), "a\u{0345}\u{03C2}");
        assert_eq!(py_lower("A\u{03A3}\u{0345}B"), "a\u{03C3}\u{0345}b");
        assert_eq!(py_lower("A\u{03A3}\u{0345}!"), "a\u{03C2}\u{0345}!");
        assert_eq!(py_lower("\u{0345}\u{03A3}"), "\u{0345}\u{03C3}");
    }

    #[test]
    fn parser_skips_junk_and_sorts() {
        let t = CasingTables::parse(
            "# comment\n\nisupper_cased\t61\t7a\nisupper_cased\t41\t5a\nlower\t42\t62\nlower\t41\t61 300\nlower\tzz\t61\nmystery\t1\t2\nisupper_false\tqq\t2\ncased\t1\tzz\nshort\n",
        );
        assert!(t.is_isupper_cased('B'));
        assert!(t.is_isupper_cased('b'));
        assert!(!t.is_isupper_cased('!'));
        assert!(!t.is_isupper_false('a'));
        assert_eq!(t.lower_of('A'), Some("a\u{300}"));
        assert_eq!(t.lower_of('B'), Some("b"));
        assert_eq!(t.lower_of('C'), None);
    }

    /// The all-codepoint casing parity sweep. `gen_casing.py` probed every
    /// codepoint through the oracle interpreter's `str.isupper()`,
    /// `str.lower()`, `str.isspace()`, and the final-sigma context, wrote a
    /// canonical serialization of the results, and recorded its sha256 in
    /// the table header. This test rebuilds the identical serialization from
    /// the crate's own predicates, so one differing codepoint anywhere in
    /// the space changes the digest and fails.
    ///
    /// The last-char comparisons stay spelled through `next_back` because
    /// the M2 grep bans the substring predicate methods across the crate,
    /// which is what the lint would rewrite them to.
    #[allow(clippy::chars_last_cmp)]
    #[test]
    fn all_codepoints_match_the_recorded_oracle_sweep() {
        let recorded = CASING_TABLES
            .lines()
            .find_map(|l| l.strip_prefix("# sweep-sha256\t"))
            .map(str::to_string);
        assert!(recorded.is_some(), "no sweep digest in casing_tables.tsv");
        let mut serial = String::new();
        for cp in 0..=0x10FFFF_u32 {
            let Some(c) = char::from_u32(cp) else {
                continue; // surrogates, unreachable from Rust text
            };
            let u1 = py_isupper(&String::from(c));
            let u2 = py_isupper(&format!("{c}A"));
            let low = py_lower(&String::from(c));
            let s1 = py_lower(&format!("A{c}\u{03A3}")).chars().next_back() == Some('\u{03C2}');
            let s2 = py_lower(&format!("{c}\u{03A3}")).chars().next_back() == Some('\u{03C2}');
            let sp = is_py_space(c);
            let _ = write!(serial, "{cp:06x}\t");
            for (k, mc) in low.chars().enumerate() {
                if k > 0 {
                    serial.push(' ');
                }
                let _ = write!(serial, "{:x}", mc as u32);
            }
            let _ = writeln!(
                serial,
                "\t{}{}{}{}{}",
                u32::from(u1),
                u32::from(u2),
                u32::from(s1),
                u32::from(s2),
                u32::from(sp)
            );
        }
        let computed = format!("{:x}", Sha256::digest(serial.as_bytes()));
        assert_eq!(
            recorded.unwrap_or_default(),
            computed,
            "casing predicates diverge from the recorded oracle sweep"
        );
    }

    /// The vendored header names the interpreter the tables came from. The
    /// cross-check against `data/PROVENANCE` lives in `tests/provenance.rs`.
    #[test]
    fn table_header_names_the_pinned_interpreter() {
        assert!(CASING_TABLES.lines().any(|l| l == "# cpython\t3.14.6"));
        assert!(CASING_TABLES.lines().any(|l| l == "# unidata\t16.0.0"));
        // Loaded table sanity: the parsed sets are non-empty and sorted use
        // finds a known member of each.
        let t = tables();
        assert!(t.is_isupper_false('a'));
        assert!(t.is_isupper_cased('A'));
        assert!(t.is_case_ignorable('\u{0345}'));
        assert!(t.is_cased('a'));
    }
}