paginate-core 0.1.0

//! Text normalization for search and filtering.
//!
//! Port of pypaginate's `text/normalize.py`:
//!
//! * **ASCII fast path** — lowercase then collapse whitespace. On ASCII,
//!   `str.casefold()` equals lowercasing, so this is byte-identical to Python.
//! * **Non-ASCII path** — NFKD-decompose, drop combining marks, lowercase,
//!   collapse whitespace.
//!
//! The bounded result cache stays in the binding layer (exactly where the
//! Python module keeps it), so this function is referentially transparent and
//! trivially portable.
//!
//! Known minor divergences from `str.casefold()` on the non-ASCII path: full
//! case folds such as `ß → ss` are not applied (we lowercase instead), and
//! spacing/enclosing marks (Mc/Me) are also dropped, not just nonspacing (Mn).
//! These affect a tiny fraction of inputs and never the ASCII fast path.

use unicode_normalization::char::is_combining_mark;
use unicode_normalization::UnicodeNormalization;

/// Normalize `value` for case- and accent-insensitive matching.
#[must_use]
pub fn normalize_text(value: &str) -> String {
    if value.is_ascii() {
        return collapse_whitespace(&value.to_ascii_lowercase());
    }
    let stripped: String = value.nfkd().filter(|c| !is_combining_mark(*c)).collect();
    collapse_whitespace(&stripped.to_lowercase())
}

/// Collapse whitespace runs to single spaces and trim the ends —
/// equivalent to Python's `" ".join(s.split())`.
fn collapse_whitespace(s: &str) -> String {
    s.split_whitespace().collect::<Vec<_>>().join(" ")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ascii_lowercases_and_collapses() {
        assert_eq!(normalize_text("Hello World"), "hello world");
        assert_eq!(normalize_text("  Foo   Bar  "), "foo bar");
        assert_eq!(normalize_text("UPPER"), "upper");
        assert_eq!(normalize_text(""), "");
        assert_eq!(normalize_text("   "), "");
    }

    #[test]
    fn strips_accents_on_latin_text() {
        assert_eq!(normalize_text("Café"), "cafe");
        assert_eq!(normalize_text("JOSÉ"), "jose");
        assert_eq!(normalize_text("naïve"), "naive");
        assert_eq!(normalize_text("Crème Brûlée"), "creme brulee");
    }

    #[test]
    fn idempotent() {
        let once = normalize_text("Héllo   WÖRLD");
        assert_eq!(normalize_text(&once), once);
    }
}