morpheme 0.4.0

//! Unicode normalization forms and the NMT cleanup normalizer.

use crate::error::Result;
use crate::normalized_string::NormalizedString;
use crate::pre_tokenizers::impl_unit_serde;
use crate::traits::Normalizer;

/// Unicode canonical decomposition (NFD).
///
/// # Example
///
/// ```
/// use morpheme::{NormalizedString, Normalizer};
///
/// fn normalize(n: &impl Normalizer, text: &str) -> String {
///     let mut s = NormalizedString::from(text);
///     n.normalize(&mut s).unwrap();
///     s.get().to_owned()
/// }
///
/// use morpheme::normalizers::Nfd;
///
/// // "é" (1 char) decomposes into "e" + U+0301 (2 chars).
/// assert_eq!(normalize(&Nfd, "é").chars().count(), 2);
/// ```
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Nfd;

impl_unit_serde!(Nfd);

impl Normalizer for Nfd {
    fn normalize(&self, normalized: &mut NormalizedString) -> Result<()> {
        normalized.nfd();
        Ok(())
    }
}

/// Unicode compatibility decomposition (NFKD).
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Nfkd;

impl_unit_serde!(Nfkd);

impl Normalizer for Nfkd {
    fn normalize(&self, normalized: &mut NormalizedString) -> Result<()> {
        normalized.nfkd();
        Ok(())
    }
}

/// Unicode canonical composition (NFC).
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Nfc;

impl_unit_serde!(Nfc);

impl Normalizer for Nfc {
    fn normalize(&self, normalized: &mut NormalizedString) -> Result<()> {
        normalized.nfc();
        Ok(())
    }
}

/// Unicode compatibility composition (NFKC).
///
/// # Example
///
/// ```
/// use morpheme::{NormalizedString, Normalizer};
///
/// fn normalize(n: &impl Normalizer, text: &str) -> String {
///     let mut s = NormalizedString::from(text);
///     n.normalize(&mut s).unwrap();
///     s.get().to_owned()
/// }
///
/// use morpheme::normalizers::Nfkc;
///
/// // Compatibility forms fold into their plain equivalents.
/// assert_eq!(normalize(&Nfkc, "fine ①"), "fine 1");
/// ```
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Nfkc;

impl_unit_serde!(Nfkc);

impl Normalizer for Nfkc {
    fn normalize(&self, normalized: &mut NormalizedString) -> Result<()> {
        normalized.nfkc();
        Ok(())
    }
}

/// The cleanup SentencePiece's `nmt_nfkc` rules apply before NFKC:
/// drop most C0 control chars and map assorted spaces/separators
/// (including `▁` itself) to a plain space.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Nmt;

impl_unit_serde!(Nmt);

impl Normalizer for Nmt {
    fn normalize(&self, normalized: &mut NormalizedString) -> Result<()> {
        normalized
            .filter(|c| {
                !matches!(
                    c as u32,
                    0x0001..=0x0008 | 0x000B | 0x000E..=0x001F | 0x007F | 0x008F | 0x009F
                )
            })
            .map(|c| match c as u32 {
                0x0009
                | 0x000A
                | 0x000C
                | 0x000D
                | 0x1680
                | 0x200B..=0x200F
                | 0x2028
                | 0x2029
                | 0x2581
                | 0xFEFF
                | 0xFFFD => ' ',
                _ => c,
            });
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::normalized_string::OffsetRange;

    fn norm<N: Normalizer>(n: N, s: &str) -> String {
        let mut ns = NormalizedString::from(s);
        n.normalize(&mut ns).unwrap();
        ns.get().to_owned()
    }

    // Expected outputs from Python `tokenizers` 0.23.2.
    #[test]
    fn forms_match_hf() {
        assert_eq!(norm(Nfkd, "fi ½ ①"), "fi 1\u{2044}2 1");
        assert_eq!(norm(Nfkc, "fi ½ ①"), "fi 1\u{2044}2 1");
        assert_eq!(norm(Nfkc, "你好，世界！"), "你好,世界!");
        assert_eq!(norm(Nfkc, "™\u{1e}g"), "TM\u{1e}g");
        assert_eq!(norm(Nfkc, "and\u{a0}nbsp"), "and nbsp");
        assert_eq!(norm(Nfc, "e\u{301} x"), "\u{e9} x");
        assert_eq!(norm(Nfd, "\u{e9}"), "e\u{301}");
    }

    #[test]
    fn nmt_matches_hf() {
        assert_eq!(
            norm(Nmt, "a\u{1}b\tc\u{200b}d\u{2581}e\u{feff}"),
            "ab c d e "
        );
        assert_eq!(norm(Nmt, "x\u{7f}y"), "xy");
    }

    #[test]
    fn nfkc_expansion_offsets() {
        let mut ns = NormalizedString::from("a™b");
        Nfkc.normalize(&mut ns).unwrap();
        assert_eq!(ns.get(), "aTMb");
        // Both "T" and "M" come from "™" (original bytes 1..4).
        assert_eq!(
            ns.convert_offsets(OffsetRange::Normalized(1..2)),
            Some(1..4)
        );
        assert_eq!(
            ns.convert_offsets(OffsetRange::Normalized(2..3)),
            Some(1..4)
        );
        assert_eq!(
            ns.get_range_original(OffsetRange::Normalized(3..4)),
            Some("b")
        );
    }
}