gigastt-core 2.20.0

//! Unicode category helpers for the WordPiece BertNormalizer / pre-tokenizer.
//!
//! The `CONTROL_FORMAT_RANGES` / `PUNCTUATION_RANGES` tables are a compact
//! range encoding of the category tables in the `unicode_categories` 0.1.1
//! crate — the exact data source the upstream `tokenizers` 0.23 normalizer
//! and pre-tokenizer consult — so `is_other` / `is_punctuation` decisions
//! match the upstream crate char for char.

/// `char::is_ascii_punctuation` or Unicode punctuation category (P*), as in
/// the upstream `is_bert_punc`.
pub(crate) fn is_bert_punc(c: char) -> bool {
    c.is_ascii_punctuation() || in_ranges(c as u32, PUNCTUATION_RANGES)
}

/// Control per the upstream BertNormalizer: any Unicode "Other" (Cc, Cf, Co)
/// except \t \n \r, which count as whitespace instead.
pub(crate) fn is_control(c: char) -> bool {
    if matches!(c, '\t' | '\n' | '\r') {
        return false;
    }
    let cp = c as u32;
    // Category Co per unicode_categories 0.1.1 (hardcoded ranges there; the
    // OTHER_PRIVATE_USE table only repeats the range boundaries).
    if matches!(cp, 0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD) {
        return true;
    }
    in_ranges(cp, CONTROL_FORMAT_RANGES)
}

/// CJK Unified Ideographs ranges, verbatim from the upstream `is_chinese_char`.
pub(crate) fn is_chinese_char(c: char) -> bool {
    matches!(
        c as u32,
        0x4E00..=0x9FFF
            | 0x3400..=0x4DBF
            | 0x20000..=0x2A6DF
            | 0x2A700..=0x2B73F
            | 0x2B740..=0x2B81F
            | 0x2B920..=0x2CEAF
            | 0xF900..=0xFAFF
            | 0x2F800..=0x2FA1F
    )
}

/// Binary search over sorted, non-overlapping inclusive `(lo, hi)` ranges.
fn in_ranges(cp: u32, ranges: &[(u32, u32)]) -> bool {
    let i = ranges.partition_point(|&(lo, _)| lo <= cp);
    i > 0 && cp <= ranges[i - 1].1
}

/// Unicode categories Cc + Cf (control / format), from the OTHER_CONTROL and
/// OTHER_FORMAT tables of `unicode_categories` 0.1.1, merged into ranges.
const CONTROL_FORMAT_RANGES: &[(u32, u32)] = &[
    (0x0, 0x1F),
    (0x7F, 0x9F),
    (0xAD, 0xAD),
    (0x600, 0x605),
    (0x61C, 0x61C),
    (0x6DD, 0x6DD),
    (0x70F, 0x70F),
    (0x180E, 0x180E),
    (0x200B, 0x200F),
    (0x202A, 0x202E),
    (0x2060, 0x2064),
    (0x2066, 0x206F),
    (0xFEFF, 0xFEFF),
    (0xFFF9, 0xFFFB),
    (0x110BD, 0x110BD),
    (0x1BCA0, 0x1BCA3),
    (0x1D173, 0x1D17A),
    (0xE0001, 0xE0001),
    (0xE0020, 0xE007F),
];

/// Unicode punctuation categories Pc + Pd + Pe + Pf + Pi + Po + Ps, from the
/// PUNCTUATION_* tables of `unicode_categories` 0.1.1, merged into ranges.
const PUNCTUATION_RANGES: &[(u32, u32)] = &[
    (0x21, 0x23),
    (0x25, 0x2A),
    (0x2C, 0x2F),
    (0x3A, 0x3B),
    (0x3F, 0x40),
    (0x5B, 0x5D),
    (0x5F, 0x5F),
    (0x7B, 0x7B),
    (0x7D, 0x7D),
    (0xA1, 0xA1),
    (0xA7, 0xA7),
    (0xAB, 0xAB),
    (0xB6, 0xB7),
    (0xBB, 0xBB),
    (0xBF, 0xBF),
    (0x37E, 0x37E),
    (0x387, 0x387),
    (0x55A, 0x55F),
    (0x589, 0x58A),
    (0x5BE, 0x5BE),
    (0x5C0, 0x5C0),
    (0x5C3, 0x5C3),
    (0x5C6, 0x5C6),
    (0x5F3, 0x5F4),
    (0x609, 0x60A),
    (0x60C, 0x60D),
    (0x61B, 0x61B),
    (0x61E, 0x61F),
    (0x66A, 0x66D),
    (0x6D4, 0x6D4),
    (0x700, 0x70D),
    (0x7F7, 0x7F9),
    (0x830, 0x83E),
    (0x85E, 0x85E),
    (0x964, 0x965),
    (0x970, 0x970),
    (0xAF0, 0xAF0),
    (0xDF4, 0xDF4),
    (0xE4F, 0xE4F),
    (0xE5A, 0xE5B),
    (0xF04, 0xF12),
    (0xF14, 0xF14),
    (0xF3A, 0xF3D),
    (0xF85, 0xF85),
    (0xFD0, 0xFD4),
    (0xFD9, 0xFDA),
    (0x104A, 0x104F),
    (0x10FB, 0x10FB),
    (0x1360, 0x1368),
    (0x1400, 0x1400),
    (0x166D, 0x166E),
    (0x169B, 0x169C),
    (0x16EB, 0x16ED),
    (0x1735, 0x1736),
    (0x17D4, 0x17D6),
    (0x17D8, 0x17DA),
    (0x1800, 0x180A),
    (0x1944, 0x1945),
    (0x1A1E, 0x1A1F),
    (0x1AA0, 0x1AA6),
    (0x1AA8, 0x1AAD),
    (0x1B5A, 0x1B60),
    (0x1BFC, 0x1BFF),
    (0x1C3B, 0x1C3F),
    (0x1C7E, 0x1C7F),
    (0x1CC0, 0x1CC7),
    (0x1CD3, 0x1CD3),
    (0x2010, 0x2027),
    (0x2030, 0x2043),
    (0x2045, 0x2051),
    (0x2053, 0x205E),
    (0x207D, 0x207E),
    (0x208D, 0x208E),
    (0x2308, 0x230B),
    (0x2329, 0x232A),
    (0x2768, 0x2775),
    (0x27C5, 0x27C6),
    (0x27E6, 0x27EF),
    (0x2983, 0x2998),
    (0x29D8, 0x29DB),
    (0x29FC, 0x29FD),
    (0x2CF9, 0x2CFC),
    (0x2CFE, 0x2CFF),
    (0x2D70, 0x2D70),
    (0x2E00, 0x2E2E),
    (0x2E30, 0x2E42),
    (0x3001, 0x3003),
    (0x3008, 0x3011),
    (0x3014, 0x301F),
    (0x3030, 0x3030),
    (0x303D, 0x303D),
    (0x30A0, 0x30A0),
    (0x30FB, 0x30FB),
    (0xA4FE, 0xA4FF),
    (0xA60D, 0xA60F),
    (0xA673, 0xA673),
    (0xA67E, 0xA67E),
    (0xA6F2, 0xA6F7),
    (0xA874, 0xA877),
    (0xA8CE, 0xA8CF),
    (0xA8F8, 0xA8FA),
    (0xA8FC, 0xA8FC),
    (0xA92E, 0xA92F),
    (0xA95F, 0xA95F),
    (0xA9C1, 0xA9CD),
    (0xA9DE, 0xA9DF),
    (0xAA5C, 0xAA5F),
    (0xAADE, 0xAADF),
    (0xAAF0, 0xAAF1),
    (0xABEB, 0xABEB),
    (0xFD3E, 0xFD3F),
    (0xFE10, 0xFE19),
    (0xFE30, 0xFE52),
    (0xFE54, 0xFE61),
    (0xFE63, 0xFE63),
    (0xFE68, 0xFE68),
    (0xFE6A, 0xFE6B),
    (0xFF01, 0xFF03),
    (0xFF05, 0xFF0A),
    (0xFF0C, 0xFF0F),
    (0xFF1A, 0xFF1B),
    (0xFF1F, 0xFF20),
    (0xFF3B, 0xFF3D),
    (0xFF3F, 0xFF3F),
    (0xFF5B, 0xFF5B),
    (0xFF5D, 0xFF5D),
    (0xFF5F, 0xFF65),
    (0x10100, 0x10102),
    (0x1039F, 0x1039F),
    (0x103D0, 0x103D0),
    (0x1056F, 0x1056F),
    (0x10857, 0x10857),
    (0x1091F, 0x1091F),
    (0x1093F, 0x1093F),
    (0x10A50, 0x10A58),
    (0x10A7F, 0x10A7F),
    (0x10AF0, 0x10AF6),
    (0x10B39, 0x10B3F),
    (0x10B99, 0x10B9C),
    (0x11047, 0x1104D),
    (0x110BB, 0x110BC),
    (0x110BE, 0x110C1),
    (0x11140, 0x11143),
    (0x11174, 0x11175),
    (0x111C5, 0x111C9),
    (0x111CD, 0x111CD),
    (0x111DB, 0x111DB),
    (0x111DD, 0x111DF),
    (0x11238, 0x1123D),
    (0x112A9, 0x112A9),
    (0x114C6, 0x114C6),
    (0x115C1, 0x115D7),
    (0x11641, 0x11643),
    (0x1173C, 0x1173E),
    (0x12470, 0x12474),
    (0x16A6E, 0x16A6F),
    (0x16AF5, 0x16AF5),
    (0x16B37, 0x16B3B),
    (0x16B44, 0x16B44),
    (0x1BC9F, 0x1BC9F),
    (0x1DA87, 0x1DA8B),
];