rehuman 0.2.0

Unicode-safe text cleaning & typographic normalization for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

//! Unicode character-set helpers used by the cleaning engine.

use std::sync::OnceLock;

use icu_properties::props::{GeneralCategory, Script};
use icu_properties::{
    props, CodePointMapData, CodePointMapDataBorrowed, CodePointSetData, CodePointSetDataBorrowed,
};

static DEFAULT_IGNORABLES: OnceLock<CodePointSetDataBorrowed<'static>> = OnceLock::new();

static EMOJI_SET: OnceLock<CodePointSetDataBorrowed<'static>> = OnceLock::new();

static SCRIPT_MAP: OnceLock<CodePointMapDataBorrowed<'static, Script>> = OnceLock::new();

static GENERAL_CATEGORY_MAP: OnceLock<CodePointMapDataBorrowed<'static, GeneralCategory>> =
    OnceLock::new();

fn script(c: char) -> Script {
    SCRIPT_MAP
        .get_or_init(CodePointMapData::<Script>::new)
        .get(c)
}

fn general_category(c: char) -> GeneralCategory {
    GENERAL_CATEGORY_MAP
        .get_or_init(CodePointMapData::<GeneralCategory>::new)
        .get(c)
}

/// Latin-script characters (per the Unicode `Script` property), the candidates
/// for deunicode-based transliteration of letters: diacritics, ligatures,
/// phonetic/IPA extensions, and every Latin Extended block — without
/// maintaining a hardcoded block list.
///
/// # Returns
/// `true` when `c` has the Unicode `Latin` script property.
pub(crate) fn is_latin_script(c: char) -> bool {
    script(c) == Script::Latin
}

/// Unicode control characters (General_Category=Control).
///
/// # Returns
/// `true` when `c` has Unicode general category `Control`.
pub(crate) fn is_control_char(c: char) -> bool {
    general_category(c) == GeneralCategory::Control
}

/// Script-neutral symbols and punctuation: characters whose `General_Category`
/// is a symbol, punctuation, separator, other-number, or modifier-letter class
/// and whose `Script` is Common or Inherited. This is the transliteration
/// fallback surface — it deliberately excludes letters, marks, and digits of
/// concrete scripts (CJK, Cyrillic, Braille, ...), which keep dropping rather
/// than being romanized.
///
/// # Returns
/// `true` when `c` is a Common/Inherited character in one of the supported
/// symbol, punctuation, separator, number, or modifier-letter categories.
pub(crate) fn is_common_symbol_or_punctuation(c: char) -> bool {
    matches!(script(c), Script::Common | Script::Inherited)
        && matches!(
            general_category(c),
            GeneralCategory::MathSymbol
                | GeneralCategory::CurrencySymbol
                | GeneralCategory::ModifierSymbol
                | GeneralCategory::OtherSymbol
                | GeneralCategory::DashPunctuation
                | GeneralCategory::OpenPunctuation
                | GeneralCategory::ClosePunctuation
                | GeneralCategory::InitialPunctuation
                | GeneralCategory::FinalPunctuation
                | GeneralCategory::OtherPunctuation
                | GeneralCategory::OtherNumber
                | GeneralCategory::ModifierLetter
                | GeneralCategory::SpaceSeparator
                | GeneralCategory::LineSeparator
                | GeneralCategory::ParagraphSeparator
        )
}

/// Hidden/format-like characters defined by Default_Ignorable_Code_Point (DI).
///
/// # Returns
/// `true` when `c` should be treated as hidden/invisible.
pub fn is_hidden_char(c: char) -> bool {
    DEFAULT_IGNORABLES
        .get_or_init(CodePointSetData::new::<props::DefaultIgnorableCodePoint>)
        .contains(c)
}

/// ASCII keyboard (US) characters + whitespace controls typically produced by keyboards.
///
/// # Returns
/// `true` when `c` is accepted by keyboard-only output mode.
pub fn is_keyboard_ascii(c: char) -> bool {
    matches!(c, '\n' | '\r' | '\t') || (c.is_ascii() && !c.is_ascii_control())
}

/// Curated non-ASCII characters allowed in extended keyboard mode.
///
/// # Returns
/// `true` when `c` is accepted by the extended keyboard allowlist.
pub fn is_extended_keyboard_char(c: char) -> bool {
    matches!(
        c,
        '€' | '£' | '¥' | '¢' | '§' | '°' | '±' | '×' | '÷' | '–' | '—' | '…' | '•' | '·'
    )
}

/// Emoji detection via the Unicode `Emoji` binary property.
///
/// # Returns
/// `true` when `c` has the Unicode `Emoji` property.
pub fn is_emoji(c: char) -> bool {
    EMOJI_SET
        .get_or_init(CodePointSetData::new::<props::Emoji>)
        .contains(c)
}