toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

use std::borrow::Cow;

use icu_properties::CodePointMapDataBorrowed;
use icu_properties::props::GeneralCategory;

/// Unicode `General_Category` lookup backed by compiled ICU data.
static GENERAL_CATEGORY: CodePointMapDataBorrowed<'static, GeneralCategory> =
    CodePointMapDataBorrowed::new();

/// Check whether a character is a non-spacing combining mark (`Mn`).
#[inline(always)]
fn is_nonspacing_mark(c: char) -> bool {
    GENERAL_CATEGORY.get(c) == GeneralCategory::NonspacingMark
}

/// Remove all non-spacing marks (`Mn`), matching HuggingFace's `StripAccents`.
///
/// No decomposition is performed here: configs that need it pair this step
/// with `NFD` (e.g. BERT's `Sequence [NFD, Lowercase, StripAccents]`).
#[inline]
pub fn normalize(input: &str) -> Cow<'_, str> {
    // Fast path: borrow the input unchanged when there is nothing to strip.
    let Some((head, _)) = input.char_indices().find(|&(_, c)| is_nonspacing_mark(c)) else {
        return Cow::Borrowed(input);
    };

    let mut out = String::with_capacity(input.len());
    out.push_str(&input[..head]);
    out.extend(input[head..].chars().filter(|&c| !is_nonspacing_mark(c)));
    Cow::Owned(out)
}