tpt-tokenizer-core 0.1.0

Pure-Rust BPE and WordPiece tokenizer — no_std + alloc compatible, zero dependencies.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

//! Unicode normalization (NFC / NFD / NFKC / NFKD) applied before tokenization.
//!
//! Correct Unicode normalization requires the full Unicode Character Database
//! (composition/decomposition mappings and combining classes), which is far too
//! large to hand-roll while staying correct. To keep the crate's default build
//! **dependency-free**, this capability is gated behind the opt-in
//! `normalization` Cargo feature, which pulls in the well-tested
//! [`unicode-normalization`] crate (itself `no_std` + `alloc`).
//!
//! Enable it with:
//!
//! ```toml
//! tpt-tokenizer-core = { version = "0.1", features = ["normalization"] }
//! ```
//!
//! [`unicode-normalization`]: https://docs.rs/unicode-normalization

use alloc::string::String;
use unicode_normalization::UnicodeNormalization;

/// A Unicode normalization form, matching the four standard forms used by
/// Hugging Face tokenizers.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NormalizationForm {
    /// Canonical decomposition followed by canonical composition (NFC).
    Nfc,
    /// Canonical decomposition (NFD).
    Nfd,
    /// Compatibility decomposition followed by canonical composition (NFKC).
    Nfkc,
    /// Compatibility decomposition (NFKD).
    Nfkd,
}

/// Normalize `text` to the given [`NormalizationForm`].
#[must_use]
pub fn normalize(text: &str, form: NormalizationForm) -> String {
    match form {
        NormalizationForm::Nfc => text.nfc().collect(),
        NormalizationForm::Nfd => text.nfd().collect(),
        NormalizationForm::Nfkc => text.nfkc().collect(),
        NormalizationForm::Nfkd => text.nfkd().collect(),
    }
}