toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11

use std::borrow::Cow;

use icu_normalizer::DecomposingNormalizer;
use icu_normalizer::DecomposingNormalizerBorrowed;

static NORMALIZER: DecomposingNormalizerBorrowed<'static> = DecomposingNormalizer::new_nfkd();

#[inline]
pub fn normalize(input: &str) -> Cow<'_, str> {
    NORMALIZER.normalize(input)
}