toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11

use std::borrow::Cow;

use icu_normalizer::ComposingNormalizer;
use icu_normalizer::ComposingNormalizerBorrowed;

static NORMALIZER: ComposingNormalizerBorrowed<'static> = ComposingNormalizer::new_nfkc();

#[inline]
pub fn normalize(input: &str) -> Cow<'_, str> {
    NORMALIZER.normalize(input)
}