toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6

use std::borrow::Cow;

#[inline]
pub fn normalize(input: &str) -> Cow<'_, str> {
    Cow::Owned(input.to_lowercase())
}