toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

use std::borrow::Cow;

/// Prepend normalizer: inserts a fixed string at the beginning of the input.
#[derive(Debug, Clone)]
pub struct Prepend {
    prepend: String,
}

impl Prepend {
    #[inline]
    pub fn new(prepend: String) -> Self {
        Self { prepend }
    }

    pub fn normalize<'a>(&self, input: &'a str) -> Cow<'a, str> {
        if self.prepend.is_empty() {
            return Cow::Borrowed(input);
        }
        let hint = self.prepend.len() + input.len();
        let mut result = String::with_capacity(hint);
        result.push_str(&self.prepend);
        result.push_str(input);
        Cow::Owned(result)
    }
}