toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

use std::borrow::Cow;

/// Strip normalizer.
#[derive(Debug, Clone, Copy)]
pub struct Strip {
    pub strip_left: bool,
    pub strip_right: bool,
}

impl Strip {
    #[inline]
    pub fn new(strip_left: bool, strip_right: bool) -> Self {
        Self {
            strip_left,
            strip_right,
        }
    }

    #[inline]
    pub fn normalize(self, input: &str) -> Cow<'_, str> {
        match (self.strip_left, self.strip_right) {
            (true, true) => Cow::Borrowed(input.trim()),
            (true, false) => Cow::Borrowed(input.trim_start()),
            (false, true) => Cow::Borrowed(input.trim_end()),
            (false, false) => Cow::Borrowed(input),
        }
    }
}