toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

use std::borrow::Cow;

use super::Error;
use fancy_regex::Regex;
use serde::Deserialize;

#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "PascalCase")]
pub struct ReplacePattern {
    pub string: Option<String>,
    pub regex: Option<String>,
}

/// Replace normalizer: substitutes matches of a pattern with a fixed string.
#[derive(Debug, Clone)]
pub struct Replace {
    pattern: Regex,
    content: String,
}

impl Replace {
    pub fn new(pattern: ReplacePattern, content: &str) -> Result<Self, Error> {
        let source: String = match (&pattern.string, &pattern.regex) {
            (Some(s), _) => fancy_regex::escape(s).into_owned(),
            (None, Some(r)) => r.clone(),
            (None, None) => {
                return Err(Error::InvalidConfig(
                    "Replace pattern must have String or Regex field".to_owned(),
                ));
            }
        };
        let re = match Regex::new(&source) {
            Ok(r) => r,
            Err(e) => return Err(Error::InvalidConfig(format!("Replace regex: {e}"))),
        };

        Ok(Self {
            pattern: re,
            content: content.to_owned(),
        })
    }

    #[inline]
    pub fn normalize<'a>(&self, input: &'a str) -> Cow<'a, str> {
        self.pattern.replace_all(input, &self.content)
    }
}