vibrato 0.5.2

Vibrato: viterbi-based accelerated tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

use bincode::{Decode, Encode};

#[derive(Default, Decode, Encode)]
pub struct WordFeatures {
    features: Vec<String>,
}

impl WordFeatures {
    pub fn new<I, S>(features: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        Self {
            features: features
                .into_iter()
                .map(|s| s.as_ref().to_string())
                .collect(),
        }
    }

    #[inline(always)]
    pub fn get(&self, word_id: usize) -> &str {
        &self.features[word_id]
    }
}