vibrato 0.5.2

Vibrato: viterbi-based accelerated tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

use bincode::{Decode, Encode};

use crate::dictionary::mapper::ConnIdMapper;

#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, Decode, Encode)]
pub struct WordParam {
    pub left_id: u16,
    pub right_id: u16,
    pub word_cost: i16,
}

impl WordParam {
    #[inline(always)]
    pub const fn new(left_id: u16, right_id: u16, word_cost: i16) -> Self {
        Self {
            left_id,
            right_id,
            word_cost,
        }
    }
}

#[derive(Decode, Encode)]
pub struct WordParams {
    params: Vec<WordParam>,
}

impl WordParams {
    pub fn new<I>(params: I) -> Self
    where
        I: IntoIterator<Item = WordParam>,
    {
        Self {
            params: params.into_iter().collect(),
        }
    }

    #[inline(always)]
    pub fn get(&self, word_id: usize) -> WordParam {
        self.params[word_id]
    }

    #[inline(always)]
    pub fn len(&self) -> usize {
        self.params.len()
    }

    pub fn map_connection_ids(&mut self, mapper: &ConnIdMapper) {
        for p in &mut self.params {
            p.left_id = mapper.left(p.left_id);
            p.right_id = mapper.right(p.right_id);
        }
    }
}