1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
use std::collections::HashMap;
use crate::types::*;
#[derive(Clone,Debug)]
pub struct VocabValue {
pub text: String,
pub norm: NormString,
pub frequency: u32,
pub tokencount: u8,
pub lexweight: f32,
pub lexindex: u8
}
pub type VocabDecoder = Vec<VocabValue>;
pub type VocabEncoder = HashMap<String, VocabId>;
pub struct VocabParams {
pub text_column: u8,
pub freq_column: Option<u8>
}
impl Default for VocabParams {
fn default() -> Self {
Self {
text_column: 0,
freq_column: None,
}
}
}