1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
use crate::Vocab;
use std::collections::HashMap;
use crate::preprocessing::vocab::base_vocab::swap_key_values;
use std::process;
use std::fs::File;
use std::io::BufReader;
pub struct MarianVocab {
pub values: HashMap<String, i64>,
pub indices: HashMap<i64, String>,
pub unknown_value: &'static str,
pub special_values: HashMap<String, i64>,
pub special_indices: HashMap<i64, String>,
}
impl MarianVocab {
pub fn pad_value() -> &'static str { "<pad>" }
pub fn eos_value() -> &'static str { "</s>" }
}
impl Vocab for MarianVocab {
fn unknown_value() -> &'static str { "<unk>" }
fn get_unknown_value(&self) -> &'static str { "<unk>" }
fn values(&self) -> &HashMap<String, i64> {
&self.values
}
fn indices(&self) -> &HashMap<i64, String> { &self.indices }
fn special_values(&self) -> &HashMap<String, i64> { &self.special_values }
fn special_indices(&self) -> &HashMap<i64, String> { &self.special_indices }
fn from_file(path: &str) -> MarianVocab {
let f = File::open(path).expect("Could not open vocabulary file.");
let br = BufReader::new(f);
let values: HashMap<String, i64> = serde_json::from_reader(br).expect("could not parse vocabulary");
let mut special_values = HashMap::new();
let unknown_value = MarianVocab::unknown_value();
MarianVocab::_register_as_special_value(unknown_value, &values, &mut special_values);
let pad_value = MarianVocab::pad_value();
MarianVocab::_register_as_special_value(pad_value, &values, &mut special_values);
let eos_value = MarianVocab::eos_value();
MarianVocab::_register_as_special_value(eos_value, &values, &mut special_values);
let indices = swap_key_values(&values);
let special_indices = swap_key_values(&special_values);
MarianVocab { values, indices, unknown_value, special_values, special_indices }
}
fn token_to_id(&self, token: &str) -> i64 {
match self._token_to_id(token, &self.values, &self.special_values, &self.unknown_value) {
Ok(index) => index,
Err(err) => {
println!("{}", err);
process::exit(1);
}
}
}
fn id_to_token(&self, id: &i64) -> String {
match self._id_to_token(&id, &self.indices, &self.special_indices, &self.unknown_value) {
Ok(token) => token,
Err(err) => {
println!("{}", err);
process::exit(1);
}
}
}
}