tokenizers/models/wordpiece/
trainer.rs1use std::collections::HashSet;
2
3use super::WordPiece;
4use crate::models::bpe::{BpeTrainer, BpeTrainerBuilder, BPE};
5use crate::tokenizer::{AddedToken, Result, Trainer};
6use ahash::AHashSet;
7use serde::{Deserialize, Serialize};
8
9pub struct WordPieceTrainerBuilder {
12 bpe_trainer_builder: BpeTrainerBuilder,
13}
14
15impl Default for WordPieceTrainerBuilder {
16 fn default() -> Self {
17 Self {
18 bpe_trainer_builder: BpeTrainerBuilder::new().continuing_subword_prefix("##".into()),
19 }
20 }
21}
22
23impl WordPieceTrainerBuilder {
24 pub fn new() -> Self {
26 Self::default()
27 }
28
29 #[must_use]
31 pub fn min_frequency(mut self, frequency: u64) -> Self {
32 self.bpe_trainer_builder = self.bpe_trainer_builder.min_frequency(frequency);
33 self
34 }
35
36 #[must_use]
38 pub fn vocab_size(mut self, size: usize) -> Self {
39 self.bpe_trainer_builder = self.bpe_trainer_builder.vocab_size(size);
40 self
41 }
42
43 #[must_use]
45 pub fn show_progress(mut self, show: bool) -> Self {
46 self.bpe_trainer_builder = self.bpe_trainer_builder.show_progress(show);
47 self
48 }
49
50 #[must_use]
52 pub fn special_tokens(mut self, tokens: Vec<AddedToken>) -> Self {
53 self.bpe_trainer_builder = self.bpe_trainer_builder.special_tokens(tokens);
54 self
55 }
56
57 #[must_use]
59 pub fn limit_alphabet(mut self, limit: usize) -> Self {
60 self.bpe_trainer_builder = self.bpe_trainer_builder.limit_alphabet(limit);
61 self
62 }
63
64 #[must_use]
66 pub fn initial_alphabet(mut self, alphabet: HashSet<char>) -> Self {
67 self.bpe_trainer_builder = self.bpe_trainer_builder.initial_alphabet(alphabet);
68 self
69 }
70
71 #[must_use]
73 pub fn continuing_subword_prefix(mut self, prefix: String) -> Self {
74 self.bpe_trainer_builder = self.bpe_trainer_builder.continuing_subword_prefix(prefix);
75 self
76 }
77
78 #[must_use]
80 pub fn end_of_word_suffix(mut self, suffix: String) -> Self {
81 self.bpe_trainer_builder = self.bpe_trainer_builder.end_of_word_suffix(suffix);
82 self
83 }
84
85 pub fn build(self) -> WordPieceTrainer {
87 let bpe_trainer = self.bpe_trainer_builder.build();
88 WordPieceTrainer { bpe_trainer }
89 }
90}
91
92#[derive(Default, Clone, Deserialize, Serialize)]
94pub struct WordPieceTrainer {
95 bpe_trainer: BpeTrainer,
96}
97
98impl WordPieceTrainer {
99 pub fn min_frequency(&self) -> u64 {
100 self.bpe_trainer.min_frequency
101 }
102
103 pub fn set_min_frequency(&mut self, freq: u64) {
104 self.bpe_trainer.min_frequency = freq;
105 }
106
107 pub fn vocab_size(&self) -> usize {
108 self.bpe_trainer.vocab_size
109 }
110
111 pub fn set_vocab_size(&mut self, size: usize) {
112 self.bpe_trainer.vocab_size = size;
113 }
114
115 pub fn show_progress(&self) -> bool {
116 self.bpe_trainer.show_progress
117 }
118
119 pub fn set_show_progress(&mut self, show_progress: bool) {
120 self.bpe_trainer.show_progress = show_progress;
121 }
122
123 pub fn special_tokens(&self) -> &[AddedToken] {
124 &self.bpe_trainer.special_tokens
125 }
126
127 pub fn set_special_tokens(&mut self, special_tokens: Vec<AddedToken>) {
128 self.bpe_trainer.special_tokens = special_tokens;
129 }
130
131 pub fn limit_alphabet(&self) -> Option<usize> {
132 self.bpe_trainer.limit_alphabet
133 }
134
135 pub fn set_limit_alphabet(&mut self, limit: Option<usize>) {
136 self.bpe_trainer.limit_alphabet = limit;
137 }
138
139 pub fn initial_alphabet(&self) -> &AHashSet<char> {
140 &self.bpe_trainer.initial_alphabet
141 }
142
143 pub fn set_initial_alphabet(&mut self, alphabet: HashSet<char>) {
144 let mut initial_alphabet = AHashSet::with_capacity(alphabet.len());
145 initial_alphabet.extend(alphabet);
146 self.bpe_trainer.initial_alphabet = initial_alphabet;
147 }
148
149 pub fn continuing_subword_prefix(&self) -> &Option<String> {
150 &self.bpe_trainer.continuing_subword_prefix
151 }
152
153 pub fn set_continuing_subword_prefix(&mut self, prefix: Option<String>) {
154 self.bpe_trainer.continuing_subword_prefix = prefix;
155 }
156
157 pub fn end_of_word_suffix(&self) -> &Option<String> {
158 &self.bpe_trainer.end_of_word_suffix
159 }
160
161 pub fn set_end_of_word_suffix(&mut self, suffix: Option<String>) {
162 self.bpe_trainer.end_of_word_suffix = suffix;
163 }
164
165 pub fn builder() -> WordPieceTrainerBuilder {
166 WordPieceTrainerBuilder::default()
167 }
168
169 pub fn train(&self, model: &mut WordPiece) -> Result<Vec<AddedToken>> {
170 let mut bpe = BPE::default();
171 let special_tokens = self.bpe_trainer.train(&mut bpe)?;
172 let new_wordpiece = WordPiece::from_bpe(&bpe);
173
174 model.vocab = new_wordpiece.vocab;
176 model.vocab_r = new_wordpiece.vocab_r;
177 model.continuing_subword_prefix = new_wordpiece.continuing_subword_prefix;
179
180 Ok(special_tokens)
181 }
182}
183
184impl Trainer for WordPieceTrainer {
185 type Model = WordPiece;
186
187 fn train(&self, model: &mut WordPiece) -> Result<Vec<AddedToken>> {
188 self.train(model)
189 }
190
191 fn should_show_progress(&self) -> bool {
192 self.bpe_trainer.should_show_progress()
193 }
194
195 fn feed<I, S, F>(&mut self, iterator: I, process: F) -> Result<()>
196 where
197 I: Iterator<Item = S> + Send,
198 S: AsRef<str> + Send,
199 F: Fn(&str) -> Result<Vec<String>> + Sync,
200 {
201 self.bpe_trainer.feed(iterator, process)
202 }
203}