Skip to main content

tokenizers/models/wordpiece/
trainer.rs

1use std::collections::HashSet;
2
3use super::WordPiece;
4use crate::models::bpe::{BpeTrainer, BpeTrainerBuilder, BPE};
5use crate::tokenizer::{AddedToken, Result, Trainer};
6use ahash::AHashSet;
7use serde::{Deserialize, Serialize};
8
9/// A `WordPieceTrainerBuilder` can be used to create a `WordPieceTrainer` with a custom
10/// configuration.
11pub struct WordPieceTrainerBuilder {
12    bpe_trainer_builder: BpeTrainerBuilder,
13}
14
15impl Default for WordPieceTrainerBuilder {
16    fn default() -> Self {
17        Self {
18            bpe_trainer_builder: BpeTrainerBuilder::new().continuing_subword_prefix("##".into()),
19        }
20    }
21}
22
23impl WordPieceTrainerBuilder {
24    /// Constructs a new `WordPieceTrainerBuilder`
25    pub fn new() -> Self {
26        Self::default()
27    }
28
29    /// Set the expected minimum frequency
30    #[must_use]
31    pub fn min_frequency(mut self, frequency: u64) -> Self {
32        self.bpe_trainer_builder = self.bpe_trainer_builder.min_frequency(frequency);
33        self
34    }
35
36    /// Set the vocabulary size
37    #[must_use]
38    pub fn vocab_size(mut self, size: usize) -> Self {
39        self.bpe_trainer_builder = self.bpe_trainer_builder.vocab_size(size);
40        self
41    }
42
43    /// Set whether to show progress
44    #[must_use]
45    pub fn show_progress(mut self, show: bool) -> Self {
46        self.bpe_trainer_builder = self.bpe_trainer_builder.show_progress(show);
47        self
48    }
49
50    /// Set the special tokens
51    #[must_use]
52    pub fn special_tokens(mut self, tokens: Vec<AddedToken>) -> Self {
53        self.bpe_trainer_builder = self.bpe_trainer_builder.special_tokens(tokens);
54        self
55    }
56
57    /// Set whether to limit the alphabet
58    #[must_use]
59    pub fn limit_alphabet(mut self, limit: usize) -> Self {
60        self.bpe_trainer_builder = self.bpe_trainer_builder.limit_alphabet(limit);
61        self
62    }
63
64    /// Set the initial alphabet
65    #[must_use]
66    pub fn initial_alphabet(mut self, alphabet: HashSet<char>) -> Self {
67        self.bpe_trainer_builder = self.bpe_trainer_builder.initial_alphabet(alphabet);
68        self
69    }
70
71    /// Set the continuing_subword_prefix
72    #[must_use]
73    pub fn continuing_subword_prefix(mut self, prefix: String) -> Self {
74        self.bpe_trainer_builder = self.bpe_trainer_builder.continuing_subword_prefix(prefix);
75        self
76    }
77
78    /// Set the end_of_word_suffix
79    #[must_use]
80    pub fn end_of_word_suffix(mut self, suffix: String) -> Self {
81        self.bpe_trainer_builder = self.bpe_trainer_builder.end_of_word_suffix(suffix);
82        self
83    }
84
85    /// Constructs the final BpeTrainer
86    pub fn build(self) -> WordPieceTrainer {
87        let bpe_trainer = self.bpe_trainer_builder.build();
88        WordPieceTrainer { bpe_trainer }
89    }
90}
91
92/// Trains a `WordPiece` model.
93#[derive(Default, Clone, Deserialize, Serialize)]
94pub struct WordPieceTrainer {
95    bpe_trainer: BpeTrainer,
96}
97
98impl WordPieceTrainer {
99    pub fn min_frequency(&self) -> u64 {
100        self.bpe_trainer.min_frequency
101    }
102
103    pub fn set_min_frequency(&mut self, freq: u64) {
104        self.bpe_trainer.min_frequency = freq;
105    }
106
107    pub fn vocab_size(&self) -> usize {
108        self.bpe_trainer.vocab_size
109    }
110
111    pub fn set_vocab_size(&mut self, size: usize) {
112        self.bpe_trainer.vocab_size = size;
113    }
114
115    pub fn show_progress(&self) -> bool {
116        self.bpe_trainer.show_progress
117    }
118
119    pub fn set_show_progress(&mut self, show_progress: bool) {
120        self.bpe_trainer.show_progress = show_progress;
121    }
122
123    pub fn special_tokens(&self) -> &[AddedToken] {
124        &self.bpe_trainer.special_tokens
125    }
126
127    pub fn set_special_tokens(&mut self, special_tokens: Vec<AddedToken>) {
128        self.bpe_trainer.special_tokens = special_tokens;
129    }
130
131    pub fn limit_alphabet(&self) -> Option<usize> {
132        self.bpe_trainer.limit_alphabet
133    }
134
135    pub fn set_limit_alphabet(&mut self, limit: Option<usize>) {
136        self.bpe_trainer.limit_alphabet = limit;
137    }
138
139    pub fn initial_alphabet(&self) -> &AHashSet<char> {
140        &self.bpe_trainer.initial_alphabet
141    }
142
143    pub fn set_initial_alphabet(&mut self, alphabet: HashSet<char>) {
144        let mut initial_alphabet = AHashSet::with_capacity(alphabet.len());
145        initial_alphabet.extend(alphabet);
146        self.bpe_trainer.initial_alphabet = initial_alphabet;
147    }
148
149    pub fn continuing_subword_prefix(&self) -> &Option<String> {
150        &self.bpe_trainer.continuing_subword_prefix
151    }
152
153    pub fn set_continuing_subword_prefix(&mut self, prefix: Option<String>) {
154        self.bpe_trainer.continuing_subword_prefix = prefix;
155    }
156
157    pub fn end_of_word_suffix(&self) -> &Option<String> {
158        &self.bpe_trainer.end_of_word_suffix
159    }
160
161    pub fn set_end_of_word_suffix(&mut self, suffix: Option<String>) {
162        self.bpe_trainer.end_of_word_suffix = suffix;
163    }
164
165    pub fn builder() -> WordPieceTrainerBuilder {
166        WordPieceTrainerBuilder::default()
167    }
168
169    pub fn train(&self, model: &mut WordPiece) -> Result<Vec<AddedToken>> {
170        let mut bpe = BPE::default();
171        let special_tokens = self.bpe_trainer.train(&mut bpe)?;
172        let new_wordpiece = WordPiece::from_bpe(&bpe);
173
174        // Transfer the vocab
175        model.vocab = new_wordpiece.vocab;
176        model.vocab_r = new_wordpiece.vocab_r;
177        // The continuing_subword_prefix is the only other option to be overridden by the trainer
178        model.continuing_subword_prefix = new_wordpiece.continuing_subword_prefix;
179
180        Ok(special_tokens)
181    }
182}
183
184impl Trainer for WordPieceTrainer {
185    type Model = WordPiece;
186
187    fn train(&self, model: &mut WordPiece) -> Result<Vec<AddedToken>> {
188        self.train(model)
189    }
190
191    fn should_show_progress(&self) -> bool {
192        self.bpe_trainer.should_show_progress()
193    }
194
195    fn feed<I, S, F>(&mut self, iterator: I, process: F) -> Result<()>
196    where
197        I: Iterator<Item = S> + Send,
198        S: AsRef<str> + Send,
199        F: Fn(&str) -> Result<Vec<String>> + Sync,
200    {
201        self.bpe_trainer.feed(iterator, process)
202    }
203}