1use crate::logits::TokenId;
7use std::collections::HashMap;
8
9pub trait Tokenizer: Send + Sync {
11 fn encode(&self, text: &str) -> Vec<TokenId>;
13
14 fn decode(&self, tokens: &[TokenId]) -> String;
16
17 fn vocab_size(&self) -> usize;
19
20 fn token_to_piece(&self, token: TokenId) -> Option<&str>;
22
23 fn piece_to_token(&self, piece: &str) -> Option<TokenId>;
25}
26
27#[derive(Debug, Clone)]
29pub struct ByteLevelTokenizer {
30 token_to_piece_map: HashMap<TokenId, String>,
31 piece_to_token_map: HashMap<String, TokenId>,
32 vocab_size: usize,
33 pub eos_token_id: TokenId,
34 pub bos_token_id: TokenId,
35}
36
37impl ByteLevelTokenizer {
38 pub fn new() -> Self {
40 let mut token_to_piece_map = HashMap::new();
41 let mut piece_to_token_map = HashMap::new();
42
43 for b in 0..=255u8 {
45 let id = b as TokenId;
46 let s = (b as char).to_string();
47 token_to_piece_map.insert(id, s.clone());
48 piece_to_token_map.insert(s, id);
49 }
50
51 let mut next_id = 256 as TokenId;
52
53 let bos_id = next_id;
55 token_to_piece_map.insert(bos_id, "<s>".to_string());
56 piece_to_token_map.insert("<s>".to_string(), bos_id);
57 next_id += 1;
58
59 let eos_id = next_id;
60 token_to_piece_map.insert(eos_id, "</s>".to_string());
61 piece_to_token_map.insert("</s>".to_string(), eos_id);
62 next_id += 1;
63
64 let subwords = [
66 " ",
67 " ",
68 " ",
69 " ",
70 "\n",
71 "\n\n",
72 "the",
73 "The",
74 "be",
75 "to",
76 "of",
77 "and",
78 "a",
79 "in",
80 "that",
81 "have",
82 "I",
83 "it",
84 "for",
85 "not",
86 "on",
87 "with",
88 "he",
89 "as",
90 "you",
91 "do",
92 "at",
93 "this",
94 "but",
95 "his",
96 "by",
97 "from",
98 "they",
99 "we",
100 "say",
101 "her",
102 "she",
103 "or",
104 "an",
105 "will",
106 "my",
107 "one",
108 "all",
109 "would",
110 "there",
111 "their",
112 "what",
113 "so",
114 "up",
115 "out",
116 "if",
117 "about",
118 "who",
119 "get",
120 "which",
121 "go",
122 "me",
123 "when",
124 "make",
125 "can",
126 "like",
127 "time",
128 "no",
129 "just",
130 "him",
131 "know",
132 "take",
133 "people",
134 "into",
135 "year",
136 "your",
137 "good",
138 "some",
139 "could",
140 "them",
141 "see",
142 "other",
143 "than",
144 "then",
145 "now",
146 "look",
147 "only",
148 "come",
149 "its",
150 "over",
151 "think",
152 "also",
153 "back",
154 "after",
155 "use",
156 "two",
157 "how",
158 "our",
159 "work",
160 "first",
161 "well",
162 "way",
163 "even",
164 "new",
165 "want",
166 "because",
167 "any",
168 "these",
169 "give",
170 "day",
171 "most",
172 "us",
173 "spec",
174 "stream",
175 "speculative",
176 "decoding",
177 "cache",
178 "token",
179 "model",
180 "el",
181 "la",
182 "los",
183 "las",
184 "un",
185 "una",
186 "unos",
187 "unas",
188 "de",
189 "en",
190 "que",
191 "es",
192 "por",
193 "para",
194 "con",
195 "no",
196 "si",
197 "su",
198 "al",
199 "lo",
200 "como",
201 "mas",
202 "más",
203 "pero",
204 "sus",
205 "le",
206 "ya",
207 "o",
208 "fue",
209 "este",
210 "ha",
211 "me",
212 "si",
213 "sin",
214 "sobre",
215 "este",
216 "ya",
217 "entre",
218 "cuando",
219 "todo",
220 "esta",
221 "ser",
222 "son",
223 "dos",
224 "también",
225 "tambien",
226 "era",
227 "muy",
228 "hasta",
229 "desde",
230 "mi",
231 "porque",
232 "cada",
233 "rust",
234 "fn",
235 "let",
236 "mut",
237 "pub",
238 "struct",
239 "impl",
240 "enum",
241 "match",
242 "async",
243 "await",
244 "return",
245 "true",
246 "false",
247 "None",
248 "Some",
249 "Ok",
250 "Err",
251 "Self",
252 "vec!",
253 ];
254
255 for &word in &subwords {
256 if !piece_to_token_map.contains_key(word) {
257 token_to_piece_map.insert(next_id, word.to_string());
258 piece_to_token_map.insert(word.to_string(), next_id);
259 next_id += 1;
260 }
261 }
262
263 let vocab_size = next_id as usize;
264
265 Self {
266 token_to_piece_map,
267 piece_to_token_map,
268 vocab_size,
269 eos_token_id: eos_id,
270 bos_token_id: bos_id,
271 }
272 }
273
274 pub fn add_token(&mut self, piece: impl Into<String>) -> TokenId {
276 let piece_str = piece.into();
277 if let Some(&id) = self.piece_to_token_map.get(&piece_str) {
278 return id;
279 }
280
281 let id = self.vocab_size as TokenId;
282 self.token_to_piece_map.insert(id, piece_str.clone());
283 self.piece_to_token_map.insert(piece_str, id);
284 self.vocab_size += 1;
285 id
286 }
287}
288
289impl Default for ByteLevelTokenizer {
290 fn default() -> Self {
291 Self::new()
292 }
293}
294
295impl Tokenizer for ByteLevelTokenizer {
296 fn encode(&self, text: &str) -> Vec<TokenId> {
297 let mut tokens = Vec::new();
298 let mut remaining = text;
299
300 while !remaining.is_empty() {
301 let mut matched_len = 0;
303 let mut matched_id = None;
304
305 let max_check = remaining.len().min(24);
307 for len in (1..=max_check).rev() {
308 if let Some(sub) = remaining.get(..len) {
309 if let Some(&id) = self.piece_to_token_map.get(sub) {
310 matched_len = len;
311 matched_id = Some(id);
312 break;
313 }
314 }
315 }
316
317 if let (Some(id), len) = (matched_id, matched_len) {
318 tokens.push(id);
319 remaining = &remaining[len..];
320 } else {
321 let b = remaining.as_bytes()[0];
323 tokens.push(b as TokenId);
324 remaining = &remaining[1..];
325 }
326 }
327
328 tokens
329 }
330
331 fn decode(&self, tokens: &[TokenId]) -> String {
332 let mut out = String::new();
333 let mut raw_bytes = Vec::new();
334
335 for &tok in tokens {
336 if let Some(piece) = self.token_to_piece_map.get(&tok) {
337 if !raw_bytes.is_empty() {
339 out.push_str(&String::from_utf8_lossy(&raw_bytes));
340 raw_bytes.clear();
341 }
342
343 if piece == "<s>" || piece == "</s>" {
345 continue;
346 }
347 out.push_str(piece);
348 } else if tok < 256 {
349 raw_bytes.push(tok as u8);
350 } else {
351 if !raw_bytes.is_empty() {
353 out.push_str(&String::from_utf8_lossy(&raw_bytes));
354 raw_bytes.clear();
355 }
356 out.push_str(&format!("<|token_{tok}|>"));
357 }
358 }
359
360 if !raw_bytes.is_empty() {
361 out.push_str(&String::from_utf8_lossy(&raw_bytes));
362 }
363
364 out
365 }
366
367 #[inline]
368 fn vocab_size(&self) -> usize {
369 self.vocab_size
370 }
371
372 #[inline]
373 fn token_to_piece(&self, token: TokenId) -> Option<&str> {
374 self.token_to_piece_map.get(&token).map(|s| s.as_str())
375 }
376
377 #[inline]
378 fn piece_to_token(&self, piece: &str) -> Option<TokenId> {
379 self.piece_to_token_map.get(piece).copied()
380 }
381}
382
383#[cfg(test)]
384mod tests {
385 use super::*;
386
387 #[test]
388 fn test_byte_level_tokenizer_roundtrip() {
389 let tokenizer = ByteLevelTokenizer::new();
390 let input = "Speculative decoding with spec-stream in Rust!";
391 let encoded = tokenizer.encode(input);
392 assert!(!encoded.is_empty());
393
394 let decoded = tokenizer.decode(&encoded);
395 assert_eq!(decoded, input);
396 }
397
398 #[test]
399 fn test_tokenizer_spanish_roundtrip() {
400 let tokenizer = ByteLevelTokenizer::new();
401 let input = "Inferencia acelerada con caché de prefijos y decodificación especulativa.";
402 let encoded = tokenizer.encode(input);
403 let decoded = tokenizer.decode(&encoded);
404 assert_eq!(decoded, input);
405 }
406
407 #[test]
408 fn test_special_tokens() {
409 let tokenizer = ByteLevelTokenizer::new();
410 let tokens = vec![tokenizer.bos_token_id, 100, tokenizer.eos_token_id];
411 let decoded = tokenizer.decode(&tokens);
412 assert!(!decoded.contains("<s>"));
414 assert!(!decoded.contains("</s>"));
415 }
416}