Skip to main content

spec_stream_core/
tokenizer.rs

1//! # Tokenizer
2//!
3//! Provides lightweight, pure-Rust tokenization and detokenization routines
4//! with byte-fallback support and subword vocabulary mapping.
5
6use crate::logits::TokenId;
7use std::collections::HashMap;
8
9/// Standard abstract interface for tokenizers in `spec-stream`.
10pub trait Tokenizer: Send + Sync {
11    /// Encodes a UTF-8 text string into a sequence of token IDs.
12    fn encode(&self, text: &str) -> Vec<TokenId>;
13
14    /// Decodes a sequence of token IDs back into a UTF-8 string.
15    fn decode(&self, tokens: &[TokenId]) -> String;
16
17    /// Total vocabulary size.
18    fn vocab_size(&self) -> usize;
19
20    /// Maps a token ID to its string piece representation, if present.
21    fn token_to_piece(&self, token: TokenId) -> Option<&str>;
22
23    /// Maps a string piece to its token ID, if present.
24    fn piece_to_token(&self, piece: &str) -> Option<TokenId>;
25}
26
27/// A robust byte-level tokenizer with subword vocabulary lookup and lossless byte-fallback.
28#[derive(Debug, Clone)]
29pub struct ByteLevelTokenizer {
30    token_to_piece_map: HashMap<TokenId, String>,
31    piece_to_token_map: HashMap<String, TokenId>,
32    vocab_size: usize,
33    pub eos_token_id: TokenId,
34    pub bos_token_id: TokenId,
35}
36
37impl ByteLevelTokenizer {
38    /// Creates a new tokenizer with common English and Spanish subwords plus a 256-byte fallback table.
39    pub fn new() -> Self {
40        let mut token_to_piece_map = HashMap::new();
41        let mut piece_to_token_map = HashMap::new();
42
43        // 0..256: Single byte fallback tokens
44        for b in 0..=255u8 {
45            let id = b as TokenId;
46            let s = (b as char).to_string();
47            token_to_piece_map.insert(id, s.clone());
48            piece_to_token_map.insert(s, id);
49        }
50
51        let mut next_id = 256 as TokenId;
52
53        // Special tokens
54        let bos_id = next_id;
55        token_to_piece_map.insert(bos_id, "<s>".to_string());
56        piece_to_token_map.insert("<s>".to_string(), bos_id);
57        next_id += 1;
58
59        let eos_id = next_id;
60        token_to_piece_map.insert(eos_id, "</s>".to_string());
61        piece_to_token_map.insert("</s>".to_string(), eos_id);
62        next_id += 1;
63
64        // Frequent subwords and words (English + Spanish + Code)
65        let subwords = [
66            " ",
67            "  ",
68            "   ",
69            "    ",
70            "\n",
71            "\n\n",
72            "the",
73            "The",
74            "be",
75            "to",
76            "of",
77            "and",
78            "a",
79            "in",
80            "that",
81            "have",
82            "I",
83            "it",
84            "for",
85            "not",
86            "on",
87            "with",
88            "he",
89            "as",
90            "you",
91            "do",
92            "at",
93            "this",
94            "but",
95            "his",
96            "by",
97            "from",
98            "they",
99            "we",
100            "say",
101            "her",
102            "she",
103            "or",
104            "an",
105            "will",
106            "my",
107            "one",
108            "all",
109            "would",
110            "there",
111            "their",
112            "what",
113            "so",
114            "up",
115            "out",
116            "if",
117            "about",
118            "who",
119            "get",
120            "which",
121            "go",
122            "me",
123            "when",
124            "make",
125            "can",
126            "like",
127            "time",
128            "no",
129            "just",
130            "him",
131            "know",
132            "take",
133            "people",
134            "into",
135            "year",
136            "your",
137            "good",
138            "some",
139            "could",
140            "them",
141            "see",
142            "other",
143            "than",
144            "then",
145            "now",
146            "look",
147            "only",
148            "come",
149            "its",
150            "over",
151            "think",
152            "also",
153            "back",
154            "after",
155            "use",
156            "two",
157            "how",
158            "our",
159            "work",
160            "first",
161            "well",
162            "way",
163            "even",
164            "new",
165            "want",
166            "because",
167            "any",
168            "these",
169            "give",
170            "day",
171            "most",
172            "us",
173            "spec",
174            "stream",
175            "speculative",
176            "decoding",
177            "cache",
178            "token",
179            "model",
180            "el",
181            "la",
182            "los",
183            "las",
184            "un",
185            "una",
186            "unos",
187            "unas",
188            "de",
189            "en",
190            "que",
191            "es",
192            "por",
193            "para",
194            "con",
195            "no",
196            "si",
197            "su",
198            "al",
199            "lo",
200            "como",
201            "mas",
202            "más",
203            "pero",
204            "sus",
205            "le",
206            "ya",
207            "o",
208            "fue",
209            "este",
210            "ha",
211            "me",
212            "si",
213            "sin",
214            "sobre",
215            "este",
216            "ya",
217            "entre",
218            "cuando",
219            "todo",
220            "esta",
221            "ser",
222            "son",
223            "dos",
224            "también",
225            "tambien",
226            "era",
227            "muy",
228            "hasta",
229            "desde",
230            "mi",
231            "porque",
232            "cada",
233            "rust",
234            "fn",
235            "let",
236            "mut",
237            "pub",
238            "struct",
239            "impl",
240            "enum",
241            "match",
242            "async",
243            "await",
244            "return",
245            "true",
246            "false",
247            "None",
248            "Some",
249            "Ok",
250            "Err",
251            "Self",
252            "vec!",
253        ];
254
255        for &word in &subwords {
256            if !piece_to_token_map.contains_key(word) {
257                token_to_piece_map.insert(next_id, word.to_string());
258                piece_to_token_map.insert(word.to_string(), next_id);
259                next_id += 1;
260            }
261        }
262
263        let vocab_size = next_id as usize;
264
265        Self {
266            token_to_piece_map,
267            piece_to_token_map,
268            vocab_size,
269            eos_token_id: eos_id,
270            bos_token_id: bos_id,
271        }
272    }
273
274    /// Registers a custom token string piece in the vocabulary.
275    pub fn add_token(&mut self, piece: impl Into<String>) -> TokenId {
276        let piece_str = piece.into();
277        if let Some(&id) = self.piece_to_token_map.get(&piece_str) {
278            return id;
279        }
280
281        let id = self.vocab_size as TokenId;
282        self.token_to_piece_map.insert(id, piece_str.clone());
283        self.piece_to_token_map.insert(piece_str, id);
284        self.vocab_size += 1;
285        id
286    }
287}
288
289impl Default for ByteLevelTokenizer {
290    fn default() -> Self {
291        Self::new()
292    }
293}
294
295impl Tokenizer for ByteLevelTokenizer {
296    fn encode(&self, text: &str) -> Vec<TokenId> {
297        let mut tokens = Vec::new();
298        let mut remaining = text;
299
300        while !remaining.is_empty() {
301            // Greedy longest matching piece search
302            let mut matched_len = 0;
303            let mut matched_id = None;
304
305            // Try matching prefixes up to 24 chars
306            let max_check = remaining.len().min(24);
307            for len in (1..=max_check).rev() {
308                if let Some(sub) = remaining.get(..len) {
309                    if let Some(&id) = self.piece_to_token_map.get(sub) {
310                        matched_len = len;
311                        matched_id = Some(id);
312                        break;
313                    }
314                }
315            }
316
317            if let (Some(id), len) = (matched_id, matched_len) {
318                tokens.push(id);
319                remaining = &remaining[len..];
320            } else {
321                // Byte fallback
322                let b = remaining.as_bytes()[0];
323                tokens.push(b as TokenId);
324                remaining = &remaining[1..];
325            }
326        }
327
328        tokens
329    }
330
331    fn decode(&self, tokens: &[TokenId]) -> String {
332        let mut out = String::new();
333        let mut raw_bytes = Vec::new();
334
335        for &tok in tokens {
336            if let Some(piece) = self.token_to_piece_map.get(&tok) {
337                // If we accumulated raw bytes, flush them to UTF-8
338                if !raw_bytes.is_empty() {
339                    out.push_str(&String::from_utf8_lossy(&raw_bytes));
340                    raw_bytes.clear();
341                }
342
343                // Skip special tags like <s> or </s>
344                if piece == "<s>" || piece == "</s>" {
345                    continue;
346                }
347                out.push_str(piece);
348            } else if tok < 256 {
349                raw_bytes.push(tok as u8);
350            } else {
351                // Unknown high token fallback
352                if !raw_bytes.is_empty() {
353                    out.push_str(&String::from_utf8_lossy(&raw_bytes));
354                    raw_bytes.clear();
355                }
356                out.push_str(&format!("<|token_{tok}|>"));
357            }
358        }
359
360        if !raw_bytes.is_empty() {
361            out.push_str(&String::from_utf8_lossy(&raw_bytes));
362        }
363
364        out
365    }
366
367    #[inline]
368    fn vocab_size(&self) -> usize {
369        self.vocab_size
370    }
371
372    #[inline]
373    fn token_to_piece(&self, token: TokenId) -> Option<&str> {
374        self.token_to_piece_map.get(&token).map(|s| s.as_str())
375    }
376
377    #[inline]
378    fn piece_to_token(&self, piece: &str) -> Option<TokenId> {
379        self.piece_to_token_map.get(piece).copied()
380    }
381}
382
383#[cfg(test)]
384mod tests {
385    use super::*;
386
387    #[test]
388    fn test_byte_level_tokenizer_roundtrip() {
389        let tokenizer = ByteLevelTokenizer::new();
390        let input = "Speculative decoding with spec-stream in Rust!";
391        let encoded = tokenizer.encode(input);
392        assert!(!encoded.is_empty());
393
394        let decoded = tokenizer.decode(&encoded);
395        assert_eq!(decoded, input);
396    }
397
398    #[test]
399    fn test_tokenizer_spanish_roundtrip() {
400        let tokenizer = ByteLevelTokenizer::new();
401        let input = "Inferencia acelerada con caché de prefijos y decodificación especulativa.";
402        let encoded = tokenizer.encode(input);
403        let decoded = tokenizer.decode(&encoded);
404        assert_eq!(decoded, input);
405    }
406
407    #[test]
408    fn test_special_tokens() {
409        let tokenizer = ByteLevelTokenizer::new();
410        let tokens = vec![tokenizer.bos_token_id, 100, tokenizer.eos_token_id];
411        let decoded = tokenizer.decode(&tokens);
412        // Special tokens <s> and </s> are stripped from clean output
413        assert!(!decoded.contains("<s>"));
414        assert!(!decoded.contains("</s>"));
415    }
416}