1use std::collections::HashSet;
14use std::path::Path;
15
16use rayon::prelude::*;
17use rustc_hash::FxHashMap;
18
19use super::{
20 EncodeSegment, Encoding, Error, Result, TokenIdType,
21 hf::HuggingFaceTokenizer,
22 tiktoken,
23 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
24};
25
26fn fast_encode(encoder: &fastokens::Tokenizer, input: &str) -> Result<Encoding> {
27 let ids = encoder
28 .encode(input)
29 .map_err(|e| Error::msg(format!("Fastokens encode error: {e}")))?;
30 Ok(Encoding::Sp(ids))
31}
32
33fn fast_encode_segments(
34 encoder: &fastokens::Tokenizer,
35 segments: &[EncodeSegment<'_>],
36) -> Result<Encoding> {
37 let segments: Vec<fastokens::EncodeSegment<'_>> = segments
38 .iter()
39 .map(|segment| fastokens::EncodeSegment {
40 text: segment.text,
41 allow_special: segment.allow_special,
42 })
43 .collect();
44 let ids = encoder
45 .encode_segments(&segments)
46 .map_err(|e| Error::msg(format!("Fastokens segmented encode error: {e}")))?;
47 Ok(Encoding::Sp(ids))
48}
49
50pub struct FastTokenizer {
54 fast_encoder: fastokens::Tokenizer,
55 hf_decoder: HuggingFaceTokenizer,
56}
57
58impl FastTokenizer {
59 pub fn from_file(path: &str) -> Result<Self> {
60 let fast_encoder = fastokens::Tokenizer::from_file(Path::new(path))
61 .map_err(|e| Error::msg(format!("Error loading fastokens tokenizer: {e}")))?;
62 let hf_decoder = HuggingFaceTokenizer::from_file(path)?;
63 Ok(Self {
64 fast_encoder,
65 hf_decoder,
66 })
67 }
68}
69
70impl Encoder for FastTokenizer {
71 fn encode(&self, input: &str) -> Result<Encoding> {
72 fast_encode(&self.fast_encoder, input)
73 }
74
75 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
76 inputs.par_iter().map(|input| self.encode(input)).collect()
77 }
78
79 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
80 fast_encode_segments(&self.fast_encoder, segments)
81 }
82}
83
84impl Decoder for FastTokenizer {
85 fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
86 self.hf_decoder
87 .has_unstable_suffix(token_ids, skip_special_tokens)
88 }
89
90 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
91 self.hf_decoder.decode(token_ids, skip_special_tokens)
92 }
93}
94
95impl Tokenizer for FastTokenizer {
96 fn validate_prefix_cache(&self) -> Result<()> {
97 Ok(())
98 }
99
100 fn vocab_size(&self) -> Option<usize> {
103 self.hf_decoder.vocab_size()
104 }
105
106 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
107 self.hf_decoder.token_to_id(token)
108 }
109
110 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
111 self.hf_decoder.special_token_ids()
112 }
113
114 fn num_special_tokens_added(&self) -> Result<usize> {
115 Ok(0)
116 }
117}
118
119pub struct FastTikTokenTokenizer {
129 inner: fastokens::Tokenizer,
130 id_to_bytes: FxHashMap<u32, Vec<u8>>,
133 special_token_ids: HashSet<u32>,
134 special_tokens: Vec<String>,
135}
136
137impl FastTikTokenTokenizer {
138 pub fn from_file_auto(path: &str) -> Result<Self> {
141 let directory = Path::new(path)
142 .parent()
143 .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
144 let pattern = tiktoken::detect_bpe_pattern(directory)?;
145 let encoder = tiktoken::parse_tiktoken_file(path)?;
146 let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
147 let special_tokens = tiktoken::load_special_tokens(directory, num_base_tokens)?;
148
149 let ranks: Vec<(Vec<u8>, u32)> = encoder.into_iter().collect();
150 let mut id_to_bytes: FxHashMap<u32, Vec<u8>> = ranks
151 .iter()
152 .map(|(bytes, rank)| (*rank, bytes.clone()))
153 .collect();
154 id_to_bytes.extend(
155 special_tokens
156 .iter()
157 .map(|(content, &id)| (id, content.as_bytes().to_vec())),
158 );
159 let special_token_ids = special_tokens.values().copied().collect();
160 let special_token_strings = tiktoken::sorted_special_token_strings(&special_tokens);
161
162 let config =
163 fastokens::tiktoken::TiktokenConfig::new(pattern, special_tokens.into_iter().collect());
164 let inner = fastokens::Tokenizer::from_tiktoken_ranks(&ranks, config).map_err(|e| {
165 Error::msg(format!(
166 "Error loading fastokens tiktoken tokenizer from {path}: {e}"
167 ))
168 })?;
169 Ok(Self {
170 inner,
171 id_to_bytes,
172 special_token_ids,
173 special_tokens: special_token_strings,
174 })
175 }
176
177 pub fn special_tokens(&self) -> &[String] {
180 &self.special_tokens
181 }
182}
183
184impl Encoder for FastTikTokenTokenizer {
185 fn encode(&self, input: &str) -> Result<Encoding> {
186 fast_encode(&self.inner, input)
187 }
188
189 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
190 inputs.par_iter().map(|input| self.encode(input)).collect()
191 }
192
193 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
194 fast_encode_segments(&self.inner, segments)
195 }
196}
197
198impl Decoder for FastTikTokenTokenizer {
199 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
200 let mut bytes = Vec::new();
201 for id in token_ids {
202 if skip_special_tokens && self.special_token_ids.contains(id) {
203 continue;
204 }
205 if let Some(token) = self.id_to_bytes.get(id) {
206 bytes.extend_from_slice(token);
207 }
208 }
209 match String::from_utf8(bytes) {
210 Ok(text) => Ok(DecodeResult::Complete(text)),
211 Err(e) => Ok(DecodeResult::from_decoded(
212 String::from_utf8_lossy(e.as_bytes()).into_owned(),
213 )),
214 }
215 }
216}
217
218impl Tokenizer for FastTikTokenTokenizer {
219 fn validate_prefix_cache(&self) -> Result<()> {
220 Err(Error::msg(
221 "fastokens over tiktoken.model plain-text prefix caching is disabled pending \
222 validation of the full cache matrix",
223 ))
224 }
225
226 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
227 Ok(self.inner.token_to_id(token))
228 }
229
230 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
231 let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
232 ids.sort_unstable();
233 Ok(ids)
234 }
235
236 fn num_special_tokens_added(&self) -> Result<usize> {
237 Ok(0)
238 }
239}
240
241#[cfg(test)]
242mod tests {
243 use super::*;
244 use crate::{HuggingFaceTokenizer, TokenizerOptions};
245
246 const TOKENIZER_PATH: &str = concat!(
249 env!("CARGO_MANIFEST_DIR"),
250 "/tests/data/minimal-bpe/tokenizer.json"
251 );
252 const SEGMENTED_TOKENIZER_PATH: &str = concat!(
253 env!("CARGO_MANIFEST_DIR"),
254 "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
255 );
256
257 #[test]
258 fn byte_fallback_stream_matches_full_decode() {
259 let tokenizer: crate::Tokenizer =
260 std::sync::Arc::new(FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap()).into();
261 for (pieces, skip) in [
262 (vec!["<0x61>", "<0xF5>"], false),
263 (vec!["<0x61>", "</s>", "<0xF5>"], false),
264 (vec!["<0x61>", "</s>", "<0xF5>"], true),
265 ] {
266 let ids: Vec<_> = pieces
267 .iter()
268 .map(|piece| tokenizer.token_to_id(piece).unwrap().unwrap())
269 .collect();
270 let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
271 let mut stream = tokenizer.decode_stream(&[], skip);
272 let mut actual = String::new();
273 for id in ids {
274 actual.push_str(&stream.step(id).unwrap().unwrap_or_default());
275 }
276 actual.push_str(&stream.finish().unwrap().unwrap_or_default());
277 assert_eq!(actual, expected, "{pieces:?}, skip={skip}");
278 }
279 }
280
281 #[test]
282 fn test_fast_encode_decode_roundtrip() {
283 let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
284 let text = "Hello, world!";
289 let encoding = tokenizer.encode(text).unwrap();
290 assert!(!encoding.token_ids().is_empty());
291 let decoded: String = tokenizer.decode(encoding.token_ids(), true).unwrap().into();
292 assert!(!decoded.is_empty());
293 let enc_chars: String = text.chars().filter(|c| !c.is_whitespace()).collect();
295 let dec_chars: String = decoded.chars().filter(|c| !c.is_whitespace()).collect();
296 assert_eq!(
297 enc_chars, dec_chars,
298 "non-space characters must be preserved"
299 );
300 }
301
302 #[test]
303 fn test_fast_matches_hf_encoding() {
304 let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
305 let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
306
307 for text in &["Hello, world!", "Hello", " world", "He llo"] {
308 let fast_ids = fast.encode(text).unwrap();
309 let hf_ids = hf.encode(text).unwrap();
310 assert_eq!(
311 fast_ids.token_ids(),
312 hf_ids.token_ids(),
313 "fastokens and HuggingFace must produce identical token IDs for '{text}'"
314 );
315 }
316 }
317
318 #[test]
319 fn test_fast_batch_encode() {
320 let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
321 let inputs = &["Hello", " world", "Hello, world!"];
322 let encodings = tokenizer.encode_batch(inputs).unwrap();
323 assert_eq!(encodings.len(), inputs.len());
324 for (enc, input) in encodings.iter().zip(inputs.iter()) {
325 assert!(
326 !enc.token_ids().is_empty(),
327 "encoding for '{input}' must be non-empty"
328 );
329 }
330 }
331
332 #[test]
333 fn test_fast_segmented_encoding_preserves_trust_boundaries() {
334 let tokenizer = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
335 let upstream =
336 fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
337 .unwrap();
338 let marker = "<s>";
339
340 let trusted = tokenizer
341 .encode_segments(&[EncodeSegment::control(marker)])
342 .unwrap();
343 assert_eq!(
344 trusted.token_ids(),
345 &[upstream.token_to_id(marker).unwrap()],
346 "trusted renderer output must recognize the control token"
347 );
348
349 let ordinary = tokenizer
350 .encode_segments(&[EncodeSegment::ordinary(marker)])
351 .unwrap();
352 assert_ne!(
353 ordinary.token_ids(),
354 trusted.token_ids(),
355 "untrusted content must encode the control-token spelling as ordinary text"
356 );
357
358 let segments = [
359 EncodeSegment::ordinary("hello "),
360 EncodeSegment::control(marker),
361 EncodeSegment::ordinary(marker),
362 ];
363 let upstream_segments = [
364 fastokens::EncodeSegment::ordinary("hello "),
365 fastokens::EncodeSegment::special(marker),
366 fastokens::EncodeSegment::ordinary(marker),
367 ];
368 let actual = tokenizer.encode_segments(&segments).unwrap();
369 let expected = upstream.encode_segments(&upstream_segments).unwrap();
370 assert_eq!(actual.token_ids(), expected);
371
372 assert!(
373 tokenizer
374 .encode_segments(&[])
375 .unwrap()
376 .token_ids()
377 .is_empty()
378 );
379 }
380
381 #[test]
382 fn test_fast_with_decode_stream() {
383 use crate::Tokenizer as TokenizerWrapper;
384 use std::sync::Arc;
385
386 let tokenizer = Arc::new(FastTokenizer::from_file(TOKENIZER_PATH).unwrap());
387 let wrapper = TokenizerWrapper::from(tokenizer);
388
389 let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
391 let continuation = ", world!";
392 let cont_ids = wrapper.encode(continuation).unwrap().token_ids().to_vec();
393
394 let mut stream = wrapper.decode_stream(&prompt_ids, true);
395 let mut accumulated = String::new();
397 for id in &cont_ids {
398 if let Some(chunk) = stream.step(*id).unwrap() {
399 accumulated.push_str(&chunk);
400 }
401 }
402
403 let mut all_ids = prompt_ids.clone();
407 all_ids.extend_from_slice(&cont_ids);
408 let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
409 let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
410 let expected = &full_text[prompt_text.len()..];
411 assert_eq!(
412 accumulated, expected,
413 "streamed chunks must equal context-aware decoded continuation"
414 );
415 }
416
417 #[test]
418 fn vocabulary_metadata_forwards_to_hf_decoder() {
419 let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
420 let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
421 assert_eq!(fast.vocab_size(), hf.vocab_size());
422 assert_eq!(
423 fast.token_to_id("Hello").unwrap(),
424 hf.token_to_id("Hello").unwrap()
425 );
426 assert_eq!(
427 fast.special_token_ids().unwrap(),
428 hf.special_token_ids().unwrap()
429 );
430 }
431
432 #[test]
433 fn special_token_accounting_matches_fast_encoder() {
434 let fast = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
435 let upstream =
436 fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
437 .unwrap();
438 let hf = HuggingFaceTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
439
440 assert_eq!(hf.num_special_tokens_added().unwrap(), 1);
441 assert_eq!(fast.num_special_tokens_added().unwrap(), 0);
442 let hf_with_special_tokens = hf.with_options(TokenizerOptions {
443 add_special_tokens: true,
444 });
445
446 for text in ["hello", "hello there"] {
447 let fast_ids = fast.encode(text).unwrap();
448 assert_eq!(
449 fast_ids.token_ids(),
450 upstream.encode(text).unwrap(),
451 "FastTokenizer must match the encoder that omits the HF post-processor"
452 );
453 assert_eq!(
454 hf_with_special_tokens
455 .encode(text)
456 .unwrap()
457 .token_ids()
458 .len(),
459 fast_ids.token_ids().len() + 1,
460 "the HF post-processor must add the BOS token FastTokenizer omits"
461 );
462 }
463 }
464}
465
466#[cfg(test)]
467mod tiktoken_parity_tests {
468 use super::*;
469 use crate::TikTokenTokenizer;
470
471 const TIKTOKEN_PATH: &str = concat!(
472 env!("CARGO_MANIFEST_DIR"),
473 "/tests/data/sample-models/mock-tiktoken-bpe/tiktoken.model"
474 );
475 const CONTRACTION_PATH: &str = concat!(
478 env!("CARGO_MANIFEST_DIR"),
479 "/tests/data/sample-models/mock-tiktoken-contraction/tiktoken.model"
480 );
481
482 fn pair() -> (TikTokenTokenizer, FastTikTokenTokenizer) {
483 let reference = TikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
484 let fast = FastTikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
485 (reference, fast)
486 }
487
488 fn corpus() -> Vec<String> {
489 vec![
490 String::new(),
491 "hello world".into(),
492 "Hello, World! 123 4567 89".into(),
493 " leading and trailing ".into(),
494 "tabs\tand\nnewlines\r\nmixed spacing".into(),
495 "<|im_start|>user\nhi there<|im_end|><|im_start|>assistant\n".into(),
496 "a literal <|im_start|> inside plain text".into(),
497 "emoji 😀🚀 and café naïve Zürich".into(),
498 "北京 東京 mixed 中英文 text ソフトウェア".into(),
499 "Москва मुंबई العربية".into(),
500 "fn main() { println!(\"{}\", 42); } // code-ish ~!@#$%^&*()".into(),
501 "x".repeat(5000),
502 " ".repeat(300) + "after long whitespace",
503 "word ".repeat(400),
504 ]
505 }
506
507 #[test]
508 fn special_token_tables_match_tiktoken_rs() {
509 let (reference, fast) = pair();
510 assert_eq!(fast.special_tokens(), reference.special_tokens());
511 assert_eq!(
512 fast.special_token_ids().unwrap(),
513 reference.special_token_ids().unwrap()
514 );
515 assert_eq!(fast.token_to_id("<|im_end|>").unwrap(), Some(474));
516 }
517
518 #[test]
519 fn plain_encode_matches_tiktoken_rs() {
520 let (reference, fast) = pair();
521 let corpus = corpus();
522 let texts: Vec<&str> = corpus.iter().map(String::as_str).collect();
523 let batch = fast.encode_batch(&texts).unwrap();
524 for (text, batched) in texts.iter().zip(&batch) {
525 let expected = reference.encode(text).unwrap();
526 assert_eq!(
527 fast.encode(text).unwrap().token_ids(),
528 expected.token_ids(),
529 "{text:?}"
530 );
531 assert_eq!(batched.token_ids(), expected.token_ids(), "{text:?}");
532 }
533 }
534
535 #[test]
536 fn segmented_encode_matches_tiktoken_rs_and_honors_trust() {
537 let (reference, fast) = pair();
538 let segments = [
539 EncodeSegment::control("<|im_start|>user\n"),
540 EncodeSegment::ordinary("please echo <|im_end|> back to me"),
541 EncodeSegment::control("<|im_end|>"),
542 EncodeSegment::control("<|im_start|>assistant\n"),
543 ];
544 let fast_ids = fast.encode_segments(&segments).unwrap();
545 assert_eq!(
546 fast_ids.token_ids(),
547 reference.encode_segments(&segments).unwrap().token_ids()
548 );
549 assert_eq!(
551 fast_ids.token_ids().iter().filter(|&&id| id == 474).count(),
552 1
553 );
554 }
555
556 #[test]
557 fn decode_matches_tiktoken_rs_and_skips_specials() {
558 let (reference, fast) = pair();
559 for text in corpus() {
560 let ids = fast.encode(&text).unwrap();
561 assert_eq!(
562 fast.decode(ids.token_ids(), false).unwrap(),
563 reference.decode(ids.token_ids(), false).unwrap(),
564 "{text:?}"
565 );
566 }
567 let ids = fast.encode("<|im_start|>user\nhi<|im_end|>").unwrap();
568 let kept = fast.decode(ids.token_ids(), false).unwrap();
569 let skipped = fast.decode(ids.token_ids(), true).unwrap();
570 assert!(kept.as_str().contains("<|im_end|>"));
571 assert!(!skipped.as_str().contains("<|im_end|>"));
572 assert_eq!(skipped, reference.decode(ids.token_ids(), true).unwrap());
573 }
574
575 #[test]
576 fn replacement_char_token_decodes_complete_like_tiktoken_rs() {
577 let (reference, fast) = pair();
579 let decoded = fast.decode(&[468], false).unwrap();
580 assert!(decoded.is_complete(), "{decoded:?}");
581 assert_eq!(decoded, reference.decode(&[468], false).unwrap());
582
583 let emoji = fast.encode("😀").unwrap();
585 let cut = &emoji.token_ids()[..emoji.token_ids().len() - 1];
586 let fast_cut = fast.decode(cut, false).unwrap();
587 assert!(fast_cut.is_partial(), "{fast_cut:?}");
588 assert_eq!(fast_cut, reference.decode(cut, false).unwrap());
589
590 assert_eq!(fast.decode(&[468, 9_999_999], false).unwrap(), decoded);
592 }
593
594 #[test]
595 fn plain_text_prefix_cache_is_rejected() {
596 let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
597 let specials = fast.special_tokens().to_vec();
598 let error = crate::CachedTokenizer::new(std::sync::Arc::new(fast), specials, 1 << 20)
599 .err()
600 .expect("fastokens over tiktoken.model must not be prefix-cached")
601 .to_string();
602 assert!(error.contains("fastokens"), "{error}");
603 }
604
605 #[test]
609 fn unicode_contraction_is_consistent_across_special_token_boundary() {
610 let reference = TikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
611 let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
612 let special = "<|end_of_msg|>";
613 let suffix = " I'\u{17f}";
614 let full = format!("{special}{suffix}");
615
616 let ref_full = reference.encode(&full).unwrap().token_ids().to_vec();
617 let ref_suffix = reference.encode(suffix).unwrap().token_ids().to_vec();
618 assert_eq!(
619 ref_full[1..],
620 ref_suffix[..],
621 "tiktoken-rs must be self-consistent"
622 );
623
624 let fast_full = fast.encode(&full).unwrap().token_ids().to_vec();
625 let fast_suffix = fast.encode(suffix).unwrap().token_ids().to_vec();
626 assert_ne!(
627 fast_full, ref_full,
628 "revisit the known Unicode contraction mismatch"
629 );
630 assert_eq!(
631 fast_full[1..],
632 fast_suffix[..],
633 "splitting at this special-token boundary must preserve fastokens ids"
634 );
635 }
636}