1use std::collections::HashSet;
14use std::path::Path;
15
16use rayon::prelude::*;
17use rustc_hash::FxHashMap;
18
19use super::{
20 EncodeSegment, Encoding, Error, Result, TokenIdType,
21 hf::HuggingFaceTokenizer,
22 tiktoken,
23 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
24};
25
26fn fast_encode(encoder: &fastokens::Tokenizer, input: &str) -> Result<Encoding> {
27 let ids = encoder
28 .encode(input)
29 .map_err(|e| Error::msg(format!("Fastokens encode error: {e}")))?;
30 Ok(Encoding::Sp(ids))
31}
32
33fn fast_encode_segments(
34 encoder: &fastokens::Tokenizer,
35 segments: &[EncodeSegment<'_>],
36) -> Result<Encoding> {
37 let segments: Vec<fastokens::EncodeSegment<'_>> = segments
38 .iter()
39 .map(|segment| fastokens::EncodeSegment {
40 text: segment.text,
41 allow_special: segment.allow_special,
42 })
43 .collect();
44 let ids = encoder
45 .encode_segments(&segments)
46 .map_err(|e| Error::msg(format!("Fastokens segmented encode error: {e}")))?;
47 Ok(Encoding::Sp(ids))
48}
49
50pub struct FastTokenizer {
54 fast_encoder: fastokens::Tokenizer,
55 hf_decoder: HuggingFaceTokenizer,
56}
57
58impl FastTokenizer {
59 pub fn from_file(path: &str) -> Result<Self> {
60 let fast_encoder = fastokens::Tokenizer::from_file(Path::new(path))
61 .map_err(|e| Error::msg(format!("Error loading fastokens tokenizer: {e}")))?;
62 let hf_decoder = HuggingFaceTokenizer::from_file(path)?;
63 Ok(Self {
64 fast_encoder,
65 hf_decoder,
66 })
67 }
68}
69
70impl Encoder for FastTokenizer {
71 fn encode(&self, input: &str) -> Result<Encoding> {
72 fast_encode(&self.fast_encoder, input)
73 }
74
75 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
76 inputs.par_iter().map(|input| self.encode(input)).collect()
77 }
78
79 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
80 fast_encode_segments(&self.fast_encoder, segments)
81 }
82}
83
84impl Decoder for FastTokenizer {
85 fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
86 self.hf_decoder
87 .has_unstable_suffix(token_ids, skip_special_tokens)
88 }
89
90 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
91 self.hf_decoder.decode(token_ids, skip_special_tokens)
92 }
93}
94
95impl Tokenizer for FastTokenizer {
96 fn validate_prefix_cache(&self) -> Result<()> {
97 Ok(())
98 }
99
100 fn vocab_size(&self) -> Option<usize> {
103 self.hf_decoder.vocab_size()
104 }
105
106 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
107 self.hf_decoder.token_to_id(token)
108 }
109
110 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
111 self.hf_decoder.special_token_ids()
112 }
113
114 fn num_special_tokens_added(&self) -> Result<usize> {
115 Ok(0)
116 }
117}
118
119pub struct FastTikTokenTokenizer {
129 inner: fastokens::Tokenizer,
130 id_to_bytes: FxHashMap<u32, Vec<u8>>,
133 special_token_ids: HashSet<u32>,
134 special_tokens: Vec<String>,
135}
136
137impl FastTikTokenTokenizer {
138 pub fn from_file_auto(path: &str) -> Result<Self> {
141 let directory = Path::new(path)
142 .parent()
143 .ok_or_else(|| Error::msg("Cannot determine parent directory of tiktoken file"))?;
144 let pattern = tiktoken::detect_bpe_pattern(directory)?;
145 let encoder = tiktoken::parse_tiktoken_file(path)?;
146 let num_base_tokens = encoder.values().max().map_or(0, |&m| m + 1) as usize;
147 let tiktoken::AddedTokens { tokens, skip_ids } =
148 tiktoken::load_added_tokens(directory, num_base_tokens)?;
149
150 let ranks: Vec<(Vec<u8>, u32)> = encoder.into_iter().collect();
151 let mut id_to_bytes: FxHashMap<u32, Vec<u8>> = ranks
152 .iter()
153 .map(|(bytes, rank)| (*rank, bytes.clone()))
154 .collect();
155 id_to_bytes.extend(
156 tokens
157 .iter()
158 .map(|(content, &id)| (id, content.as_bytes().to_vec())),
159 );
160 let special_token_strings = tiktoken::sorted_special_token_strings(&tokens);
161
162 let config =
163 fastokens::tiktoken::TiktokenConfig::new(pattern, tokens.into_iter().collect());
164 let inner = fastokens::Tokenizer::from_tiktoken_ranks(&ranks, config).map_err(|e| {
165 Error::msg(format!(
166 "Error loading fastokens tiktoken tokenizer from {path}: {e}"
167 ))
168 })?;
169 Ok(Self {
170 inner,
171 id_to_bytes,
172 special_token_ids: skip_ids,
173 special_tokens: special_token_strings,
174 })
175 }
176
177 pub fn special_tokens(&self) -> &[String] {
181 &self.special_tokens
182 }
183}
184
185impl Encoder for FastTikTokenTokenizer {
186 fn encode(&self, input: &str) -> Result<Encoding> {
187 fast_encode(&self.inner, input)
188 }
189
190 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
191 inputs.par_iter().map(|input| self.encode(input)).collect()
192 }
193
194 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
195 fast_encode_segments(&self.inner, segments)
196 }
197}
198
199impl Decoder for FastTikTokenTokenizer {
200 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
201 let mut bytes = Vec::new();
202 for id in token_ids {
203 if skip_special_tokens && self.special_token_ids.contains(id) {
204 continue;
205 }
206 if let Some(token) = self.id_to_bytes.get(id) {
207 bytes.extend_from_slice(token);
208 }
209 }
210 match String::from_utf8(bytes) {
211 Ok(text) => Ok(DecodeResult::Complete(text)),
212 Err(e) => Ok(DecodeResult::from_decoded(
213 String::from_utf8_lossy(e.as_bytes()).into_owned(),
214 )),
215 }
216 }
217}
218
219impl Tokenizer for FastTikTokenTokenizer {
220 fn validate_prefix_cache(&self) -> Result<()> {
221 Err(Error::msg(
222 "fastokens over tiktoken.model plain-text prefix caching is disabled pending \
223 validation of the full cache matrix",
224 ))
225 }
226
227 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
228 Ok(self.inner.token_to_id(token))
229 }
230
231 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
232 let mut ids: Vec<TokenIdType> = self.special_token_ids.iter().copied().collect();
233 ids.sort_unstable();
234 Ok(ids)
235 }
236
237 fn num_special_tokens_added(&self) -> Result<usize> {
238 Ok(0)
239 }
240}
241
242#[cfg(test)]
243mod tests {
244 use super::*;
245 use crate::{HuggingFaceTokenizer, TokenizerOptions};
246
247 const TOKENIZER_PATH: &str = concat!(
250 env!("CARGO_MANIFEST_DIR"),
251 "/tests/data/minimal-bpe/tokenizer.json"
252 );
253 const SEGMENTED_TOKENIZER_PATH: &str = concat!(
254 env!("CARGO_MANIFEST_DIR"),
255 "/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
256 );
257
258 #[test]
259 fn byte_fallback_stream_matches_full_decode() {
260 let tokenizer: crate::Tokenizer =
261 std::sync::Arc::new(FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap()).into();
262 for (pieces, skip) in [
263 (vec!["<0x61>", "<0xF5>"], false),
264 (vec!["<0x61>", "</s>", "<0xF5>"], false),
265 (vec!["<0x61>", "</s>", "<0xF5>"], true),
266 ] {
267 let ids: Vec<_> = pieces
268 .iter()
269 .map(|piece| tokenizer.token_to_id(piece).unwrap().unwrap())
270 .collect();
271 let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
272 let mut stream = tokenizer.decode_stream(&[], skip);
273 let mut actual = String::new();
274 for id in ids {
275 actual.push_str(&stream.step(id).unwrap().unwrap_or_default());
276 }
277 actual.push_str(&stream.finish().unwrap().unwrap_or_default());
278 assert_eq!(actual, expected, "{pieces:?}, skip={skip}");
279 }
280 }
281
282 #[test]
283 fn test_fast_encode_decode_roundtrip() {
284 let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
285 let text = "Hello, world!";
290 let encoding = tokenizer.encode(text).unwrap();
291 assert!(!encoding.token_ids().is_empty());
292 let decoded: String = tokenizer.decode(encoding.token_ids(), true).unwrap().into();
293 assert!(!decoded.is_empty());
294 let enc_chars: String = text.chars().filter(|c| !c.is_whitespace()).collect();
296 let dec_chars: String = decoded.chars().filter(|c| !c.is_whitespace()).collect();
297 assert_eq!(
298 enc_chars, dec_chars,
299 "non-space characters must be preserved"
300 );
301 }
302
303 #[test]
304 fn test_fast_matches_hf_encoding() {
305 let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
306 let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
307
308 for text in &["Hello, world!", "Hello", " world", "He llo"] {
309 let fast_ids = fast.encode(text).unwrap();
310 let hf_ids = hf.encode(text).unwrap();
311 assert_eq!(
312 fast_ids.token_ids(),
313 hf_ids.token_ids(),
314 "fastokens and HuggingFace must produce identical token IDs for '{text}'"
315 );
316 }
317 }
318
319 #[test]
320 fn test_fast_batch_encode() {
321 let tokenizer = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
322 let inputs = &["Hello", " world", "Hello, world!"];
323 let encodings = tokenizer.encode_batch(inputs).unwrap();
324 assert_eq!(encodings.len(), inputs.len());
325 for (enc, input) in encodings.iter().zip(inputs.iter()) {
326 assert!(
327 !enc.token_ids().is_empty(),
328 "encoding for '{input}' must be non-empty"
329 );
330 }
331 }
332
333 #[test]
334 fn test_fast_segmented_encoding_preserves_trust_boundaries() {
335 let tokenizer = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
336 let upstream =
337 fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
338 .unwrap();
339 let marker = "<s>";
340
341 let trusted = tokenizer
342 .encode_segments(&[EncodeSegment::control(marker)])
343 .unwrap();
344 assert_eq!(
345 trusted.token_ids(),
346 &[upstream.token_to_id(marker).unwrap()],
347 "trusted renderer output must recognize the control token"
348 );
349
350 let ordinary = tokenizer
351 .encode_segments(&[EncodeSegment::ordinary(marker)])
352 .unwrap();
353 assert_ne!(
354 ordinary.token_ids(),
355 trusted.token_ids(),
356 "untrusted content must encode the control-token spelling as ordinary text"
357 );
358
359 let segments = [
360 EncodeSegment::ordinary("hello "),
361 EncodeSegment::control(marker),
362 EncodeSegment::ordinary(marker),
363 ];
364 let upstream_segments = [
365 fastokens::EncodeSegment::ordinary("hello "),
366 fastokens::EncodeSegment::special(marker),
367 fastokens::EncodeSegment::ordinary(marker),
368 ];
369 let actual = tokenizer.encode_segments(&segments).unwrap();
370 let expected = upstream.encode_segments(&upstream_segments).unwrap();
371 assert_eq!(actual.token_ids(), expected);
372
373 assert!(
374 tokenizer
375 .encode_segments(&[])
376 .unwrap()
377 .token_ids()
378 .is_empty()
379 );
380 }
381
382 #[test]
383 fn test_fast_with_decode_stream() {
384 use crate::Tokenizer as TokenizerWrapper;
385 use std::sync::Arc;
386
387 let tokenizer = Arc::new(FastTokenizer::from_file(TOKENIZER_PATH).unwrap());
388 let wrapper = TokenizerWrapper::from(tokenizer);
389
390 let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
392 let continuation = ", world!";
393 let cont_ids = wrapper.encode(continuation).unwrap().token_ids().to_vec();
394
395 let mut stream = wrapper.decode_stream(&prompt_ids, true);
396 let mut accumulated = String::new();
398 for id in &cont_ids {
399 if let Some(chunk) = stream.step(*id).unwrap() {
400 accumulated.push_str(&chunk);
401 }
402 }
403
404 let mut all_ids = prompt_ids.clone();
408 all_ids.extend_from_slice(&cont_ids);
409 let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
410 let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
411 let expected = &full_text[prompt_text.len()..];
412 assert_eq!(
413 accumulated, expected,
414 "streamed chunks must equal context-aware decoded continuation"
415 );
416 }
417
418 #[test]
419 fn vocabulary_metadata_forwards_to_hf_decoder() {
420 let fast = FastTokenizer::from_file(TOKENIZER_PATH).unwrap();
421 let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
422 assert_eq!(fast.vocab_size(), hf.vocab_size());
423 assert_eq!(
424 fast.token_to_id("Hello").unwrap(),
425 hf.token_to_id("Hello").unwrap()
426 );
427 assert_eq!(
428 fast.special_token_ids().unwrap(),
429 hf.special_token_ids().unwrap()
430 );
431 }
432
433 #[test]
434 fn special_token_accounting_matches_fast_encoder() {
435 let fast = FastTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
436 let upstream =
437 fastokens::Tokenizer::from_file(std::path::Path::new(SEGMENTED_TOKENIZER_PATH))
438 .unwrap();
439 let hf = HuggingFaceTokenizer::from_file(SEGMENTED_TOKENIZER_PATH).unwrap();
440
441 assert_eq!(hf.num_special_tokens_added().unwrap(), 1);
442 assert_eq!(fast.num_special_tokens_added().unwrap(), 0);
443 let hf_with_special_tokens = hf.with_options(TokenizerOptions {
444 add_special_tokens: true,
445 });
446
447 for text in ["hello", "hello there"] {
448 let fast_ids = fast.encode(text).unwrap();
449 assert_eq!(
450 fast_ids.token_ids(),
451 upstream.encode(text).unwrap(),
452 "FastTokenizer must match the encoder that omits the HF post-processor"
453 );
454 assert_eq!(
455 hf_with_special_tokens
456 .encode(text)
457 .unwrap()
458 .token_ids()
459 .len(),
460 fast_ids.token_ids().len() + 1,
461 "the HF post-processor must add the BOS token FastTokenizer omits"
462 );
463 }
464 }
465}
466
467#[cfg(test)]
468mod tiktoken_parity_tests {
469 use super::*;
470 use crate::TikTokenTokenizer;
471
472 const TIKTOKEN_PATH: &str = concat!(
473 env!("CARGO_MANIFEST_DIR"),
474 "/tests/data/sample-models/mock-tiktoken-bpe/tiktoken.model"
475 );
476 const CONTRACTION_PATH: &str = concat!(
479 env!("CARGO_MANIFEST_DIR"),
480 "/tests/data/sample-models/mock-tiktoken-contraction/tiktoken.model"
481 );
482
483 fn pair() -> (TikTokenTokenizer, FastTikTokenTokenizer) {
484 let reference = TikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
485 let fast = FastTikTokenTokenizer::from_file_auto(TIKTOKEN_PATH).unwrap();
486 (reference, fast)
487 }
488
489 fn corpus() -> Vec<String> {
490 vec![
491 String::new(),
492 "hello world".into(),
493 "Hello, World! 123 4567 89".into(),
494 " leading and trailing ".into(),
495 "tabs\tand\nnewlines\r\nmixed spacing".into(),
496 "<|im_start|>user\nhi there<|im_end|><|im_start|>assistant\n".into(),
497 "a literal <|im_start|> inside plain text".into(),
498 "emoji 😀🚀 and café naïve Zürich".into(),
499 "北京 東京 mixed 中英文 text ソフトウェア".into(),
500 "Москва मुंबई العربية".into(),
501 "fn main() { println!(\"{}\", 42); } // code-ish ~!@#$%^&*()".into(),
502 "x".repeat(5000),
503 " ".repeat(300) + "after long whitespace",
504 "word ".repeat(400),
505 ]
506 }
507
508 #[test]
509 fn special_token_tables_match_tiktoken_rs() {
510 let (reference, fast) = pair();
511 assert_eq!(fast.special_tokens(), reference.special_tokens());
512 assert_eq!(
513 fast.special_token_ids().unwrap(),
514 reference.special_token_ids().unwrap()
515 );
516 assert_eq!(fast.token_to_id("<|im_end|>").unwrap(), Some(474));
517 }
518
519 #[test]
520 fn non_special_added_tokens_survive_skip_special_tokens() {
521 let source = Path::new(TIKTOKEN_PATH).parent().unwrap();
522 let dir = tempfile::tempdir().unwrap();
523 for file in ["tiktoken.model", "config.json"] {
524 std::fs::copy(source.join(file), dir.path().join(file)).unwrap();
525 }
526 let mut config: serde_json::Value = serde_json::from_str(
527 &std::fs::read_to_string(source.join("tokenizer_config.json")).unwrap(),
528 )
529 .unwrap();
530 config["added_tokens_decoder"]["475"] =
531 serde_json::json!({"content": "<|open|>", "special": false});
532 std::fs::write(dir.path().join("tokenizer_config.json"), config.to_string()).unwrap();
533 let path = dir.path().join("tiktoken.model");
534 let path = path.to_str().unwrap();
535 let reference = TikTokenTokenizer::from_file_auto(path).unwrap();
536 let fast = FastTikTokenTokenizer::from_file_auto(path).unwrap();
537
538 let special_ids = fast.special_token_ids().unwrap();
539 assert_eq!(special_ids, reference.special_token_ids().unwrap());
540 assert!(special_ids.contains(&474));
541 assert!(!special_ids.contains(&475));
542
543 let ids = fast.encode("<|open|>hello<|im_end|>").unwrap();
544 assert_eq!(
545 ids.token_ids(),
546 reference
547 .encode("<|open|>hello<|im_end|>")
548 .unwrap()
549 .token_ids()
550 );
551 assert_eq!(ids.token_ids().first(), Some(&475));
552 let fast_text: String = fast.decode(ids.token_ids(), true).unwrap().into();
553 let reference_text: String = reference.decode(ids.token_ids(), true).unwrap().into();
554 assert_eq!(fast_text, "<|open|>hello");
555 assert_eq!(reference_text, "<|open|>hello");
556 }
557
558 #[test]
559 fn plain_encode_matches_tiktoken_rs() {
560 let (reference, fast) = pair();
561 let corpus = corpus();
562 let texts: Vec<&str> = corpus.iter().map(String::as_str).collect();
563 let batch = fast.encode_batch(&texts).unwrap();
564 for (text, batched) in texts.iter().zip(&batch) {
565 let expected = reference.encode(text).unwrap();
566 assert_eq!(
567 fast.encode(text).unwrap().token_ids(),
568 expected.token_ids(),
569 "{text:?}"
570 );
571 assert_eq!(batched.token_ids(), expected.token_ids(), "{text:?}");
572 }
573 }
574
575 #[test]
576 fn segmented_encode_matches_tiktoken_rs_and_honors_trust() {
577 let (reference, fast) = pair();
578 let segments = [
579 EncodeSegment::control("<|im_start|>user\n"),
580 EncodeSegment::ordinary("please echo <|im_end|> back to me"),
581 EncodeSegment::control("<|im_end|>"),
582 EncodeSegment::control("<|im_start|>assistant\n"),
583 ];
584 let fast_ids = fast.encode_segments(&segments).unwrap();
585 assert_eq!(
586 fast_ids.token_ids(),
587 reference.encode_segments(&segments).unwrap().token_ids()
588 );
589 assert_eq!(
591 fast_ids.token_ids().iter().filter(|&&id| id == 474).count(),
592 1
593 );
594 }
595
596 #[test]
597 fn decode_matches_tiktoken_rs_and_skips_specials() {
598 let (reference, fast) = pair();
599 for text in corpus() {
600 let ids = fast.encode(&text).unwrap();
601 assert_eq!(
602 fast.decode(ids.token_ids(), false).unwrap(),
603 reference.decode(ids.token_ids(), false).unwrap(),
604 "{text:?}"
605 );
606 }
607 let ids = fast.encode("<|im_start|>user\nhi<|im_end|>").unwrap();
608 let kept = fast.decode(ids.token_ids(), false).unwrap();
609 let skipped = fast.decode(ids.token_ids(), true).unwrap();
610 assert!(kept.as_str().contains("<|im_end|>"));
611 assert!(!skipped.as_str().contains("<|im_end|>"));
612 assert_eq!(skipped, reference.decode(ids.token_ids(), true).unwrap());
613 }
614
615 #[test]
616 fn replacement_char_token_decodes_complete_like_tiktoken_rs() {
617 let (reference, fast) = pair();
619 let decoded = fast.decode(&[468], false).unwrap();
620 assert!(decoded.is_complete(), "{decoded:?}");
621 assert_eq!(decoded, reference.decode(&[468], false).unwrap());
622
623 let emoji = fast.encode("😀").unwrap();
625 let cut = &emoji.token_ids()[..emoji.token_ids().len() - 1];
626 let fast_cut = fast.decode(cut, false).unwrap();
627 assert!(fast_cut.is_partial(), "{fast_cut:?}");
628 assert_eq!(fast_cut, reference.decode(cut, false).unwrap());
629
630 assert_eq!(fast.decode(&[468, 9_999_999], false).unwrap(), decoded);
632 }
633
634 #[test]
635 fn plain_text_prefix_cache_is_rejected() {
636 let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
637 let specials = fast.special_tokens().to_vec();
638 let error = crate::CachedTokenizer::new(std::sync::Arc::new(fast), specials, 1 << 20)
639 .err()
640 .expect("fastokens over tiktoken.model must not be prefix-cached")
641 .to_string();
642 assert!(error.contains("fastokens"), "{error}");
643 }
644
645 #[test]
649 fn unicode_contraction_is_consistent_across_special_token_boundary() {
650 let reference = TikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
651 let fast = FastTikTokenTokenizer::from_file_auto(CONTRACTION_PATH).unwrap();
652 let special = "<|end_of_msg|>";
653 let suffix = " I'\u{17f}";
654 let full = format!("{special}{suffix}");
655
656 let ref_full = reference.encode(&full).unwrap().token_ids().to_vec();
657 let ref_suffix = reference.encode(suffix).unwrap().token_ids().to_vec();
658 assert_eq!(
659 ref_full[1..],
660 ref_suffix[..],
661 "tiktoken-rs must be self-consistent"
662 );
663
664 let fast_full = fast.encode(&full).unwrap().token_ids().to_vec();
665 let fast_suffix = fast.encode(suffix).unwrap().token_ids().to_vec();
666 assert_ne!(
667 fast_full, ref_full,
668 "revisit the known Unicode contraction mismatch"
669 );
670 assert_eq!(
671 fast_full[1..],
672 fast_suffix[..],
673 "splitting at this special-token boundary must preserve fastokens ids"
674 );
675 }
676}