1use std::path::Path;
18
19use aho_corasick::{AhoCorasick, MatchKind};
20use anyhow::Context as _;
21use base64::Engine as _;
22use fancy_regex::Regex;
23use rayon::prelude::*;
24use rustc_hash::FxHashMap;
25use serde::Deserialize;
26use tiktoken_rs::{CoreBPE, byte_pair_split};
27
28use super::{
29 EncodeSegment, Encoding, Error, Result, TokenIdType, TokenizerOptions,
30 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
31};
32
33const PARSER_MARKERS: [&str; 4] = ["[TOOL_CALLS]", "[ARGS]", "[THINK]", "[/THINK]"];
39
40const MAX_SPECIAL_TOKENS: usize = 1 << 16;
43
44const BYTE_TOKENS: usize = 256;
46
47#[derive(Deserialize)]
48struct TekkenFile {
49 config: TekkenConfig,
50 vocab: Vec<TekkenTokenInfo>,
51 special_tokens: Option<Vec<TekkenSpecialToken>>,
52 model_settings_builder: Option<serde_json::Value>,
53}
54
55#[derive(Deserialize)]
56struct TekkenConfig {
57 pattern: String,
58 default_vocab_size: usize,
59 default_num_special_tokens: usize,
60 version: String,
61}
62
63#[derive(Deserialize)]
64struct TekkenTokenInfo {
65 rank: usize,
66 token_bytes: String,
67}
68
69#[derive(Deserialize)]
70struct TekkenSpecialToken {
71 rank: usize,
72 token_str: String,
73}
74
75pub struct TekkenTokenizer {
76 bpe: CoreBPE,
77 pattern: Regex,
78 special_token_matcher: AhoCorasick,
79 ordinary_token_ids: FxHashMap<Vec<u8>, TokenIdType>,
80 num_special_tokens: u32,
81 vocab_size: u32,
82 special_token_ids: FxHashMap<String, TokenIdType>,
83 kept_on_skip: Vec<TokenIdType>,
84 version: String,
85 model_settings_builder: Option<serde_json::Value>,
86}
87
88impl TekkenTokenizer {
89 pub fn from_file(path: &str) -> Result<Self> {
91 let file: TekkenFile = {
92 let contents = std::fs::read(path)
93 .with_context(|| format!("Failed to read tekken file '{path}'"))?;
94 serde_json::from_slice(&contents)
95 .with_context(|| format!("Failed to parse tekken file '{path}'"))?
96 };
97 Self::from_parts(file)
98 }
99
100 fn from_parts(file: TekkenFile) -> Result<Self> {
101 let TekkenConfig {
102 pattern,
103 default_vocab_size,
104 default_num_special_tokens,
105 version,
106 } = file.config;
107 let vocab_size = u32::try_from(default_vocab_size)
108 .context("tekken default_vocab_size does not fit a token id")?;
109 let num_special_tokens = u32::try_from(default_num_special_tokens)
110 .context("tekken default_num_special_tokens does not fit a token id")?;
111 if default_num_special_tokens > MAX_SPECIAL_TOKENS {
112 anyhow::bail!(
113 "tekken reserves {default_num_special_tokens} special tokens, more than the supported {MAX_SPECIAL_TOKENS}"
114 );
115 }
116
117 let Some(specials) = file.special_tokens else {
118 anyhow::bail!(
119 "tekken {version} file has no special_tokens; only files that list them are supported"
120 );
121 };
122 if specials.len() > default_num_special_tokens {
123 anyhow::bail!(
124 "tekken file lists {} special tokens but reserves only {default_num_special_tokens}",
125 specials.len()
126 );
127 }
128 let Some(inner_vocab_size) = default_vocab_size.checked_sub(default_num_special_tokens)
129 else {
130 anyhow::bail!("tekken default_vocab_size is smaller than its special tokens");
131 };
132 if inner_vocab_size < BYTE_TOKENS {
134 anyhow::bail!(
135 "tekken keeps {inner_vocab_size} ordinary tokens, fewer than the {BYTE_TOKENS} single bytes"
136 );
137 }
138 if file.vocab.len() < inner_vocab_size {
139 anyhow::bail!(
140 "tekken vocab has {} entries, fewer than the {inner_vocab_size} the config requires",
141 file.vocab.len()
142 );
143 }
144
145 let mut special_tokens: Vec<String> = (0..default_num_special_tokens)
147 .map(|id| format!("<SPECIAL_{id}>"))
148 .collect();
149 for (index, token) in specials.into_iter().enumerate() {
150 if token.rank != index {
151 anyhow::bail!(
152 "tekken special token '{}' has rank {}, expected {index}",
153 token.token_str,
154 token.rank
155 );
156 }
157 if token.token_str.is_empty() {
160 anyhow::bail!("tekken special token {index} is empty");
161 }
162 special_tokens[index] = token.token_str;
163 }
164
165 let mut special_token_ids = FxHashMap::default();
166 for (id, token) in (0..num_special_tokens).zip(special_tokens) {
167 if let Some(previous) = special_token_ids.insert(token, id) {
168 anyhow::bail!("tekken special token ids {previous} and {id} share one string");
169 }
170 }
171
172 let engine = base64::engine::general_purpose::STANDARD;
173 let mut ordinary_token_ids = FxHashMap::default();
174 for (rank, token) in (0..).zip(file.vocab.into_iter().take(inner_vocab_size)) {
175 if token.rank != rank as usize {
176 anyhow::bail!("tekken vocab entry {rank} has rank {}", token.rank);
177 }
178 let bytes = engine
179 .decode(&token.token_bytes)
180 .with_context(|| format!("Invalid base64 in tekken vocab entry {rank}"))?;
181 if (rank as usize) < BYTE_TOKENS && bytes != [rank as u8] {
183 anyhow::bail!("tekken vocab entry {rank} is not the single byte {rank}");
184 }
185 if ordinary_token_ids
187 .insert(bytes, num_special_tokens + rank)
188 .is_some()
189 {
190 anyhow::bail!("tekken vocab entry {rank} duplicates an earlier token");
191 }
192 }
193
194 let pattern = Regex::new(&pattern).context("Error creating tekken split regex")?;
195 let bpe = CoreBPE::new(
198 ordinary_token_ids.clone(),
199 special_token_ids.clone(),
200 pattern.as_str(),
201 )
202 .map_err(|err| Error::msg(format!("Error creating tekken BPE: {err}")))?;
203 let special_token_matcher = AhoCorasick::builder()
204 .match_kind(MatchKind::LeftmostFirst)
205 .build(special_token_ids.keys())
206 .context("Error creating tekken special-token matcher")?;
207
208 let kept_on_skip = PARSER_MARKERS
209 .iter()
210 .filter_map(|marker| special_token_ids.get(*marker).copied())
211 .collect();
212
213 Ok(Self {
214 bpe,
215 pattern,
216 special_token_matcher,
217 ordinary_token_ids,
218 num_special_tokens,
219 vocab_size,
220 special_token_ids,
221 kept_on_skip,
222 version,
223 model_settings_builder: file.model_settings_builder,
224 })
225 }
226
227 pub fn is_tekken_file(path: &Path) -> bool {
230 path.file_name()
231 .and_then(std::ffi::OsStr::to_str)
232 .is_some_and(|name| name.starts_with("tekken") && name.ends_with(".json"))
233 }
234
235 pub fn version(&self) -> &str {
237 &self.version
238 }
239
240 pub fn version_number(&self) -> Option<u32> {
242 self.version.strip_prefix('v')?.parse().ok()
243 }
244
245 pub fn model_settings_builder(&self) -> Option<&serde_json::Value> {
248 self.model_settings_builder.as_ref()
249 }
250
251 fn validate_ids(&self, token_ids: &[TokenIdType]) -> Result<()> {
257 match token_ids.iter().find(|&&id| id >= self.vocab_size) {
258 Some(id) => anyhow::bail!(
259 "token id {id} is outside the tekken vocabulary of {}",
260 self.vocab_size
261 ),
262 None => Ok(()),
263 }
264 }
265
266 fn encode_ordinary_into(&self, input: &str, token_ids: &mut Vec<u32>) -> Result<()> {
267 for piece in self.pattern.find_iter(input) {
268 let piece = piece.context("Failed to match tekken split regex")?;
269 let bytes = piece.as_str().as_bytes();
270 if let Some(&id) = self.ordinary_token_ids.get(bytes) {
271 token_ids.push(id);
272 } else {
273 anyhow::ensure!(
274 !bytes.is_empty(),
275 "tekken split regex matched an empty string"
276 );
277 token_ids.extend(
280 byte_pair_split(bytes, &self.ordinary_token_ids)
281 .into_iter()
282 .map(|piece| self.ordinary_token_ids[piece]),
283 );
284 }
285 }
286 Ok(())
287 }
288
289 fn encode_special_into(&self, input: &str, token_ids: &mut Vec<u32>) -> Result<()> {
290 let mut end = 0;
291 for matched in self.special_token_matcher.find_iter(input) {
292 self.encode_ordinary_into(&input[end..matched.start()], token_ids)?;
293 let id = self.special_token_ids[&input[matched.start()..matched.end()]];
294 token_ids.push(id);
295 end = matched.end();
296 }
297 self.encode_ordinary_into(&input[end..], token_ids)
298 }
299}
300
301impl Encoder for TekkenTokenizer {
302 fn encode(&self, input: &str) -> Result<Encoding> {
303 let mut token_ids = Vec::new();
304 self.encode_ordinary_into(input, &mut token_ids)?;
305 Ok(Encoding::Sp(token_ids))
306 }
307
308 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
309 inputs.par_iter().map(|input| self.encode(input)).collect()
310 }
311
312 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
313 let mut token_ids = Vec::new();
314 for segment in segments {
315 if !segment.allow_special {
316 self.encode_ordinary_into(segment.text, &mut token_ids)?;
317 } else if let Some(&id) = self.special_token_ids.get(segment.text) {
318 token_ids.push(id);
320 } else {
321 self.encode_special_into(segment.text, &mut token_ids)?;
322 }
323 }
324 Ok(Encoding::Sp(token_ids))
325 }
326}
327
328impl Decoder for TekkenTokenizer {
329 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
330 self.validate_ids(token_ids)?;
331 let ids: Vec<u32> = if skip_special_tokens {
332 token_ids
333 .iter()
334 .filter(|&&id| id >= self.num_special_tokens || self.kept_on_skip.contains(&id))
335 .copied()
336 .collect()
337 } else {
338 token_ids.to_vec()
339 };
340
341 let bytes: Vec<u8> = self.bpe._decode_native_and_split(ids).flatten().collect();
344 match String::from_utf8(bytes) {
345 Ok(text) => Ok(DecodeResult::Complete(text)),
346 Err(e) => {
347 let text = String::from_utf8_lossy(e.as_bytes()).into_owned();
348 Ok(DecodeResult::from_decoded(text))
349 }
350 }
351 }
352}
353
354impl Tokenizer for TekkenTokenizer {
355 fn with_options(self, _options: TokenizerOptions) -> Self {
361 self
362 }
363
364 fn vocab_size(&self) -> Option<usize> {
365 Some(self.vocab_size as usize)
366 }
367
368 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
369 if let Some(&id) = self.special_token_ids.get(token) {
370 return Ok(Some(id));
371 }
372 Ok(match self.encode(token)?.token_ids() {
374 &[id] => Some(id),
375 _ => None,
376 })
377 }
378
379 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
380 Ok((0..self.num_special_tokens).collect())
381 }
382
383 fn num_special_tokens_added(&self) -> Result<usize> {
385 Ok(0)
386 }
387}
388
389#[cfg(test)]
390mod tests {
391 use super::*;
392 use crate::DecodeStream;
393 use std::sync::Arc;
394
395 const FIXTURE: &str = concat!(
396 env!("CARGO_MANIFEST_DIR"),
397 "/tests/data/sample-models/mock-tekken/tekken.json"
398 );
399 const EXPECTED_IDS: &str = concat!(
400 env!("CARGO_MANIFEST_DIR"),
401 "/tests/data/sample-models/mock-tekken/expected_ids.json"
402 );
403
404 #[derive(Deserialize)]
405 struct ExpectedIds {
406 text: String,
407 ids: Vec<TokenIdType>,
408 }
409
410 fn id_of(tokenizer: &TekkenTokenizer, token: &str) -> Option<TokenIdType> {
411 tokenizer.token_to_id(token).unwrap()
412 }
413
414 fn fixture() -> TekkenTokenizer {
415 TekkenTokenizer::from_file(FIXTURE).unwrap()
416 }
417
418 #[test]
419 fn encode_matches_mistral_common() {
420 let tokenizer = fixture();
421 let expected: Vec<ExpectedIds> =
422 serde_json::from_str(&std::fs::read_to_string(EXPECTED_IDS).unwrap()).unwrap();
423 assert!(!expected.is_empty());
424 for case in expected {
425 let ids = tokenizer.encode(&case.text).unwrap();
426 assert_eq!(ids.token_ids(), case.ids.as_slice(), "text {:?}", case.text);
427 let decoded: String = tokenizer.decode(&case.ids, false).unwrap().into();
428 assert_eq!(decoded, case.text);
429 }
430 }
431
432 #[test]
433 fn metadata_comes_from_the_file() {
434 let tokenizer = fixture();
435 assert_eq!(tokenizer.version(), "v15");
436 assert_eq!(tokenizer.version_number(), Some(15));
437 assert_eq!(tokenizer.vocab_size(), Some(1040));
438 assert_eq!(id_of(&tokenizer, "<s>"), Some(1));
439 assert_eq!(id_of(&tokenizer, "</s>"), Some(2));
440 assert_eq!(id_of(&tokenizer, "[TOOL_CALLS]"), Some(9));
441 assert_eq!(id_of(&tokenizer, "[THINK]"), Some(34));
442 assert_eq!(id_of(&tokenizer, "<SPECIAL_39>"), Some(39));
444 assert_eq!(tokenizer.special_token_ids().unwrap().len(), 40);
445 assert_eq!(tokenizer.token_to_id("[ARGS]").unwrap(), Some(32));
446 assert_eq!(
447 tokenizer.token_to_id("a").unwrap(),
448 Some(40 + u32::from(b'a'))
449 );
450 assert_eq!(tokenizer.token_to_id("not-a-token-xyz").unwrap(), None);
451 let builder = tokenizer.model_settings_builder().unwrap();
452 assert_eq!(builder["reasoning_effort"]["default"], "none");
453 }
454
455 #[test]
456 fn special_token_text_is_ordinary_unless_marked_control() {
457 let tokenizer = fixture();
458 let inst = id_of(&tokenizer, "[INST]").unwrap();
459
460 let plain = tokenizer.encode("[INST]").unwrap();
461 assert!(!plain.token_ids().contains(&inst));
462
463 let segments = tokenizer
464 .encode_segments(&[
465 EncodeSegment::control("<s>[INST]"),
466 EncodeSegment::ordinary("hi [INST]"),
467 EncodeSegment::control("[/INST]"),
468 ])
469 .unwrap();
470 let ids = segments.token_ids();
471 assert_eq!(ids[..2], [1, inst]);
472 assert_eq!(*ids.last().unwrap(), 4);
473 assert_eq!(ids.iter().filter(|&&id| id == inst).count(), 1);
474 }
475
476 #[test]
477 fn skipping_special_tokens_keeps_parser_markers() {
478 let tokenizer = fixture();
479 let id = |token: &str| id_of(&tokenizer, token).unwrap();
480 let mut ids = vec![id("<s>"), id("[THINK]")];
481 ids.extend(tokenizer.encode("plan").unwrap().token_ids());
482 ids.extend([id("[/THINK]"), id("[TOOL_CALLS]")]);
483 ids.extend(tokenizer.encode("f").unwrap().token_ids());
484 ids.push(id("[ARGS]"));
485 ids.extend(tokenizer.encode("{}").unwrap().token_ids());
486 ids.push(id("</s>"));
487
488 let skipped: String = tokenizer.decode(&ids, true).unwrap().into();
489 assert_eq!(skipped, "[THINK]plan[/THINK][TOOL_CALLS]f[ARGS]{}");
490 let kept: String = tokenizer.decode(&ids, false).unwrap().into();
491 assert_eq!(kept, "<s>[THINK]plan[/THINK][TOOL_CALLS]f[ARGS]{}</s>");
492 }
493
494 #[test]
495 fn decode_rejects_ids_outside_the_vocabulary() {
496 let tokenizer = fixture();
497 assert!(tokenizer.decode(&[1040], false).is_err());
498 }
499
500 #[test]
501 fn decode_stream_reassembles_byte_fallback_characters() {
502 let tokenizer = fixture();
503 let ids = tokenizer.encode("北京 😀").unwrap().token_ids().to_vec();
504 let tokenizer: Arc<dyn Tokenizer> = Arc::new(tokenizer);
505 let mut stream = DecodeStream::new(tokenizer, &[], false);
506 let mut text = String::new();
507 for id in ids {
508 if let Some(chunk) = stream.step(id).unwrap() {
509 assert!(!chunk.contains('\u{FFFD}'), "chunk {chunk:?}");
510 text.push_str(&chunk);
511 }
512 }
513 if let Some(chunk) = stream.finish().unwrap() {
514 text.push_str(&chunk);
515 }
516 assert_eq!(text, "北京 😀");
517 }
518
519 #[test]
520 fn file_dispatch_selects_tekken() {
521 let tokenizer = crate::create_tokenizer_from_file(FIXTURE).unwrap();
522 assert_eq!(tokenizer.vocab_size(), Some(1040));
523 let ids = tokenizer.encode("Hello, world!").unwrap();
524 assert_eq!(
525 ids.token_ids(),
526 fixture().encode("Hello, world!").unwrap().token_ids()
527 );
528 }
529
530 #[test]
531 fn control_segments_match_special_token_matching() {
532 let tokenizer = fixture();
533 let fast = tokenizer
534 .encode_segments(&[EncodeSegment::control("[INST]")])
535 .unwrap();
536 let mixed = tokenizer
537 .encode_segments(&[EncodeSegment::control("[INST][/INST]")])
538 .unwrap();
539 assert_eq!(fast.token_ids(), [3]);
540 assert_eq!(mixed.token_ids(), [3, 4]);
541 }
542
543 #[test]
544 fn fallible_encoding_matches_core_bpe() {
545 let tokenizer = fixture();
546 for text in [
547 "We're in Beijing 北京 😀\nDon't split café's accents.",
548 "'re\u{0301} I'M 123\r\n\t /[INST]/ [ARGS]{}",
549 "<s>[INST]hi[/INST][TOOL_CALLS]f[ARGS]{}[THINK]plan[/THINK]",
550 ] {
551 assert_eq!(
552 tokenizer.encode(text).unwrap().token_ids(),
553 tokenizer.bpe.encode_ordinary(text)
554 );
555 assert_eq!(
556 tokenizer
557 .encode_segments(&[EncodeSegment::control(text)])
558 .unwrap()
559 .token_ids(),
560 tokenizer.bpe.encode_with_special_tokens(text)
561 );
562 }
563
564 let long = "A".repeat(100_000);
565 assert_eq!(
566 tokenizer.encode(&long).unwrap().token_ids(),
567 tokenizer.bpe.encode_ordinary(&long)
568 );
569 }
570
571 #[test]
572 fn long_regex_failures_are_returned_from_all_encoding_entry_points() {
573 let tokenizer = fixture();
574 for text in ["A".repeat(1_000_000), " ".repeat(1_000_000)] {
575 let error = tokenizer.encode(&text).unwrap_err();
576 assert!(error.to_string().contains("split regex"), "{error:#}");
577 assert!(tokenizer.encode_batch(&[&text]).is_err());
578 assert!(
579 tokenizer
580 .encode_segments(&[EncodeSegment::ordinary(&text)])
581 .is_err()
582 );
583 let controlled = format!("<s>[INST]{text}[/INST]");
584 assert!(
585 tokenizer
586 .encode_segments(&[EncodeSegment::control(&controlled)])
587 .is_err()
588 );
589 assert!(tokenizer.token_to_id(&text).is_err());
590 }
591 }
592
593 fn load_error(edit: impl FnOnce(&mut serde_json::Value)) -> String {
595 let mut file: serde_json::Value =
596 serde_json::from_str(&std::fs::read_to_string(FIXTURE).unwrap()).unwrap();
597 edit(&mut file);
598 let file: TekkenFile = serde_json::from_value(file).unwrap();
599 TekkenTokenizer::from_parts(file).err().unwrap().to_string()
600 }
601
602 #[test]
603 fn rejects_empty_special_tokens() {
604 let error = load_error(|file| file["special_tokens"][5]["token_str"] = "".into());
605 assert!(error.contains("empty"), "{error}");
606 }
607
608 #[test]
609 fn rejects_vocab_without_the_byte_alphabet() {
610 let error = load_error(|file| {
611 let bytes = base64::engine::general_purpose::STANDARD.encode([0xff, 0xfe, 0xfd]);
612 file["vocab"][65]["token_bytes"] = bytes.into();
613 });
614 assert!(error.contains("single byte 65"), "{error}");
615
616 let error = load_error(|file| file["config"]["default_vocab_size"] = (40 + 255).into());
617 assert!(error.contains("fewer than the 256"), "{error}");
618 }
619
620 #[test]
621 fn rejects_oversized_special_token_reservation() {
622 let error = load_error(|file| {
623 file["config"]["default_vocab_size"] = u32::MAX.into();
624 file["config"]["default_num_special_tokens"] = u32::MAX.into();
625 file["special_tokens"] = serde_json::json!([]);
626 file["vocab"] = serde_json::json!([]);
627 });
628 assert!(error.contains("special tokens, more than"), "{error}");
629 }
630
631 #[test]
632 fn prefix_cache_is_refused() {
633 assert!(fixture().validate_prefix_cache().is_err());
634 }
635
636 #[test]
637 fn recognizes_tekken_file_names() {
638 assert!(TekkenTokenizer::is_tekken_file(Path::new("/m/tekken.json")));
639 assert!(TekkenTokenizer::is_tekken_file(Path::new(
640 "tekken_240911.json"
641 )));
642 assert!(!TekkenTokenizer::is_tekken_file(Path::new(
643 "/m/tokenizer.json"
644 )));
645 assert!(!TekkenTokenizer::is_tekken_file(Path::new(
646 "/m/tekken.model"
647 )));
648 }
649}