Skip to main content

dynamo_tokenizers/
basetenkenizer.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Baseten Tokenizer backend for high-performance BPE encoding and decoding.
5//!
6//! Some Kimi repositories ship tiktoken assets without a directly loadable
7//! `tokenizer.json`. Baseten publishes compatible tokenizer artifacts,
8//! including [`baseten/kimi-k3-tokenizer`](https://huggingface.co/baseten/kimi-k3-tokenizer).
9
10use std::{collections::HashSet, path::Path};
11
12use super::{
13    EncodeSegment, Encoding, Error, Result, TokenIdType, TokenizerOptions,
14    traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17/// Tokenizer backed by the `basetenkenizer` crate.
18pub struct BasetenTokenizer {
19    tokenizer: basetenkenizer::Tokenizer,
20    options: TokenizerOptions,
21    /// Every added-token id, `special: true` or not; see
22    /// [`crate::reject_control_ids_in_ordinary_segment`].
23    added_ids: HashSet<u32>,
24}
25
26impl BasetenTokenizer {
27    /// Load a tokenizer from a Hugging Face `tokenizer.json` file.
28    pub fn from_file(path: &str) -> Result<Self> {
29        let path = Path::new(path);
30        let raw = std::fs::read_to_string(path)
31            .map_err(|e| Error::msg(format!("Error reading Baseten tokenizer: {e}")))?;
32        let mut json: serde_json::Value = serde_json::from_str(&raw)
33            .map_err(|e| Error::msg(format!("Error parsing Baseten tokenizer: {e}")))?;
34        if let Some(parent) = path.parent() {
35            merge_special_tokens_from_config(&mut json, parent);
36        }
37        let tokenizer = basetenkenizer::Tokenizer::from_json(json)
38            .map_err(|e| Error::msg(format!("Error loading Baseten tokenizer: {e}")))?;
39        let added_ids = tokenizer
40            .added_tokens()
41            .into_iter()
42            .flat_map(|added| added.iter().map(|info| info.id))
43            .collect();
44        Ok(Self {
45            tokenizer,
46            options: TokenizerOptions::default(),
47            added_ids,
48        })
49    }
50}
51
52fn merge_special_tokens_from_config(json: &mut serde_json::Value, model_dir: &Path) {
53    let config_path = model_dir.join("tokenizer_config.json");
54    let Ok(raw) = std::fs::read_to_string(&config_path) else {
55        return;
56    };
57    let config: serde_json::Value = match serde_json::from_str(&raw) {
58        Ok(value) => value,
59        Err(error) => {
60            tracing::debug!(
61                target: "tokenizer",
62                path = %config_path.display(),
63                error = %error,
64                "tokenizer_config.json parse failed; skipping special-token merge"
65            );
66            return;
67        }
68    };
69    let Some(decoder) = config
70        .get("added_tokens_decoder")
71        .and_then(serde_json::Value::as_object)
72    else {
73        return;
74    };
75
76    if json.get("added_tokens").is_none() {
77        json["added_tokens"] = serde_json::json!([]);
78    }
79    let Some(added_tokens) = json
80        .get_mut("added_tokens")
81        .and_then(serde_json::Value::as_array_mut)
82    else {
83        return;
84    };
85
86    for (id, spec) in decoder {
87        let Some(id) = id.parse::<u32>().ok() else {
88            continue;
89        };
90        let Some(spec) = spec.as_object() else {
91            continue;
92        };
93        if spec.get("special").and_then(serde_json::Value::as_bool) != Some(true) {
94            continue;
95        }
96        let Some(content) = spec
97            .get("content")
98            .and_then(serde_json::Value::as_str)
99            .filter(|content| !content.is_empty())
100        else {
101            continue;
102        };
103
104        if let Some(existing) = added_tokens
105            .iter_mut()
106            .find(|token| token.get("content").and_then(serde_json::Value::as_str) == Some(content))
107        {
108            existing["special"] = serde_json::Value::Bool(true);
109            continue;
110        }
111
112        let mut token = serde_json::Map::from_iter([
113            ("id".to_string(), serde_json::json!(id)),
114            ("content".to_string(), serde_json::json!(content)),
115            ("special".to_string(), serde_json::Value::Bool(true)),
116        ]);
117        for field in ["single_word", "lstrip", "rstrip", "normalized"] {
118            token.insert(
119                field.to_string(),
120                serde_json::Value::Bool(
121                    spec.get(field)
122                        .and_then(serde_json::Value::as_bool)
123                        .unwrap_or(false),
124                ),
125            );
126        }
127        added_tokens.push(serde_json::Value::Object(token));
128    }
129}
130
131impl Encoder for BasetenTokenizer {
132    fn encode(&self, input: &str) -> Result<Encoding> {
133        let ids = self
134            .tokenizer
135            .encode_with_special_tokens(input, self.options.add_special_tokens)
136            .map_err(|e| Error::msg(format!("Baseten tokenizer encode error: {e}")))?;
137        Ok(Encoding::Sp(ids))
138    }
139
140    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
141        self.tokenizer
142            .encode_batch(inputs, self.options.add_special_tokens)
143            .map(|ids| ids.into_iter().map(Encoding::Sp).collect())
144            .map_err(|e| Error::msg(format!("Baseten tokenizer batch encode error: {e}")))
145    }
146
147    fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
148        // Each segment is encoded once, without post-processing, which then
149        // runs on the joined ids as it does for a single segmented encode.
150        let mut ids = Vec::new();
151        for segment in segments {
152            let start = ids.len();
153            ids.extend(
154                self.tokenizer
155                    .encode_segments_tiktoken_safe([(segment.text, segment.allow_special)], false)
156                    .map_err(|e| {
157                        Error::msg(format!("Baseten tokenizer segment encode error: {e}"))
158                    })?,
159            );
160            if !segment.allow_special {
161                crate::reject_control_ids_in_ordinary_segment(&ids[start..], &self.added_ids)?;
162            }
163        }
164        Ok(Encoding::Sp(
165            self.tokenizer
166                .post_process(ids, self.options.add_special_tokens),
167        ))
168    }
169}
170
171impl Decoder for BasetenTokenizer {
172    fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
173        self.tokenizer
174            .decode(token_ids, skip_special_tokens)
175            .map(DecodeResult::from)
176            .map_err(|e| Error::msg(format!("Baseten tokenizer decode error: {e}")))
177    }
178}
179
180impl Tokenizer for BasetenTokenizer {
181    fn validate_prefix_cache(&self) -> Result<()> {
182        if self.options.add_special_tokens {
183            return Err(Error::msg(
184                "Baseten tokenizers configured with add_special_tokens=true must remain uncached",
185            ));
186        }
187        Ok(())
188    }
189
190    fn with_options(mut self, options: TokenizerOptions) -> Self {
191        self.options = options;
192        self
193    }
194
195    fn vocab_size(&self) -> Option<usize> {
196        // `basetenkenizer::Tokenizer::vocab_size` sums the model vocab size
197        // and the added-tokens count with no dedup, but real tokenizer.json
198        // files (this crate's own TinyLlama_v1.1 fixture included) list
199        // `added_tokens` entries whose ids already exist in `model.vocab` --
200        // metadata for special tokens, not additional vocabulary. Count only
201        // added tokens whose content isn't already in the model vocab.
202        let model = self.tokenizer.model();
203        let model_size = model.vocab_size();
204        let extra = self.tokenizer.added_tokens().map_or(0, |added| {
205            added
206                .iter()
207                .filter(|info| model.token_to_id(info.content).is_none())
208                .count()
209        });
210        Some(model_size + extra)
211    }
212
213    fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
214        Ok(self.tokenizer.token_to_id(token))
215    }
216
217    fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
218        let Some(added_tokens) = self.tokenizer.added_tokens() else {
219            return Ok(Vec::new());
220        };
221        let mut ids: Vec<TokenIdType> = added_tokens
222            .iter()
223            .filter_map(|info| info.special.then_some(info.id))
224            .collect();
225        ids.sort_unstable();
226        Ok(ids)
227    }
228
229    fn num_special_tokens_added(&self) -> Result<usize> {
230        // basetenkenizer exposes no direct count, but post-processing an
231        // empty sequence with add_special_tokens=true is content-length
232        // independent for every basetenkenizer::PostProcessor variant:
233        // ByteLevel is identity (adds 0); TemplateProcessing::apply_single
234        // walks a fixed template where SpecialToken pieces insert a
235        // fixed-length id vector and Sequence pieces are replaced 1:1 by
236        // whatever content came in, so the *added* length never depends on
237        // content length; Sequence(steps) folds that same guarantee across
238        // steps. So this reveals exactly what the post-processor inserts
239        // around a bare encoding, for real content of any length.
240        Ok(self.tokenizer.post_process(Vec::new(), true).len())
241    }
242}
243
244#[cfg(test)]
245mod tests {
246    use std::sync::Arc;
247
248    use super::*;
249    use crate::{
250        HuggingFaceTokenizer, Tokenizer as TokenizerWrapper, traits::Tokenizer as TokenizerTrait,
251    };
252
253    const TOKENIZER_PATH: &str = concat!(
254        env!("CARGO_MANIFEST_DIR"),
255        "/tests/data/minimal-bpe/tokenizer.json"
256    );
257
258    #[test]
259    fn encode_matches_hugging_face() {
260        let baseten = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
261        let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
262
263        for text in ["Hello, world!", "Hello", " world", "He llo"] {
264            let baseten_ids = baseten.encode(text).unwrap();
265            let hf_ids = hf.encode(text).unwrap();
266            assert_eq!(
267                baseten_ids.token_ids(),
268                hf_ids.token_ids(),
269                "Baseten and Hugging Face must produce identical token IDs for '{text}'"
270            );
271        }
272    }
273
274    #[test]
275    fn batch_encode_matches_sequential_encode() {
276        let tokenizer = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
277        let inputs = ["Hello", " world", "Hello, world!"];
278        let batch = tokenizer.encode_batch(&inputs).unwrap();
279
280        for (encoding, input) in batch.iter().zip(inputs) {
281            let sequential = tokenizer.encode(input).unwrap();
282            assert_eq!(encoding.token_ids(), sequential.token_ids());
283        }
284    }
285
286    #[test]
287    fn encode_decode_roundtrip() {
288        let tokenizer = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
289        let encoding = tokenizer.encode("Hello, world!").unwrap();
290        let decoded = tokenizer.decode(encoding.token_ids(), true).unwrap();
291
292        assert_eq!(decoded.as_str(), "Hello, world!");
293    }
294
295    #[test]
296    fn works_with_decode_stream() {
297        let tokenizer = Arc::new(BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap());
298        let wrapper = TokenizerWrapper::from(tokenizer);
299        let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
300        let continuation_ids = wrapper.encode(", world!").unwrap().token_ids().to_vec();
301        let mut stream = wrapper.decode_stream(&prompt_ids, true);
302        let mut accumulated = String::new();
303
304        for id in &continuation_ids {
305            if let Some(chunk) = stream.step(*id).unwrap() {
306                accumulated.push_str(&chunk);
307            }
308        }
309
310        let mut all_ids = prompt_ids.clone();
311        all_ids.extend_from_slice(&continuation_ids);
312        let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
313        let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
314        assert_eq!(accumulated, full_text[prompt_text.len()..]);
315    }
316
317    #[test]
318    fn prefix_cache_rejects_special_token_post_processing() {
319        let plain = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
320        assert!(plain.validate_prefix_cache().is_ok());
321
322        let with_special_tokens = BasetenTokenizer::from_file(TOKENIZER_PATH)
323            .unwrap()
324            .with_options(TokenizerOptions {
325                add_special_tokens: true,
326            });
327        assert!(with_special_tokens.validate_prefix_cache().is_err());
328    }
329
330    #[test]
331    fn segments_preserve_special_token_trust_boundary() {
332        let temp = tempfile::tempdir().unwrap();
333        let path = temp.path().join("tokenizer.json");
334        let mut json: serde_json::Value =
335            serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
336        let vocab = json["model"]["vocab"].as_object_mut().unwrap();
337        vocab.insert("<".to_string(), serde_json::json!(23));
338        vocab.insert(">".to_string(), serde_json::json!(24));
339        vocab.insert("c".to_string(), serde_json::json!(25));
340        json["added_tokens"] = serde_json::json!([{
341            "id": 26,
342            "content": "<ctl>",
343            "single_word": false,
344            "lstrip": false,
345            "rstrip": false,
346            "normalized": false,
347            "special": true
348        }]);
349        std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
350
351        let tokenizer: Arc<dyn TokenizerTrait> =
352            Arc::new(BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap());
353        let segments = [
354            EncodeSegment::new("Hello", true),
355            EncodeSegment::new("<ctl>", false),
356            EncodeSegment::new(" world!", true),
357        ];
358
359        let segmented = tokenizer.encode_segments(&segments).unwrap();
360        let flattened = tokenizer.encode("Hello<ctl> world!").unwrap();
361
362        assert_ne!(
363            segmented.token_ids(),
364            flattened.token_ids(),
365            "untrusted control-token-looking text must not become an added token"
366        );
367        assert!(flattened.token_ids().contains(&26));
368        assert!(!segmented.token_ids().contains(&26));
369    }
370
371    #[test]
372    fn segments_reject_ordinary_control_ids() {
373        // `H`+`e` merges to id 18, declared here as the added token `He`:
374        // ordinary text must not come back as that control id.
375        let temp = tempfile::tempdir().unwrap();
376        let path = temp.path().join("tokenizer.json");
377        let mut json: serde_json::Value =
378            serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
379        json["added_tokens"] = serde_json::json!([{
380            "id": 18,
381            "content": "He",
382            "single_word": false,
383            "lstrip": false,
384            "rstrip": false,
385            "normalized": false,
386            "special": true
387        }]);
388        std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
389
390        let tokenizer: Arc<dyn TokenizerTrait> =
391            Arc::new(BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap());
392        let control = tokenizer
393            .encode_segments(&[EncodeSegment::new("He", true)])
394            .unwrap();
395        assert_eq!(control.token_ids(), &[18]);
396        let error = tokenizer
397            .encode_segments(&[EncodeSegment::new("He", false)])
398            .unwrap_err();
399        assert!(
400            error.to_string().contains("control-token id 18"),
401            "unexpected error: {error}"
402        );
403    }
404
405    #[test]
406    fn segments_honor_add_special_tokens_option() {
407        let temp = tempfile::tempdir().unwrap();
408        let path = temp.path().join("tokenizer.json");
409        let mut json: serde_json::Value =
410            serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
411        json["model"]["vocab"]["<bos>"] = serde_json::json!(23);
412        json["added_tokens"] = serde_json::json!([{
413            "id": 23,
414            "content": "<bos>",
415            "single_word": false,
416            "lstrip": false,
417            "rstrip": false,
418            "normalized": false,
419            "special": true
420        }]);
421        json["post_processor"] = serde_json::json!({
422            "type": "TemplateProcessing",
423            "single": [
424                {"SpecialToken": {"id": "<bos>", "type_id": 0}},
425                {"Sequence": {"id": "A", "type_id": 0}}
426            ],
427            "pair": [
428                {"SpecialToken": {"id": "<bos>", "type_id": 0}},
429                {"Sequence": {"id": "A", "type_id": 0}},
430                {"Sequence": {"id": "B", "type_id": 0}}
431            ],
432            "special_tokens": {
433                "<bos>": {"id": "<bos>", "ids": [23], "tokens": ["<bos>"]}
434            }
435        });
436        std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
437        let segments = [EncodeSegment::new("Hello", false)];
438
439        let plain = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
440        let plain_ids = plain
441            .encode_segments(&segments)
442            .unwrap()
443            .token_ids()
444            .to_vec();
445
446        let with_bos = BasetenTokenizer::from_file(path.to_str().unwrap())
447            .unwrap()
448            .with_options(TokenizerOptions {
449                add_special_tokens: true,
450            });
451        assert_eq!(
452            with_bos.encode_segments(&segments).unwrap().token_ids(),
453            [&[23], plain_ids.as_slice()].concat()
454        );
455    }
456
457    #[test]
458    fn num_special_tokens_added_reflects_bos_post_processor() {
459        let temp = tempfile::tempdir().unwrap();
460        let path = temp.path().join("tokenizer.json");
461        let mut json: serde_json::Value =
462            serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
463        json["model"]["vocab"]["<bos>"] = serde_json::json!(23);
464        json["added_tokens"] = serde_json::json!([{
465            "id": 23,
466            "content": "<bos>",
467            "single_word": false,
468            "lstrip": false,
469            "rstrip": false,
470            "normalized": false,
471            "special": true
472        }]);
473        json["post_processor"] = serde_json::json!({
474            "type": "TemplateProcessing",
475            "single": [
476                {"SpecialToken": {"id": "<bos>", "type_id": 0}},
477                {"Sequence": {"id": "A", "type_id": 0}}
478            ],
479            "pair": [
480                {"SpecialToken": {"id": "<bos>", "type_id": 0}},
481                {"Sequence": {"id": "A", "type_id": 0}},
482                {"Sequence": {"id": "B", "type_id": 0}}
483            ],
484            "special_tokens": {
485                "<bos>": {"id": "<bos>", "ids": [23], "tokens": ["<bos>"]}
486            }
487        });
488        std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
489
490        let with_bos = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
491        assert_eq!(with_bos.num_special_tokens_added().unwrap(), 1);
492
493        let plain = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
494        assert_eq!(plain.num_special_tokens_added().unwrap(), 0);
495    }
496
497    #[test]
498    fn num_special_tokens_added_is_length_independent_under_sequence_post_processor() {
499        let temp = tempfile::tempdir().unwrap();
500        let path = temp.path().join("tokenizer.json");
501        let mut json: serde_json::Value =
502            serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
503        json["model"]["vocab"]["<bos>"] = serde_json::json!(23);
504        json["model"]["vocab"]["<eos>"] = serde_json::json!(24);
505        json["added_tokens"] = serde_json::json!([
506            {
507                "id": 23,
508                "content": "<bos>",
509                "single_word": false,
510                "lstrip": false,
511                "rstrip": false,
512                "normalized": false,
513                "special": true
514            },
515            {
516                "id": 24,
517                "content": "<eos>",
518                "single_word": false,
519                "lstrip": false,
520                "rstrip": false,
521                "normalized": false,
522                "special": true
523            }
524        ]);
525        json["post_processor"] = serde_json::json!({
526            "type": "Sequence",
527            "processors": [
528                {
529                    "type": "TemplateProcessing",
530                    "single": [
531                        {"SpecialToken": {"id": "<bos>", "type_id": 0}},
532                        {"Sequence": {"id": "A", "type_id": 0}}
533                    ],
534                    "pair": [],
535                    "special_tokens": {
536                        "<bos>": {"id": "<bos>", "ids": [23], "tokens": ["<bos>"]}
537                    }
538                },
539                {
540                    "type": "TemplateProcessing",
541                    "single": [
542                        {"Sequence": {"id": "A", "type_id": 0}},
543                        {"SpecialToken": {"id": "<eos>", "type_id": 0}}
544                    ],
545                    "pair": [],
546                    "special_tokens": {
547                        "<eos>": {"id": "<eos>", "ids": [24], "tokens": ["<eos>"]}
548                    }
549                }
550            ]
551        });
552        std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
553
554        let tokenizer = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
555        assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 2);
556
557        let with_specials = tokenizer.with_options(TokenizerOptions {
558            add_special_tokens: true,
559        });
560        let plain = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
561
562        for text in ["h", "hello there world"] {
563            let without = plain.encode(text).unwrap().token_ids().len();
564            let with = with_specials.encode(text).unwrap().token_ids().len();
565            assert_eq!(
566                with - without,
567                2,
568                "'{text}' should grow by exactly num_special_tokens_added()"
569            );
570        }
571    }
572
573    #[test]
574    fn merges_config_only_special_tokens() {
575        let temp = tempfile::tempdir().unwrap();
576        let tokenizer_path = temp.path().join("tokenizer.json");
577        std::fs::copy(TOKENIZER_PATH, &tokenizer_path).unwrap();
578        std::fs::write(
579            temp.path().join("tokenizer_config.json"),
580            serde_json::json!({
581                "added_tokens_decoder": {
582                    "23": {
583                        "content": "<ctl>",
584                        "special": true,
585                        "single_word": false,
586                        "lstrip": false,
587                        "rstrip": false,
588                        "normalized": false
589                    }
590                }
591            })
592            .to_string(),
593        )
594        .unwrap();
595
596        let tokenizer = BasetenTokenizer::from_file(tokenizer_path.to_str().unwrap()).unwrap();
597        let encoding = tokenizer.encode("<ctl>").unwrap();
598        assert_eq!(encoding.token_ids(), &[23]);
599        assert_eq!(tokenizer.decode(&[23], false).unwrap().as_str(), "<ctl>");
600        assert_eq!(tokenizer.decode(&[23], true).unwrap().as_str(), "");
601    }
602
603    #[test]
604    fn vocab_introspection_accessors() {
605        let plain = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
606        assert_eq!(plain.vocab_size(), Some(23));
607        assert_eq!(plain.token_to_id("hello").unwrap(), None);
608        assert_eq!(plain.token_to_id("h").unwrap(), Some(10));
609        assert!(plain.special_token_ids().unwrap().is_empty());
610
611        let temp = tempfile::tempdir().unwrap();
612        let path = temp.path().join("tokenizer.json");
613        let mut json: serde_json::Value =
614            serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
615        json["model"]["vocab"]["<bos>"] = serde_json::json!(23);
616        json["added_tokens"] = serde_json::json!([{
617            "id": 23,
618            "content": "<bos>",
619            "single_word": false,
620            "lstrip": false,
621            "rstrip": false,
622            "normalized": false,
623            "special": true
624        }]);
625        std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
626
627        let with_added = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
628        assert_eq!(with_added.vocab_size(), Some(24));
629        assert_eq!(with_added.token_to_id("<bos>").unwrap(), Some(23));
630        assert_eq!(with_added.special_token_ids().unwrap(), vec![23]);
631
632        let path2 = temp.path().join("tokenizer2.json");
633        let mut json2: serde_json::Value =
634            serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
635        json2["added_tokens"] = serde_json::json!([{
636            "id": 23,
637            "content": "<extra>",
638            "single_word": false,
639            "lstrip": false,
640            "rstrip": false,
641            "normalized": false,
642            "special": true
643        }]);
644        std::fs::write(&path2, serde_json::to_vec(&json2).unwrap()).unwrap();
645
646        let genuinely_added = BasetenTokenizer::from_file(path2.to_str().unwrap()).unwrap();
647        assert_eq!(genuinely_added.vocab_size(), Some(24));
648        assert_eq!(genuinely_added.token_to_id("<extra>").unwrap(), Some(23));
649    }
650}