1use std::{collections::HashSet, path::Path};
11
12use super::{
13 EncodeSegment, Encoding, Error, Result, TokenIdType, TokenizerOptions,
14 traits::{DecodeResult, Decoder, Encoder, Tokenizer},
15};
16
17pub struct BasetenTokenizer {
19 tokenizer: basetenkenizer::Tokenizer,
20 options: TokenizerOptions,
21 added_ids: HashSet<u32>,
24}
25
26impl BasetenTokenizer {
27 pub fn from_file(path: &str) -> Result<Self> {
29 let path = Path::new(path);
30 let raw = std::fs::read_to_string(path)
31 .map_err(|e| Error::msg(format!("Error reading Baseten tokenizer: {e}")))?;
32 let mut json: serde_json::Value = serde_json::from_str(&raw)
33 .map_err(|e| Error::msg(format!("Error parsing Baseten tokenizer: {e}")))?;
34 if let Some(parent) = path.parent() {
35 merge_special_tokens_from_config(&mut json, parent);
36 }
37 let tokenizer = basetenkenizer::Tokenizer::from_json(json)
38 .map_err(|e| Error::msg(format!("Error loading Baseten tokenizer: {e}")))?;
39 let added_ids = tokenizer
40 .added_tokens()
41 .into_iter()
42 .flat_map(|added| added.iter().map(|info| info.id))
43 .collect();
44 Ok(Self {
45 tokenizer,
46 options: TokenizerOptions::default(),
47 added_ids,
48 })
49 }
50}
51
52fn merge_special_tokens_from_config(json: &mut serde_json::Value, model_dir: &Path) {
53 let config_path = model_dir.join("tokenizer_config.json");
54 let Ok(raw) = std::fs::read_to_string(&config_path) else {
55 return;
56 };
57 let config: serde_json::Value = match serde_json::from_str(&raw) {
58 Ok(value) => value,
59 Err(error) => {
60 tracing::debug!(
61 target: "tokenizer",
62 path = %config_path.display(),
63 error = %error,
64 "tokenizer_config.json parse failed; skipping special-token merge"
65 );
66 return;
67 }
68 };
69 let Some(decoder) = config
70 .get("added_tokens_decoder")
71 .and_then(serde_json::Value::as_object)
72 else {
73 return;
74 };
75
76 if json.get("added_tokens").is_none() {
77 json["added_tokens"] = serde_json::json!([]);
78 }
79 let Some(added_tokens) = json
80 .get_mut("added_tokens")
81 .and_then(serde_json::Value::as_array_mut)
82 else {
83 return;
84 };
85
86 for (id, spec) in decoder {
87 let Some(id) = id.parse::<u32>().ok() else {
88 continue;
89 };
90 let Some(spec) = spec.as_object() else {
91 continue;
92 };
93 if spec.get("special").and_then(serde_json::Value::as_bool) != Some(true) {
94 continue;
95 }
96 let Some(content) = spec
97 .get("content")
98 .and_then(serde_json::Value::as_str)
99 .filter(|content| !content.is_empty())
100 else {
101 continue;
102 };
103
104 if let Some(existing) = added_tokens
105 .iter_mut()
106 .find(|token| token.get("content").and_then(serde_json::Value::as_str) == Some(content))
107 {
108 existing["special"] = serde_json::Value::Bool(true);
109 continue;
110 }
111
112 let mut token = serde_json::Map::from_iter([
113 ("id".to_string(), serde_json::json!(id)),
114 ("content".to_string(), serde_json::json!(content)),
115 ("special".to_string(), serde_json::Value::Bool(true)),
116 ]);
117 for field in ["single_word", "lstrip", "rstrip", "normalized"] {
118 token.insert(
119 field.to_string(),
120 serde_json::Value::Bool(
121 spec.get(field)
122 .and_then(serde_json::Value::as_bool)
123 .unwrap_or(false),
124 ),
125 );
126 }
127 added_tokens.push(serde_json::Value::Object(token));
128 }
129}
130
131impl Encoder for BasetenTokenizer {
132 fn encode(&self, input: &str) -> Result<Encoding> {
133 let ids = self
134 .tokenizer
135 .encode_with_special_tokens(input, self.options.add_special_tokens)
136 .map_err(|e| Error::msg(format!("Baseten tokenizer encode error: {e}")))?;
137 Ok(Encoding::Sp(ids))
138 }
139
140 fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
141 self.tokenizer
142 .encode_batch(inputs, self.options.add_special_tokens)
143 .map(|ids| ids.into_iter().map(Encoding::Sp).collect())
144 .map_err(|e| Error::msg(format!("Baseten tokenizer batch encode error: {e}")))
145 }
146
147 fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding> {
148 let mut ids = Vec::new();
151 for segment in segments {
152 let start = ids.len();
153 ids.extend(
154 self.tokenizer
155 .encode_segments_tiktoken_safe([(segment.text, segment.allow_special)], false)
156 .map_err(|e| {
157 Error::msg(format!("Baseten tokenizer segment encode error: {e}"))
158 })?,
159 );
160 if !segment.allow_special {
161 crate::reject_control_ids_in_ordinary_segment(&ids[start..], &self.added_ids)?;
162 }
163 }
164 Ok(Encoding::Sp(
165 self.tokenizer
166 .post_process(ids, self.options.add_special_tokens),
167 ))
168 }
169}
170
171impl Decoder for BasetenTokenizer {
172 fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
173 self.tokenizer
174 .decode(token_ids, skip_special_tokens)
175 .map(DecodeResult::from)
176 .map_err(|e| Error::msg(format!("Baseten tokenizer decode error: {e}")))
177 }
178}
179
180impl Tokenizer for BasetenTokenizer {
181 fn validate_prefix_cache(&self) -> Result<()> {
182 if self.options.add_special_tokens {
183 return Err(Error::msg(
184 "Baseten tokenizers configured with add_special_tokens=true must remain uncached",
185 ));
186 }
187 Ok(())
188 }
189
190 fn with_options(mut self, options: TokenizerOptions) -> Self {
191 self.options = options;
192 self
193 }
194
195 fn vocab_size(&self) -> Option<usize> {
196 let model = self.tokenizer.model();
203 let model_size = model.vocab_size();
204 let extra = self.tokenizer.added_tokens().map_or(0, |added| {
205 added
206 .iter()
207 .filter(|info| model.token_to_id(info.content).is_none())
208 .count()
209 });
210 Some(model_size + extra)
211 }
212
213 fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
214 Ok(self.tokenizer.token_to_id(token))
215 }
216
217 fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
218 let Some(added_tokens) = self.tokenizer.added_tokens() else {
219 return Ok(Vec::new());
220 };
221 let mut ids: Vec<TokenIdType> = added_tokens
222 .iter()
223 .filter_map(|info| info.special.then_some(info.id))
224 .collect();
225 ids.sort_unstable();
226 Ok(ids)
227 }
228
229 fn num_special_tokens_added(&self) -> Result<usize> {
230 Ok(self.tokenizer.post_process(Vec::new(), true).len())
241 }
242}
243
244#[cfg(test)]
245mod tests {
246 use std::sync::Arc;
247
248 use super::*;
249 use crate::{
250 HuggingFaceTokenizer, Tokenizer as TokenizerWrapper, traits::Tokenizer as TokenizerTrait,
251 };
252
253 const TOKENIZER_PATH: &str = concat!(
254 env!("CARGO_MANIFEST_DIR"),
255 "/tests/data/minimal-bpe/tokenizer.json"
256 );
257
258 #[test]
259 fn encode_matches_hugging_face() {
260 let baseten = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
261 let hf = HuggingFaceTokenizer::from_file(TOKENIZER_PATH).unwrap();
262
263 for text in ["Hello, world!", "Hello", " world", "He llo"] {
264 let baseten_ids = baseten.encode(text).unwrap();
265 let hf_ids = hf.encode(text).unwrap();
266 assert_eq!(
267 baseten_ids.token_ids(),
268 hf_ids.token_ids(),
269 "Baseten and Hugging Face must produce identical token IDs for '{text}'"
270 );
271 }
272 }
273
274 #[test]
275 fn batch_encode_matches_sequential_encode() {
276 let tokenizer = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
277 let inputs = ["Hello", " world", "Hello, world!"];
278 let batch = tokenizer.encode_batch(&inputs).unwrap();
279
280 for (encoding, input) in batch.iter().zip(inputs) {
281 let sequential = tokenizer.encode(input).unwrap();
282 assert_eq!(encoding.token_ids(), sequential.token_ids());
283 }
284 }
285
286 #[test]
287 fn encode_decode_roundtrip() {
288 let tokenizer = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
289 let encoding = tokenizer.encode("Hello, world!").unwrap();
290 let decoded = tokenizer.decode(encoding.token_ids(), true).unwrap();
291
292 assert_eq!(decoded.as_str(), "Hello, world!");
293 }
294
295 #[test]
296 fn works_with_decode_stream() {
297 let tokenizer = Arc::new(BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap());
298 let wrapper = TokenizerWrapper::from(tokenizer);
299 let prompt_ids = wrapper.encode("Hello").unwrap().token_ids().to_vec();
300 let continuation_ids = wrapper.encode(", world!").unwrap().token_ids().to_vec();
301 let mut stream = wrapper.decode_stream(&prompt_ids, true);
302 let mut accumulated = String::new();
303
304 for id in &continuation_ids {
305 if let Some(chunk) = stream.step(*id).unwrap() {
306 accumulated.push_str(&chunk);
307 }
308 }
309
310 let mut all_ids = prompt_ids.clone();
311 all_ids.extend_from_slice(&continuation_ids);
312 let full_text: String = wrapper.decode(&all_ids, true).unwrap().into();
313 let prompt_text: String = wrapper.decode(&prompt_ids, true).unwrap().into();
314 assert_eq!(accumulated, full_text[prompt_text.len()..]);
315 }
316
317 #[test]
318 fn prefix_cache_rejects_special_token_post_processing() {
319 let plain = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
320 assert!(plain.validate_prefix_cache().is_ok());
321
322 let with_special_tokens = BasetenTokenizer::from_file(TOKENIZER_PATH)
323 .unwrap()
324 .with_options(TokenizerOptions {
325 add_special_tokens: true,
326 });
327 assert!(with_special_tokens.validate_prefix_cache().is_err());
328 }
329
330 #[test]
331 fn segments_preserve_special_token_trust_boundary() {
332 let temp = tempfile::tempdir().unwrap();
333 let path = temp.path().join("tokenizer.json");
334 let mut json: serde_json::Value =
335 serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
336 let vocab = json["model"]["vocab"].as_object_mut().unwrap();
337 vocab.insert("<".to_string(), serde_json::json!(23));
338 vocab.insert(">".to_string(), serde_json::json!(24));
339 vocab.insert("c".to_string(), serde_json::json!(25));
340 json["added_tokens"] = serde_json::json!([{
341 "id": 26,
342 "content": "<ctl>",
343 "single_word": false,
344 "lstrip": false,
345 "rstrip": false,
346 "normalized": false,
347 "special": true
348 }]);
349 std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
350
351 let tokenizer: Arc<dyn TokenizerTrait> =
352 Arc::new(BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap());
353 let segments = [
354 EncodeSegment::new("Hello", true),
355 EncodeSegment::new("<ctl>", false),
356 EncodeSegment::new(" world!", true),
357 ];
358
359 let segmented = tokenizer.encode_segments(&segments).unwrap();
360 let flattened = tokenizer.encode("Hello<ctl> world!").unwrap();
361
362 assert_ne!(
363 segmented.token_ids(),
364 flattened.token_ids(),
365 "untrusted control-token-looking text must not become an added token"
366 );
367 assert!(flattened.token_ids().contains(&26));
368 assert!(!segmented.token_ids().contains(&26));
369 }
370
371 #[test]
372 fn segments_reject_ordinary_control_ids() {
373 let temp = tempfile::tempdir().unwrap();
376 let path = temp.path().join("tokenizer.json");
377 let mut json: serde_json::Value =
378 serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
379 json["added_tokens"] = serde_json::json!([{
380 "id": 18,
381 "content": "He",
382 "single_word": false,
383 "lstrip": false,
384 "rstrip": false,
385 "normalized": false,
386 "special": true
387 }]);
388 std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
389
390 let tokenizer: Arc<dyn TokenizerTrait> =
391 Arc::new(BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap());
392 let control = tokenizer
393 .encode_segments(&[EncodeSegment::new("He", true)])
394 .unwrap();
395 assert_eq!(control.token_ids(), &[18]);
396 let error = tokenizer
397 .encode_segments(&[EncodeSegment::new("He", false)])
398 .unwrap_err();
399 assert!(
400 error.to_string().contains("control-token id 18"),
401 "unexpected error: {error}"
402 );
403 }
404
405 #[test]
406 fn segments_honor_add_special_tokens_option() {
407 let temp = tempfile::tempdir().unwrap();
408 let path = temp.path().join("tokenizer.json");
409 let mut json: serde_json::Value =
410 serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
411 json["model"]["vocab"]["<bos>"] = serde_json::json!(23);
412 json["added_tokens"] = serde_json::json!([{
413 "id": 23,
414 "content": "<bos>",
415 "single_word": false,
416 "lstrip": false,
417 "rstrip": false,
418 "normalized": false,
419 "special": true
420 }]);
421 json["post_processor"] = serde_json::json!({
422 "type": "TemplateProcessing",
423 "single": [
424 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
425 {"Sequence": {"id": "A", "type_id": 0}}
426 ],
427 "pair": [
428 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
429 {"Sequence": {"id": "A", "type_id": 0}},
430 {"Sequence": {"id": "B", "type_id": 0}}
431 ],
432 "special_tokens": {
433 "<bos>": {"id": "<bos>", "ids": [23], "tokens": ["<bos>"]}
434 }
435 });
436 std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
437 let segments = [EncodeSegment::new("Hello", false)];
438
439 let plain = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
440 let plain_ids = plain
441 .encode_segments(&segments)
442 .unwrap()
443 .token_ids()
444 .to_vec();
445
446 let with_bos = BasetenTokenizer::from_file(path.to_str().unwrap())
447 .unwrap()
448 .with_options(TokenizerOptions {
449 add_special_tokens: true,
450 });
451 assert_eq!(
452 with_bos.encode_segments(&segments).unwrap().token_ids(),
453 [&[23], plain_ids.as_slice()].concat()
454 );
455 }
456
457 #[test]
458 fn num_special_tokens_added_reflects_bos_post_processor() {
459 let temp = tempfile::tempdir().unwrap();
460 let path = temp.path().join("tokenizer.json");
461 let mut json: serde_json::Value =
462 serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
463 json["model"]["vocab"]["<bos>"] = serde_json::json!(23);
464 json["added_tokens"] = serde_json::json!([{
465 "id": 23,
466 "content": "<bos>",
467 "single_word": false,
468 "lstrip": false,
469 "rstrip": false,
470 "normalized": false,
471 "special": true
472 }]);
473 json["post_processor"] = serde_json::json!({
474 "type": "TemplateProcessing",
475 "single": [
476 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
477 {"Sequence": {"id": "A", "type_id": 0}}
478 ],
479 "pair": [
480 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
481 {"Sequence": {"id": "A", "type_id": 0}},
482 {"Sequence": {"id": "B", "type_id": 0}}
483 ],
484 "special_tokens": {
485 "<bos>": {"id": "<bos>", "ids": [23], "tokens": ["<bos>"]}
486 }
487 });
488 std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
489
490 let with_bos = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
491 assert_eq!(with_bos.num_special_tokens_added().unwrap(), 1);
492
493 let plain = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
494 assert_eq!(plain.num_special_tokens_added().unwrap(), 0);
495 }
496
497 #[test]
498 fn num_special_tokens_added_is_length_independent_under_sequence_post_processor() {
499 let temp = tempfile::tempdir().unwrap();
500 let path = temp.path().join("tokenizer.json");
501 let mut json: serde_json::Value =
502 serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
503 json["model"]["vocab"]["<bos>"] = serde_json::json!(23);
504 json["model"]["vocab"]["<eos>"] = serde_json::json!(24);
505 json["added_tokens"] = serde_json::json!([
506 {
507 "id": 23,
508 "content": "<bos>",
509 "single_word": false,
510 "lstrip": false,
511 "rstrip": false,
512 "normalized": false,
513 "special": true
514 },
515 {
516 "id": 24,
517 "content": "<eos>",
518 "single_word": false,
519 "lstrip": false,
520 "rstrip": false,
521 "normalized": false,
522 "special": true
523 }
524 ]);
525 json["post_processor"] = serde_json::json!({
526 "type": "Sequence",
527 "processors": [
528 {
529 "type": "TemplateProcessing",
530 "single": [
531 {"SpecialToken": {"id": "<bos>", "type_id": 0}},
532 {"Sequence": {"id": "A", "type_id": 0}}
533 ],
534 "pair": [],
535 "special_tokens": {
536 "<bos>": {"id": "<bos>", "ids": [23], "tokens": ["<bos>"]}
537 }
538 },
539 {
540 "type": "TemplateProcessing",
541 "single": [
542 {"Sequence": {"id": "A", "type_id": 0}},
543 {"SpecialToken": {"id": "<eos>", "type_id": 0}}
544 ],
545 "pair": [],
546 "special_tokens": {
547 "<eos>": {"id": "<eos>", "ids": [24], "tokens": ["<eos>"]}
548 }
549 }
550 ]
551 });
552 std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
553
554 let tokenizer = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
555 assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 2);
556
557 let with_specials = tokenizer.with_options(TokenizerOptions {
558 add_special_tokens: true,
559 });
560 let plain = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
561
562 for text in ["h", "hello there world"] {
563 let without = plain.encode(text).unwrap().token_ids().len();
564 let with = with_specials.encode(text).unwrap().token_ids().len();
565 assert_eq!(
566 with - without,
567 2,
568 "'{text}' should grow by exactly num_special_tokens_added()"
569 );
570 }
571 }
572
573 #[test]
574 fn merges_config_only_special_tokens() {
575 let temp = tempfile::tempdir().unwrap();
576 let tokenizer_path = temp.path().join("tokenizer.json");
577 std::fs::copy(TOKENIZER_PATH, &tokenizer_path).unwrap();
578 std::fs::write(
579 temp.path().join("tokenizer_config.json"),
580 serde_json::json!({
581 "added_tokens_decoder": {
582 "23": {
583 "content": "<ctl>",
584 "special": true,
585 "single_word": false,
586 "lstrip": false,
587 "rstrip": false,
588 "normalized": false
589 }
590 }
591 })
592 .to_string(),
593 )
594 .unwrap();
595
596 let tokenizer = BasetenTokenizer::from_file(tokenizer_path.to_str().unwrap()).unwrap();
597 let encoding = tokenizer.encode("<ctl>").unwrap();
598 assert_eq!(encoding.token_ids(), &[23]);
599 assert_eq!(tokenizer.decode(&[23], false).unwrap().as_str(), "<ctl>");
600 assert_eq!(tokenizer.decode(&[23], true).unwrap().as_str(), "");
601 }
602
603 #[test]
604 fn vocab_introspection_accessors() {
605 let plain = BasetenTokenizer::from_file(TOKENIZER_PATH).unwrap();
606 assert_eq!(plain.vocab_size(), Some(23));
607 assert_eq!(plain.token_to_id("hello").unwrap(), None);
608 assert_eq!(plain.token_to_id("h").unwrap(), Some(10));
609 assert!(plain.special_token_ids().unwrap().is_empty());
610
611 let temp = tempfile::tempdir().unwrap();
612 let path = temp.path().join("tokenizer.json");
613 let mut json: serde_json::Value =
614 serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
615 json["model"]["vocab"]["<bos>"] = serde_json::json!(23);
616 json["added_tokens"] = serde_json::json!([{
617 "id": 23,
618 "content": "<bos>",
619 "single_word": false,
620 "lstrip": false,
621 "rstrip": false,
622 "normalized": false,
623 "special": true
624 }]);
625 std::fs::write(&path, serde_json::to_vec(&json).unwrap()).unwrap();
626
627 let with_added = BasetenTokenizer::from_file(path.to_str().unwrap()).unwrap();
628 assert_eq!(with_added.vocab_size(), Some(24));
629 assert_eq!(with_added.token_to_id("<bos>").unwrap(), Some(23));
630 assert_eq!(with_added.special_token_ids().unwrap(), vec![23]);
631
632 let path2 = temp.path().join("tokenizer2.json");
633 let mut json2: serde_json::Value =
634 serde_json::from_str(&std::fs::read_to_string(TOKENIZER_PATH).unwrap()).unwrap();
635 json2["added_tokens"] = serde_json::json!([{
636 "id": 23,
637 "content": "<extra>",
638 "single_word": false,
639 "lstrip": false,
640 "rstrip": false,
641 "normalized": false,
642 "special": true
643 }]);
644 std::fs::write(&path2, serde_json::to_vec(&json2).unwrap()).unwrap();
645
646 let genuinely_added = BasetenTokenizer::from_file(path2.to_str().unwrap()).unwrap();
647 assert_eq!(genuinely_added.vocab_size(), Some(24));
648 assert_eq!(genuinely_added.token_to_id("<extra>").unwrap(), Some(23));
649 }
650}