use std::{collections::HashSet, path::Path, sync::OnceLock};
use tokenizers::tokenizer::{
AddedToken, AddedVocabulary, PostProcessor as _, Tokenizer as HfTokenizer,
};
use super::{
Encoding, Error, Result, TokenIdType, TokenizerOptions,
traits::{DecodeResult, Decoder, Encoder, Tokenizer},
};
pub struct HuggingFaceTokenizer {
tokenizer: HfTokenizer,
segment_tokenizers: OnceLock<(HfTokenizer, HfTokenizer)>,
byte_fallback_ids: HashSet<u32>,
special_ids: HashSet<u32>,
added_ids: HashSet<u32>,
options: TokenizerOptions,
}
impl HuggingFaceTokenizer {
pub(crate) fn added_ids(&self) -> &HashSet<u32> {
&self.added_ids
}
pub fn from_file(model_name: &str) -> Result<Self> {
let mut tokenizer = HfTokenizer::from_file(model_name)
.map_err(|err| Error::msg(format!("Error loading tokenizer: {}", err)))?;
if let Some(parent) = Path::new(model_name).parent() {
merge_special_tokens_from_config(&mut tokenizer, parent);
}
Ok(Self::from_tokenizer(tokenizer))
}
pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self {
let has_byte_fallback = tokenizer.get_decoder().is_some_and(|decoder| {
fn contains(value: &serde_json::Value) -> bool {
value["type"] == "ByteFallback"
|| value["decoders"]
.as_array()
.is_some_and(|items| items.iter().any(contains))
}
serde_json::to_value(decoder).is_ok_and(|value| contains(&value))
});
let byte_fallback_ids = if has_byte_fallback {
tokenizer
.get_vocab(true)
.into_iter()
.filter_map(|(token, id)| {
(token.len() == 6
&& token.starts_with("<0x")
&& token.ends_with('>')
&& u8::from_str_radix(&token[3..5], 16).is_ok())
.then_some(id)
})
.collect()
} else {
HashSet::new()
};
let added_tokens = tokenizer.get_added_tokens_decoder();
let special_ids = added_tokens
.iter()
.filter_map(|(id, token)| token.special.then_some(*id))
.collect();
let added_ids = added_tokens.into_keys().collect();
HuggingFaceTokenizer {
tokenizer,
segment_tokenizers: OnceLock::new(),
byte_fallback_ids,
special_ids,
added_ids,
options: TokenizerOptions::default(),
}
}
pub fn from_tokenizer_with_model_dir(tokenizer: HfTokenizer, model_dir: &Path) -> Self {
let mut tokenizer = tokenizer;
merge_special_tokens_from_config(&mut tokenizer, model_dir);
Self::from_tokenizer(tokenizer)
}
}
pub fn merge_special_tokens_from_config(tokenizer: &mut HfTokenizer, model_dir: &Path) {
let cfg_path = model_dir.join("tokenizer_config.json");
let Ok(raw) = std::fs::read_to_string(&cfg_path) else {
return;
};
let cfg: serde_json::Value = match serde_json::from_str(&raw) {
Ok(v) => v,
Err(e) => {
tracing::debug!(
target: "tokenizer",
path = %cfg_path.display(),
error = %e,
"tokenizer_config.json parse failed; skipping special-token merge"
);
return;
}
};
let Some(decoder) = cfg.get("added_tokens_decoder").and_then(|v| v.as_object()) else {
return;
};
let mut to_add: Vec<AddedToken> = Vec::new();
for (_id, spec) in decoder {
let obj = match spec.as_object() {
Some(o) => o,
None => continue,
};
if obj.get("special").and_then(|v| v.as_bool()) != Some(true) {
continue;
}
let Some(content) = obj.get("content").and_then(|v| v.as_str()) else {
continue;
};
if content.is_empty() {
continue;
}
let single_word = obj
.get("single_word")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let lstrip = obj.get("lstrip").and_then(|v| v.as_bool()).unwrap_or(false);
let rstrip = obj.get("rstrip").and_then(|v| v.as_bool()).unwrap_or(false);
let normalized = obj
.get("normalized")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let token = AddedToken::from(content.to_string(), true)
.single_word(single_word)
.lstrip(lstrip)
.rstrip(rstrip)
.normalized(normalized);
to_add.push(token);
}
if to_add.is_empty() {
return;
}
let added = match tokenizer.add_special_tokens(to_add.iter().cloned()) {
Ok(added) => added,
Err(e) => {
tracing::warn!(
target: "tokenizer",
path = %cfg_path.display(),
error = %e,
"special-token merge from tokenizer_config.json failed; skipping"
);
return;
}
};
if added > 0 {
let promoted: Vec<&str> = to_add.iter().map(|t| t.content.as_str()).collect();
tracing::warn!(
target: "tokenizer",
path = %cfg_path.display(),
added,
candidates = to_add.len(),
promoted = ?promoted,
"merged additional special tokens from tokenizer_config.json"
);
}
}
impl Encoder for HuggingFaceTokenizer {
fn encode(&self, input: &str) -> Result<Encoding> {
let encoding = self
.tokenizer
.encode(input, self.options.add_special_tokens)
.map_err(|err| Error::msg(format!("Error tokenizing input: {err}")))?;
Ok(Encoding::Hf(Box::new(encoding)))
}
fn encode_segments(&self, segments: &[super::EncodeSegment<'_>]) -> Result<Encoding> {
let (trusted, ordinary) = self.segment_tokenizers.get_or_init(|| {
let mut trusted = self.tokenizer.clone();
trusted.with_padding(None);
trusted
.with_truncation(None)
.expect("removing truncation cannot fail");
trusted.with_post_processor(None::<tokenizers::processors::PostProcessorWrapper>);
let mut ordinary = trusted.clone();
ordinary.with_added_vocabulary(AddedVocabulary::new());
(trusted, ordinary)
});
let encodings = segments
.iter()
.map(|segment| {
let tokenizer = if segment.allow_special {
trusted
} else {
ordinary
};
let encoding = tokenizer
.encode(segment.text, false)
.map_err(|error| Error::msg(format!("Error tokenizing segment: {error}")))?;
if !segment.allow_special {
crate::reject_control_ids_in_ordinary_segment(
encoding.get_ids(),
&self.added_ids,
)?;
}
Ok(encoding)
})
.collect::<Result<Vec<_>>>()?;
let merged = tokenizers::Encoding::merge(encodings, true);
let encoding = self
.tokenizer
.post_process(merged, None, self.options.add_special_tokens)
.map_err(|error| Error::msg(format!("Error post-processing segments: {error}")))?;
Ok(Encoding::Hf(Box::new(encoding)))
}
fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>> {
let hf_encodings = self
.tokenizer
.encode_batch(inputs.to_vec(), self.options.add_special_tokens)
.map_err(|err| Error::msg(format!("Error batch tokenizing input: {err}")))?;
let encodings = hf_encodings
.into_iter()
.map(|enc| Encoding::Hf(Box::new(enc)))
.collect();
Ok(encodings)
}
}
impl Decoder for HuggingFaceTokenizer {
fn has_unstable_suffix(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> bool {
token_ids
.iter()
.rev()
.find(|id| !skip_special_tokens || !self.special_ids.contains(id))
.is_some_and(|id| self.byte_fallback_ids.contains(id))
}
fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result<DecodeResult> {
let text = self
.tokenizer
.decode(token_ids, skip_special_tokens)
.map_err(|err| Error::msg(format!("Error de-tokenizing input: {err}")))?;
Ok(text.into())
}
}
impl Tokenizer for HuggingFaceTokenizer {
fn validate_prefix_cache(&self) -> Result<()> {
if self.options.add_special_tokens {
return Err(Error::msg(
"HuggingFace tokenizers configured with add_special_tokens=true must remain uncached",
));
}
if self.tokenizer.get_truncation().is_some() || self.tokenizer.get_padding().is_some() {
return Err(Error::msg(
"HuggingFace tokenizers with truncation or padding must remain uncached",
));
}
Ok(())
}
fn validate_segment_prefix_cache(&self) -> Result<()> {
self.validate_prefix_cache()
}
fn with_options(mut self, options: TokenizerOptions) -> Self {
self.options = options;
self
}
fn vocab_size(&self) -> Option<usize> {
Some(self.tokenizer.get_vocab_size(true))
}
fn token_to_id(&self, token: &str) -> Result<Option<TokenIdType>> {
Ok(self.tokenizer.token_to_id(token))
}
fn special_token_ids(&self) -> Result<Vec<TokenIdType>> {
let mut ids: Vec<TokenIdType> = self
.tokenizer
.get_added_tokens_decoder()
.into_iter()
.filter_map(|(id, token)| token.special.then_some(id))
.collect();
ids.sort_unstable();
Ok(ids)
}
fn num_special_tokens_added(&self) -> Result<usize> {
Ok(self
.tokenizer
.get_post_processor()
.map_or(0, |processor| processor.added_tokens(false)))
}
}
impl From<HfTokenizer> for HuggingFaceTokenizer {
fn from(tokenizer: HfTokenizer) -> Self {
Self::from_tokenizer(tokenizer)
}
}
#[cfg(test)]
mod byte_fallback_stream_tests {
use super::*;
fn tokenizer() -> super::super::Tokenizer {
let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
"version": "1.0", "truncation": null, "padding": null,
"added_tokens": [{"id": 6, "content": "<eos>", "special": true,
"single_word": false, "lstrip": false, "rstrip": false, "normalized": false}],
"normalizer": null, "pre_tokenizer": null, "post_processor": null,
"decoder": {"type": "Sequence", "decoders": [
{"type": "ByteFallback"}, {"type": "Fuse"}]},
"model": {"type": "BPE", "vocab": {
"<0x61>": 0, "<0xF5>": 1, "<0xC3>": 2, "<0xA9>": 3,
" hello": 4, "!": 5, "<eos>": 6}, "merges": [], "byte_fallback": true}
}))
.unwrap();
std::sync::Arc::new(HuggingFaceTokenizer::from_tokenizer(hf)).into()
}
#[test]
fn byte_runs_match_full_decode() {
let tokenizer = tokenizer();
for (ids, skip) in [
(vec![0, 1, 4], false),
(vec![2, 3, 1, 4], false),
(vec![2, 3, 5], false),
(vec![0], false),
(vec![2, 3], false),
(vec![2], false),
(vec![0, 1], false),
(vec![0, 6, 1, 4], false),
(vec![0, 6, 1, 4], true),
] {
let expected: String = tokenizer.decode(&ids, skip).unwrap().into();
let mut stream = tokenizer.decode_stream(&[], skip);
let mut actual = String::new();
for id in &ids {
actual.push_str(&stream.step(*id).unwrap().unwrap_or_default());
}
actual.push_str(&stream.finish().unwrap().unwrap_or_default());
assert_eq!(actual, expected, "ids={ids:?}, skip={skip}");
assert_eq!(stream.finish().unwrap(), None);
}
}
#[test]
fn only_byte_suffix_is_delayed_and_prompt_is_not_emitted() {
let tokenizer = tokenizer();
let mut stream = tokenizer.decode_stream(&[4], true);
assert_eq!(stream.step(5).unwrap().as_deref(), Some("!"));
assert_eq!(stream.step(0).unwrap(), None);
assert_eq!(stream.step(1).unwrap(), None);
assert_eq!(stream.step(4).unwrap().as_deref(), Some("�� hello"));
assert_eq!(stream.finish().unwrap(), None);
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::TempDir;
#[test]
fn merge_gate_round_trips_through_decode() {
const TOKENIZER_JSON: &str = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
],
"normalizer": null,
"pre_tokenizer": null,
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {"<unk>": 0, "hello": 1, "world": 2, "<|special_kept|>": 3, "<|special_dropped|>": 4},
"unk_token": "<unk>"
}
}"#;
const TOKENIZER_CONFIG_JSON: &str = r#"{
"added_tokens_decoder": {
"3": {"content": "<|special_kept|>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
"4": {"content": "<|special_dropped|>", "special": false, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
}
}"#;
let dir = TempDir::new().unwrap();
fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
fs::write(
dir.path().join("tokenizer_config.json"),
TOKENIZER_CONFIG_JSON,
)
.unwrap();
let mut tokenizer = HfTokenizer::from_file(dir.path().join("tokenizer.json")).unwrap();
merge_special_tokens_from_config(&mut tokenizer, dir.path());
let specials: Vec<String> = {
let mut v: Vec<String> = tokenizer
.get_added_tokens_decoder()
.values()
.filter(|t| t.special)
.map(|t| t.content.clone())
.collect();
v.sort();
v
};
assert_eq!(
specials,
vec!["<unk>".to_string(), "<|special_kept|>".to_string()],
"<|special_kept|> promoted; <|special_dropped|> stayed non-special"
);
let enc_kept = tokenizer.encode("<|special_kept|>", false).unwrap();
let decoded_strip = tokenizer.decode(enc_kept.get_ids(), true).unwrap();
assert!(
!decoded_strip.contains("<|special_kept|>"),
"promoted special:true token must be stripped under skip_special_tokens=true; got {decoded_strip:?}"
);
let enc_drop = tokenizer.encode("<|special_dropped|>", false).unwrap();
let decoded_keep = tokenizer.decode(enc_drop.get_ids(), true).unwrap();
assert!(
decoded_keep.contains("<|special_dropped|>"),
"non-promoted special:false token must survive skip_special_tokens=true; got {decoded_keep:?}"
);
}
#[test]
fn add_special_tokens_flag_controls_encode() {
const TOKENIZER_JSON: &str = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
{"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
],
"normalizer": null,
"pre_tokenizer": null,
"post_processor": {
"type": "TemplateProcessing",
"single": [
{"SpecialToken": {"id": "<bos>", "type_id": 0}},
{"Sequence": {"id": "A", "type_id": 0}}
],
"pair": [
{"SpecialToken": {"id": "<bos>", "type_id": 0}},
{"Sequence": {"id": "A", "type_id": 0}},
{"Sequence": {"id": "B", "type_id": 0}}
],
"special_tokens": {
"<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
}
},
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
"unk_token": "<unk>"
}
}"#;
let dir = TempDir::new().unwrap();
fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
let path = dir.path().join("tokenizer.json");
let path = path.to_str().unwrap();
let ids = |enc: &Encoding| match enc {
Encoding::Hf(e) => e.get_ids().to_vec(),
_ => panic!("expected Hf encoding"),
};
let plain = HuggingFaceTokenizer::from_file(path).unwrap();
assert_eq!(ids(&plain.encode("hello").unwrap()), vec![1]);
let with_bos =
HuggingFaceTokenizer::from_file(path)
.unwrap()
.with_options(TokenizerOptions {
add_special_tokens: true,
});
assert_eq!(ids(&with_bos.encode("hello").unwrap()), vec![3, 1]);
let batch = with_bos.encode_batch(&["hello", "world"]).unwrap();
assert_eq!(ids(&batch[0]), vec![3, 1]);
assert_eq!(ids(&batch[1]), vec![3, 2]);
use crate::Tokenizer as TokenizerWrapper;
let wrapper_plain = TokenizerWrapper::from_file(path).unwrap();
assert_eq!(ids(&wrapper_plain.encode("hello").unwrap()), vec![1]);
let wrapper_bos = TokenizerWrapper::from_file_with_options(
path,
TokenizerOptions {
add_special_tokens: true,
},
)
.unwrap();
assert_eq!(ids(&wrapper_bos.encode("hello").unwrap()), vec![3, 1]);
}
#[test]
fn truncation_or_padding_blocks_prefix_cache() {
let path = concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/data/sample-models/TinyLlama_v1.1/tokenizer.json"
);
let mut truncated = HfTokenizer::from_file(path).unwrap();
truncated
.with_truncation(Some(tokenizers::TruncationParams::default()))
.unwrap();
let mut padded = HfTokenizer::from_file(path).unwrap();
padded.with_padding(Some(tokenizers::PaddingParams::default()));
for hf in [truncated, padded] {
assert!(
HuggingFaceTokenizer::from_tokenizer(hf)
.validate_prefix_cache()
.is_err()
);
}
}
#[test]
fn vocab_introspection_accessors() {
const TOKENIZER_JSON: &str = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
],
"normalizer": null,
"pre_tokenizer": null,
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {"<unk>": 0, "hello": 1, "world": 2},
"unk_token": "<unk>"
}
}"#;
let dir = TempDir::new().unwrap();
fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
let path = dir.path().join("tokenizer.json");
let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
assert_eq!(tokenizer.vocab_size(), Some(3));
assert_eq!(tokenizer.token_to_id("hello").unwrap(), Some(1));
assert_eq!(tokenizer.special_token_ids().unwrap(), vec![0]);
assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 0);
}
#[test]
fn num_special_tokens_added_reflects_post_processor_additions() {
const TOKENIZER_JSON: &str = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{"id": 0, "content": "<unk>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false},
{"id": 3, "content": "<bos>", "special": true, "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}
],
"normalizer": null,
"pre_tokenizer": null,
"post_processor": {
"type": "TemplateProcessing",
"single": [
{"SpecialToken": {"id": "<bos>", "type_id": 0}},
{"Sequence": {"id": "A", "type_id": 0}}
],
"pair": [
{"SpecialToken": {"id": "<bos>", "type_id": 0}},
{"Sequence": {"id": "A", "type_id": 0}},
{"Sequence": {"id": "B", "type_id": 0}}
],
"special_tokens": {
"<bos>": {"id": "<bos>", "ids": [3], "tokens": ["<bos>"]}
}
},
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {"<unk>": 0, "hello": 1, "world": 2, "<bos>": 3},
"unk_token": "<unk>"
}
}"#;
let dir = TempDir::new().unwrap();
fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
let path = dir.path().join("tokenizer.json");
let tokenizer = HuggingFaceTokenizer::from_file(path.to_str().unwrap()).unwrap();
assert_eq!(tokenizer.num_special_tokens_added().unwrap(), 1);
}
}
#[cfg(test)]
mod segmented_tests {
use super::*;
use crate::EncodeSegment;
#[test]
fn segmented_hf_preserves_literal_controls_and_adds_bos_once() {
let mut vocab = serde_json::json!({"<unk>":0,"<|call|>":1,"<bos>":2});
for ch in "<|cal>".chars() {
let id = vocab.as_object().unwrap().len();
vocab[ch.to_string()] = serde_json::json!(id);
}
let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
"version":"1.0", "truncation":null, "padding":null,
"normalizer":null, "pre_tokenizer":null, "decoder":{"type":"Fuse"},
"added_tokens":[{"id":1,"content":"<|call|>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false},
{"id":2,"content":"<bos>","special":true,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"<bos>","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],
"pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"<bos>":{"id":"<bos>","ids":[2],"tokens":["<bos>"]}}},
"model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
})).unwrap();
for add_special_tokens in [false, true] {
let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf.clone())
.with_options(TokenizerOptions { add_special_tokens });
let encoding = tokenizer
.encode_segments(&[
EncodeSegment::control("<|call|>"),
EncodeSegment::ordinary("<|call|>"),
])
.unwrap();
let ids = encoding.token_ids();
assert_eq!(ids.iter().filter(|id| **id == 1).count(), 1);
assert_eq!(
ids.iter().filter(|id| **id == 2).count(),
usize::from(add_special_tokens)
);
let decoded: String = tokenizer.decode(ids, true).unwrap().into();
assert_eq!(decoded, "<|call|>");
assert_eq!(
tokenizer
.encode("<|call|>")
.unwrap()
.token_ids()
.iter()
.filter(|id| **id == 1)
.count(),
1
);
}
}
#[test]
fn segmented_hf_encodes_non_special_added_tokens_as_text() {
let mut vocab = serde_json::json!({"<unk>":0,"<think>":1});
for ch in "<think>".chars() {
let id = vocab.as_object().unwrap().len();
vocab[ch.to_string()] = serde_json::json!(id);
}
let hf: HfTokenizer = serde_json::from_value(serde_json::json!({
"version":"1.0", "truncation":null, "padding":null,
"normalizer":null, "pre_tokenizer":null, "post_processor":null, "decoder":{"type":"Fuse"},
"added_tokens":[{"id":1,"content":"<think>","special":false,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],
"model":{"type":"BPE","vocab":vocab,"merges":[],"unk_token":"<unk>"}
})).unwrap();
let tokenizer = HuggingFaceTokenizer::from_tokenizer(hf);
let ids = |segment| {
tokenizer
.encode_segments(&[segment])
.unwrap()
.token_ids()
.to_vec()
};
assert_eq!(ids(EncodeSegment::control("<think>")), [1]);
let ordinary = ids(EncodeSegment::ordinary("<think>"));
assert_eq!(ordinary.len(), "<think>".len());
assert!(!ordinary.contains(&1));
}
#[test]
fn segmented_hf_rejects_ordinary_control_ids() {
let mut json: serde_json::Value = serde_json::from_str(
&std::fs::read_to_string(concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/data/minimal-bpe/tokenizer.json"
))
.unwrap(),
)
.unwrap();
json["added_tokens"] = serde_json::json!([{
"id": 18,
"content": "He",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}]);
let tokenizer = HuggingFaceTokenizer::from_tokenizer(serde_json::from_value(json).unwrap());
let control = tokenizer
.encode_segments(&[EncodeSegment::control("He")])
.unwrap();
assert_eq!(control.token_ids(), &[18]);
let error = tokenizer
.encode_segments(&[EncodeSegment::ordinary("He")])
.unwrap_err();
assert!(
error.to_string().contains("control-token id 18"),
"unexpected error: {error}"
);
}
}