1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
// SPDX-License-Identifier: MIT OR Apache-2.0
//! 2026-09-26: `ChatTokenizer`: a Hugging Face tokenizer, the chat template prompts are
//! rendered with (minijinja, or the DeepSeek-V4 message encoder), and a streaming decoder.
//!
//! Owner: server (tokenizer).
//! Invariants: none beyond the types.
use anyhow::Result;
use tokenizers::Tokenizer;
/// 2026-09-26: Return a copy of `messages` in which every string
/// `tool_calls[*].function.arguments` that parses as JSON is replaced by the parsed value;
/// anything else is left as it is. Templates iterate the arguments as a map, e.g.
/// `_args.items()` in jinja-templates/openai/minimax_m2.jinja.
fn normalize_tool_call_arguments(messages: &[serde_json::Value]) -> Vec<serde_json::Value> {
let mut total_parsed = 0usize;
let mut total_seen = 0usize;
let out: Vec<_> = messages
.iter()
.map(|msg| {
let mut msg = msg.clone();
let Some(tool_calls) = msg.get_mut("tool_calls").and_then(|v| v.as_array_mut()) else {
return msg;
};
for tc in tool_calls.iter_mut() {
let Some(function) = tc.get_mut("function") else {
continue;
};
let Some(args) = function.get_mut("arguments") else {
continue;
};
total_seen += 1;
let parsed_owned = if let Some(s) = args.as_str() {
serde_json::from_str::<serde_json::Value>(s).ok()
} else {
None
};
if let Some(parsed) = parsed_owned {
*args = parsed;
total_parsed += 1;
}
}
msg
})
.collect();
if total_seen > 0 {
tracing::debug!(
"F76 normalize: {}/{} tool_call arguments parsed string→dict",
total_parsed,
total_seen,
);
}
out
}
mod chat_impl;
pub(crate) mod chat_render;
pub use chat_render::ThinkingVars;
mod deepseek_v4;
mod heap_trim;
pub use heap_trim::HeapTrimPolicy;
pub(crate) mod jinja_helpers;
mod kimi_k3;
mod message_preprocess;
pub(crate) use message_preprocess::{
autoclose_assistant_think, remap_developer_role, resolve_think_control,
};
/// 2026-10-08: Where `ChatTokenizer::from_model_dir` takes the chat template from. Each
/// variant falls back to the checkpoint's own template, then to ChatML, when it finds no
/// file of its own.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TemplateSource<'a> {
/// 2026-10-08: `jinja-templates/{model_type}.jinja` (and `jinja-templates/openai/` for
/// the OpenAI apply paths) when present: the default.
OverrideDir,
/// 2026-10-08: The checkpoint's own template; only the `openai/` variant directory is
/// still read (`--disable-template-overrides`).
Checkpoint,
/// 2026-10-08: This file, for every apply path; neither override directory is read
/// (`--chat-template`). A file that cannot be read fails the load.
File(&'a std::path::Path),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ChatEncoding {
Jinja,
DeepseekV4,
KimiK3XtmlUnsupported,
}
pub struct ChatTokenizer {
tokenizer: Tokenizer,
eos_token_id: u32,
supports_thinking: bool,
chat_encoding: ChatEncoding,
/// 2026-09-26: The checkpoint's own Qwen template renders the tool schema, so the parser's
/// tool prompt is not added (`checkpoint_owns_qwen_tool_prompt` in chat_impl.rs).
native_qwen_tool_template: bool,
/// 2026-09-26: Template source `jinja_env` was built from: the override, the model's own
/// template, or the ChatML default; empty for an official Kimi K3 checkpoint.
#[allow(dead_code)]
chat_template: String,
/// 2026-09-26: Environment holding `chat_template`, compiled once in `from_model_dir`.
jinja_env: minijinja::Environment<'static>,
/// 2026-09-26: Environment for `jinja-templates/openai/{model_type}.jinja` when that file
/// exists and compiles; without it the OpenAI apply paths use `jinja_env`.
openai_jinja_env: Option<minijinja::Environment<'static>>,
/// 2026-10-17: Heap trim after a long encode (`--tokenizer-trim-min-tokens`).
heap_trim: HeapTrimPolicy,
}
/// 2026-09-26: `tokenizers::DecodeStream` with its generic parameters fixed.
pub struct StreamingDecoder<'a> {
inner: tokenizers::DecodeStream<
'a,
tokenizers::models::ModelWrapper,
tokenizers::normalizers::NormalizerWrapper,
tokenizers::pre_tokenizers::PreTokenizerWrapper,
tokenizers::processors::PostProcessorWrapper,
tokenizers::decoders::DecoderWrapper,
>,
}
impl StreamingDecoder<'_> {
/// 2026-09-26: Feed one token. `Ok(Some(text))` returns the new text once the decode has
/// grown and does not end in U+FFFD; `Ok(None)` otherwise.
pub fn step(&mut self, id: u32) -> Result<Option<String>> {
self.inner
.step(id)
.map_err(|e| anyhow::anyhow!("Streaming decode error: {e}"))
}
}
#[cfg(test)]
mod tests;