1mod bpe;
2
3use crate::gguf::{GgufError, GgufFile, MetadataValue, MetadataValueType};
4use std::fmt;
5
6#[derive(Clone, Debug, PartialEq, Eq)]
7pub struct GgufTokenizer {
12 model: TokenizerModel,
13 tokens: Vec<String>,
14 token_types: Vec<TokenType>,
15 merges: Vec<String>,
16 bpe_metadata: Option<bpe::BpeMetadata>,
17 eos_token_id: Option<usize>,
18}
19
20impl GgufTokenizer {
21 pub fn from_gguf(file: &GgufFile) -> Result<Self, TokenizerError> {
28 let model = match metadata_string(file, "tokenizer.ggml.model")? {
29 "llama" => TokenizerModel::Llama,
30 other => TokenizerModel::Other(other.to_owned()),
31 };
32 let tokens = metadata_string_array(file, "tokenizer.ggml.tokens")?;
33 let token_types = match metadata_i32_array(file, "tokenizer.ggml.token_type")? {
34 Some(values) => {
35 if values.len() != tokens.len() {
36 return Err(TokenizerError::new(format!(
37 "token_type length {} does not match tokens length {}",
38 values.len(),
39 tokens.len()
40 )));
41 }
42 values.into_iter().map(TokenType::from_gguf).collect()
43 }
44 None => vec![TokenType::Normal; tokens.len()],
45 };
46
47 let merges = metadata_optional_string_array(file, "tokenizer.ggml.merges")?;
48 let bpe_metadata = if merges.is_empty() {
49 None
50 } else {
51 Some(bpe::BpeMetadata::new(&tokens, &merges)?)
52 };
53
54 Ok(Self {
55 model,
56 tokens,
57 token_types,
58 merges,
59 bpe_metadata,
60 eos_token_id: metadata_optional_usize(file, "tokenizer.ggml.eos_token_id")?,
61 })
62 }
63
64 pub fn model(&self) -> TokenizerModel {
66 self.model.clone()
67 }
68
69 pub fn len(&self) -> usize {
71 self.tokens.len()
72 }
73
74 pub fn is_empty(&self) -> bool {
76 self.tokens.is_empty()
77 }
78
79 pub fn token(&self, id: usize) -> Option<&str> {
81 self.tokens.get(id).map(String::as_str)
82 }
83
84 pub fn token_type(&self, id: usize) -> Option<TokenType> {
86 self.token_types.get(id).copied()
87 }
88
89 pub fn eos_token_id(&self) -> Option<usize> {
91 self.eos_token_id
92 }
93
94 pub fn decode(&self, ids: &[usize]) -> Result<String, TokenizerError> {
101 if !self.merges.is_empty() {
102 return bpe::decode(ids, &self.tokens);
103 }
104
105 let mut output = String::new();
106 for id in ids {
107 let token = self
108 .tokens
109 .get(*id)
110 .ok_or_else(|| TokenizerError::new(format!("token id {id} is out of bounds")))?;
111 output.push_str(token);
112 }
113 Ok(output)
114 }
115
116 pub fn decode_if_complete(&self, ids: &[usize]) -> Result<Option<String>, TokenizerError> {
126 match self.decode(ids) {
127 Ok(text) => Ok(Some(text)),
128 Err(error) if error.is_incomplete_utf8() => Ok(None),
129 Err(error) => Err(error),
130 }
131 }
132
133 pub fn encode_atomic(&self, input: &str) -> Result<Vec<usize>, TokenizerError> {
139 self.encode_atomic_with_cancellation(input, || TokenizationControl::Continue)
140 }
141
142 pub fn encode_atomic_with_cancellation(
151 &self,
152 input: &str,
153 mut on_cancellation_poll: impl FnMut() -> TokenizationControl,
154 ) -> Result<Vec<usize>, TokenizerError> {
155 let mut output = Vec::new();
156 let mut cursor = 0usize;
157 if on_cancellation_poll() == TokenizationControl::Cancel {
158 return Err(TokenizerError::cancelled());
159 }
160 while cursor < input.len() {
161 if on_cancellation_poll() == TokenizationControl::Cancel {
162 return Err(TokenizerError::cancelled());
163 }
164 let suffix = &input[cursor..];
165 let Some((id, token)) = self.longest_token_prefix(suffix) else {
166 return Err(TokenizerError::new(format!(
167 "no atomic token matches input at byte offset {cursor}"
168 )));
169 };
170 output.push(id);
171 cursor += token.len();
172 }
173 Ok(output)
174 }
175
176 pub fn encode(&self, input: &str) -> Result<Vec<usize>, TokenizerError> {
186 self.encode_with_cancellation(input, || TokenizationControl::Continue)
187 }
188
189 pub fn encode_with_cancellation(
196 &self,
197 input: &str,
198 on_cancellation_poll: impl FnMut() -> TokenizationControl,
199 ) -> Result<Vec<usize>, TokenizerError> {
200 if self.merges.is_empty() {
201 self.encode_atomic_with_cancellation(input, on_cancellation_poll)
202 } else {
203 self.encode_bpe_with_cancellation(input, on_cancellation_poll)
204 }
205 }
206
207 pub fn encode_bpe(&self, input: &str) -> Result<Vec<usize>, TokenizerError> {
214 self.encode_bpe_with_cancellation(input, || TokenizationControl::Continue)
215 }
216
217 pub fn encode_bpe_with_cancellation(
224 &self,
225 input: &str,
226 on_cancellation_poll: impl FnMut() -> TokenizationControl,
227 ) -> Result<Vec<usize>, TokenizerError> {
228 let Some(metadata) = &self.bpe_metadata else {
229 return Err(TokenizerError::new("BPE tokenizer has no merge metadata"));
230 };
231 bpe::encode_with_cancellation(input, metadata, on_cancellation_poll)
232 }
233
234 fn longest_token_prefix(&self, input: &str) -> Option<(usize, &str)> {
235 self.tokens
236 .iter()
237 .enumerate()
238 .filter_map(|(id, token)| {
239 if token.is_empty() || !input.starts_with(token) {
240 None
241 } else {
242 Some((id, token.as_str()))
243 }
244 })
245 .max_by_key(|(_, token)| token.len())
246 }
247}
248
249#[derive(Clone, Debug, PartialEq, Eq)]
250pub enum TokenizerModel {
252 Llama,
254 Other(String),
256}
257
258#[derive(Clone, Copy, Debug, PartialEq, Eq)]
259pub enum TokenType {
261 Normal,
263 Unknown,
265 Control,
267 UserDefined,
269 Unused,
271 Byte,
273 Other(i32),
275}
276
277#[derive(Clone, Copy, Debug, PartialEq, Eq)]
278pub enum TokenizationControl {
280 Continue,
282 Cancel,
284}
285
286impl TokenType {
287 fn from_gguf(value: i32) -> Self {
288 match value {
289 1 => Self::Normal,
290 2 => Self::Unknown,
291 3 => Self::Control,
292 4 => Self::UserDefined,
293 5 => Self::Unused,
294 6 => Self::Byte,
295 other => Self::Other(other),
296 }
297 }
298}
299
300#[derive(Clone, Debug, PartialEq, Eq)]
301pub struct TokenizerError {
303 message: String,
304 kind: TokenizerErrorKind,
305}
306
307#[derive(Clone, Copy, Debug, PartialEq, Eq)]
308enum TokenizerErrorKind {
309 Other,
310 IncompleteUtf8,
311}
312
313impl TokenizerError {
314 pub(crate) fn new(message: impl Into<String>) -> Self {
315 Self {
316 message: message.into(),
317 kind: TokenizerErrorKind::Other,
318 }
319 }
320
321 pub(crate) fn cancelled() -> Self {
322 Self::new("tokenization cancelled")
323 }
324
325 pub(crate) fn incomplete_utf8(message: impl Into<String>) -> Self {
326 Self {
327 message: message.into(),
328 kind: TokenizerErrorKind::IncompleteUtf8,
329 }
330 }
331
332 pub fn is_incomplete_utf8(&self) -> bool {
335 self.kind == TokenizerErrorKind::IncompleteUtf8
336 }
337}
338
339impl fmt::Display for TokenizerError {
340 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
341 formatter.write_str(&self.message)
342 }
343}
344
345impl std::error::Error for TokenizerError {}
346
347impl From<GgufError> for TokenizerError {
348 fn from(error: GgufError) -> Self {
349 Self::new(error.to_string())
350 }
351}
352
353fn metadata_string<'a>(file: &'a GgufFile, key: &str) -> Result<&'a str, TokenizerError> {
354 match file.metadata.get(key) {
355 Some(MetadataValue::String(value)) => Ok(value),
356 Some(other) => Err(TokenizerError::new(format!(
357 "{key} must be a string, found {other:?}"
358 ))),
359 None => Err(TokenizerError::new(format!("missing metadata {key}"))),
360 }
361}
362
363fn metadata_string_array(file: &GgufFile, key: &str) -> Result<Vec<String>, TokenizerError> {
364 match file.metadata.get(key) {
365 Some(MetadataValue::Array {
366 value_type: MetadataValueType::String,
367 values,
368 }) => values
369 .iter()
370 .map(|value| match value {
371 MetadataValue::String(token) => Ok(token.clone()),
372 other => Err(TokenizerError::new(format!(
373 "{key} contains non-string array value {other:?}"
374 ))),
375 })
376 .collect(),
377 Some(other) => Err(TokenizerError::new(format!(
378 "{key} must be a string array, found {other:?}"
379 ))),
380 None => Err(TokenizerError::new(format!("missing metadata {key}"))),
381 }
382}
383
384fn metadata_optional_string_array(
385 file: &GgufFile,
386 key: &str,
387) -> Result<Vec<String>, TokenizerError> {
388 match file.metadata.get(key) {
389 Some(_) => metadata_string_array(file, key),
390 None => Ok(Vec::new()),
391 }
392}
393
394fn metadata_i32_array(file: &GgufFile, key: &str) -> Result<Option<Vec<i32>>, TokenizerError> {
395 match file.metadata.get(key) {
396 Some(MetadataValue::Array {
397 value_type: MetadataValueType::Int32,
398 values,
399 }) => values
400 .iter()
401 .map(|value| match value {
402 MetadataValue::Int32(token_type) => Ok(*token_type),
403 other => Err(TokenizerError::new(format!(
404 "{key} contains non-int32 array value {other:?}"
405 ))),
406 })
407 .collect::<Result<Vec<_>, _>>()
408 .map(Some),
409 Some(other) => Err(TokenizerError::new(format!(
410 "{key} must be an int32 array, found {other:?}"
411 ))),
412 None => Ok(None),
413 }
414}
415
416fn metadata_optional_usize(file: &GgufFile, key: &str) -> Result<Option<usize>, TokenizerError> {
417 let Some(value) = file.metadata.get(key) else {
418 return Ok(None);
419 };
420 let value = match value {
421 MetadataValue::UInt32(value) => u64::from(*value),
422 MetadataValue::UInt64(value) => *value,
423 other => {
424 return Err(TokenizerError::new(format!(
425 "{key} must be a uint32 or uint64, found {other:?}"
426 )));
427 }
428 };
429 usize::try_from(value)
430 .map(Some)
431 .map_err(|_error| TokenizerError::new(format!("{key} value {value} does not fit usize")))
432}