Skip to main content

ferrite_model/
tokenizer.rs

1mod bpe;
2
3use crate::gguf::{GgufError, GgufFile, MetadataValue, MetadataValueType};
4use std::fmt;
5
6#[derive(Clone, Debug, PartialEq, Eq)]
7/// A tokenizer constructed from a GGUF tokenizer metadata table.
8///
9/// Tokenizers with merge metadata use byte-pair encoding. Tokenizers without
10/// merges use deterministic longest-prefix atomic token matching.
11pub struct GgufTokenizer {
12    model: TokenizerModel,
13    tokens: Vec<String>,
14    token_types: Vec<TokenType>,
15    merges: Vec<String>,
16    bpe_metadata: Option<bpe::BpeMetadata>,
17    eos_token_id: Option<usize>,
18}
19
20impl GgufTokenizer {
21    /// Builds a tokenizer from validated GGUF metadata.
22    ///
23    /// # Errors
24    ///
25    /// Returns an error when required tokenizer metadata is missing, has the
26    /// wrong type, contains inconsistent lengths, or has invalid BPE merges.
27    pub fn from_gguf(file: &GgufFile) -> Result<Self, TokenizerError> {
28        let model = match metadata_string(file, "tokenizer.ggml.model")? {
29            "llama" => TokenizerModel::Llama,
30            other => TokenizerModel::Other(other.to_owned()),
31        };
32        let tokens = metadata_string_array(file, "tokenizer.ggml.tokens")?;
33        let token_types = match metadata_i32_array(file, "tokenizer.ggml.token_type")? {
34            Some(values) => {
35                if values.len() != tokens.len() {
36                    return Err(TokenizerError::new(format!(
37                        "token_type length {} does not match tokens length {}",
38                        values.len(),
39                        tokens.len()
40                    )));
41                }
42                values.into_iter().map(TokenType::from_gguf).collect()
43            }
44            None => vec![TokenType::Normal; tokens.len()],
45        };
46
47        let merges = metadata_optional_string_array(file, "tokenizer.ggml.merges")?;
48        let bpe_metadata = if merges.is_empty() {
49            None
50        } else {
51            Some(bpe::BpeMetadata::new(&tokens, &merges)?)
52        };
53
54        Ok(Self {
55            model,
56            tokens,
57            token_types,
58            merges,
59            bpe_metadata,
60            eos_token_id: metadata_optional_usize(file, "tokenizer.ggml.eos_token_id")?,
61        })
62    }
63
64    /// Returns the tokenizer model identifier.
65    pub fn model(&self) -> TokenizerModel {
66        self.model.clone()
67    }
68
69    /// Returns the vocabulary size.
70    pub fn len(&self) -> usize {
71        self.tokens.len()
72    }
73
74    /// Returns `true` when the vocabulary contains no tokens.
75    pub fn is_empty(&self) -> bool {
76        self.tokens.is_empty()
77    }
78
79    /// Returns the token text for `id`, or `None` when the ID is out of range.
80    pub fn token(&self, id: usize) -> Option<&str> {
81        self.tokens.get(id).map(String::as_str)
82    }
83
84    /// Returns the GGUF token classification for `id`.
85    pub fn token_type(&self, id: usize) -> Option<TokenType> {
86        self.token_types.get(id).copied()
87    }
88
89    /// Returns the configured end-of-sequence token ID, when present.
90    pub fn eos_token_id(&self) -> Option<usize> {
91        self.eos_token_id
92    }
93
94    /// Decodes token IDs to UTF-8 text.
95    ///
96    /// # Errors
97    ///
98    /// Returns an error when an ID is out of range or BPE byte tokens do not
99    /// form valid UTF-8.
100    pub fn decode(&self, ids: &[usize]) -> Result<String, TokenizerError> {
101        if !self.merges.is_empty() {
102            return bpe::decode(ids, &self.tokens);
103        }
104
105        let mut output = String::new();
106        for id in ids {
107            let token = self
108                .tokens
109                .get(*id)
110                .ok_or_else(|| TokenizerError::new(format!("token id {id} is out of bounds")))?;
111            output.push_str(token);
112        }
113        Ok(output)
114    }
115
116    /// Decodes token IDs when they form complete UTF-8.
117    ///
118    /// Returns `Ok(None)` when another token may complete a partial UTF-8 byte
119    /// sequence.
120    ///
121    /// # Errors
122    ///
123    /// Returns an error for invalid token IDs or decoding failures other than
124    /// an incomplete UTF-8 suffix.
125    pub fn decode_if_complete(&self, ids: &[usize]) -> Result<Option<String>, TokenizerError> {
126        match self.decode(ids) {
127            Ok(text) => Ok(Some(text)),
128            Err(error) if error.is_incomplete_utf8() => Ok(None),
129            Err(error) => Err(error),
130        }
131    }
132
133    /// Encodes text with longest-prefix atomic token matching.
134    ///
135    /// # Errors
136    ///
137    /// Returns an error when no vocabulary token matches the remaining input.
138    pub fn encode_atomic(&self, input: &str) -> Result<Vec<usize>, TokenizerError> {
139        self.encode_atomic_with_cancellation(input, || TokenizationControl::Continue)
140    }
141
142    /// Encodes atomically while polling a caller-provided cancellation hook.
143    ///
144    /// The hook is called before work begins and before every token match.
145    ///
146    /// # Errors
147    ///
148    /// Returns an error when cancellation is requested or no vocabulary token
149    /// matches the remaining input.
150    pub fn encode_atomic_with_cancellation(
151        &self,
152        input: &str,
153        mut on_cancellation_poll: impl FnMut() -> TokenizationControl,
154    ) -> Result<Vec<usize>, TokenizerError> {
155        let mut output = Vec::new();
156        let mut cursor = 0usize;
157        if on_cancellation_poll() == TokenizationControl::Cancel {
158            return Err(TokenizerError::cancelled());
159        }
160        while cursor < input.len() {
161            if on_cancellation_poll() == TokenizationControl::Cancel {
162                return Err(TokenizerError::cancelled());
163            }
164            let suffix = &input[cursor..];
165            let Some((id, token)) = self.longest_token_prefix(suffix) else {
166                return Err(TokenizerError::new(format!(
167                    "no atomic token matches input at byte offset {cursor}"
168                )));
169            };
170            output.push(id);
171            cursor += token.len();
172        }
173        Ok(output)
174    }
175
176    /// Encodes text with the tokenizer's configured algorithm.
177    ///
178    /// GGUF tokenizers with merge metadata use BPE, while tokenizers without
179    /// merges use atomic longest-prefix matching.
180    ///
181    /// # Errors
182    ///
183    /// Returns an error when the input cannot be represented by the vocabulary
184    /// or the tokenizer metadata is invalid.
185    pub fn encode(&self, input: &str) -> Result<Vec<usize>, TokenizerError> {
186        self.encode_with_cancellation(input, || TokenizationControl::Continue)
187    }
188
189    /// Encodes with the configured algorithm and cancellation polling.
190    ///
191    /// # Errors
192    ///
193    /// Returns an error when cancellation is requested, the input cannot be
194    /// represented, or required tokenizer metadata is unavailable.
195    pub fn encode_with_cancellation(
196        &self,
197        input: &str,
198        on_cancellation_poll: impl FnMut() -> TokenizationControl,
199    ) -> Result<Vec<usize>, TokenizerError> {
200        if self.merges.is_empty() {
201            self.encode_atomic_with_cancellation(input, on_cancellation_poll)
202        } else {
203            self.encode_bpe_with_cancellation(input, on_cancellation_poll)
204        }
205    }
206
207    /// Encodes text using byte-pair encoding metadata.
208    ///
209    /// # Errors
210    ///
211    /// Returns an error when merge metadata is absent or the input cannot be
212    /// represented by the BPE vocabulary.
213    pub fn encode_bpe(&self, input: &str) -> Result<Vec<usize>, TokenizerError> {
214        self.encode_bpe_with_cancellation(input, || TokenizationControl::Continue)
215    }
216
217    /// Encodes with BPE while polling a caller-provided cancellation hook.
218    ///
219    /// # Errors
220    ///
221    /// Returns an error when cancellation is requested, merge metadata is
222    /// absent, or the input cannot be represented by the BPE vocabulary.
223    pub fn encode_bpe_with_cancellation(
224        &self,
225        input: &str,
226        on_cancellation_poll: impl FnMut() -> TokenizationControl,
227    ) -> Result<Vec<usize>, TokenizerError> {
228        let Some(metadata) = &self.bpe_metadata else {
229            return Err(TokenizerError::new("BPE tokenizer has no merge metadata"));
230        };
231        bpe::encode_with_cancellation(input, metadata, on_cancellation_poll)
232    }
233
234    fn longest_token_prefix(&self, input: &str) -> Option<(usize, &str)> {
235        self.tokens
236            .iter()
237            .enumerate()
238            .filter_map(|(id, token)| {
239                if token.is_empty() || !input.starts_with(token) {
240                    None
241                } else {
242                    Some((id, token.as_str()))
243                }
244            })
245            .max_by_key(|(_, token)| token.len())
246    }
247}
248
249#[derive(Clone, Debug, PartialEq, Eq)]
250/// The tokenizer model name declared by GGUF metadata.
251pub enum TokenizerModel {
252    /// The `llama` tokenizer model.
253    Llama,
254    /// Any tokenizer model identifier that Ferrite does not classify specially.
255    Other(String),
256}
257
258#[derive(Clone, Copy, Debug, PartialEq, Eq)]
259/// A token classification from `tokenizer.ggml.token_type` metadata.
260pub enum TokenType {
261    /// A normal vocabulary token.
262    Normal,
263    /// The tokenizer's unknown-token marker.
264    Unknown,
265    /// A control token.
266    Control,
267    /// A user-defined token.
268    UserDefined,
269    /// A vocabulary entry reserved as unused.
270    Unused,
271    /// A token representing a raw byte.
272    Byte,
273    /// An unrecognized GGUF token-type code.
274    Other(i32),
275}
276
277#[derive(Clone, Copy, Debug, PartialEq, Eq)]
278/// The result requested by a tokenization cancellation hook.
279pub enum TokenizationControl {
280    /// Continue tokenization.
281    Continue,
282    /// Stop tokenization and return an error.
283    Cancel,
284}
285
286impl TokenType {
287    fn from_gguf(value: i32) -> Self {
288        match value {
289            1 => Self::Normal,
290            2 => Self::Unknown,
291            3 => Self::Control,
292            4 => Self::UserDefined,
293            5 => Self::Unused,
294            6 => Self::Byte,
295            other => Self::Other(other),
296        }
297    }
298}
299
300#[derive(Clone, Debug, PartialEq, Eq)]
301/// An error produced while building, encoding with, or decoding a tokenizer.
302pub struct TokenizerError {
303    message: String,
304    kind: TokenizerErrorKind,
305}
306
307#[derive(Clone, Copy, Debug, PartialEq, Eq)]
308enum TokenizerErrorKind {
309    Other,
310    IncompleteUtf8,
311}
312
313impl TokenizerError {
314    pub(crate) fn new(message: impl Into<String>) -> Self {
315        Self {
316            message: message.into(),
317            kind: TokenizerErrorKind::Other,
318        }
319    }
320
321    pub(crate) fn cancelled() -> Self {
322        Self::new("tokenization cancelled")
323    }
324
325    pub(crate) fn incomplete_utf8(message: impl Into<String>) -> Self {
326        Self {
327            message: message.into(),
328            kind: TokenizerErrorKind::IncompleteUtf8,
329        }
330    }
331
332    /// Returns `true` when decoding ended with a potentially completable UTF-8
333    /// byte sequence.
334    pub fn is_incomplete_utf8(&self) -> bool {
335        self.kind == TokenizerErrorKind::IncompleteUtf8
336    }
337}
338
339impl fmt::Display for TokenizerError {
340    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
341        formatter.write_str(&self.message)
342    }
343}
344
345impl std::error::Error for TokenizerError {}
346
347impl From<GgufError> for TokenizerError {
348    fn from(error: GgufError) -> Self {
349        Self::new(error.to_string())
350    }
351}
352
353fn metadata_string<'a>(file: &'a GgufFile, key: &str) -> Result<&'a str, TokenizerError> {
354    match file.metadata.get(key) {
355        Some(MetadataValue::String(value)) => Ok(value),
356        Some(other) => Err(TokenizerError::new(format!(
357            "{key} must be a string, found {other:?}"
358        ))),
359        None => Err(TokenizerError::new(format!("missing metadata {key}"))),
360    }
361}
362
363fn metadata_string_array(file: &GgufFile, key: &str) -> Result<Vec<String>, TokenizerError> {
364    match file.metadata.get(key) {
365        Some(MetadataValue::Array {
366            value_type: MetadataValueType::String,
367            values,
368        }) => values
369            .iter()
370            .map(|value| match value {
371                MetadataValue::String(token) => Ok(token.clone()),
372                other => Err(TokenizerError::new(format!(
373                    "{key} contains non-string array value {other:?}"
374                ))),
375            })
376            .collect(),
377        Some(other) => Err(TokenizerError::new(format!(
378            "{key} must be a string array, found {other:?}"
379        ))),
380        None => Err(TokenizerError::new(format!("missing metadata {key}"))),
381    }
382}
383
384fn metadata_optional_string_array(
385    file: &GgufFile,
386    key: &str,
387) -> Result<Vec<String>, TokenizerError> {
388    match file.metadata.get(key) {
389        Some(_) => metadata_string_array(file, key),
390        None => Ok(Vec::new()),
391    }
392}
393
394fn metadata_i32_array(file: &GgufFile, key: &str) -> Result<Option<Vec<i32>>, TokenizerError> {
395    match file.metadata.get(key) {
396        Some(MetadataValue::Array {
397            value_type: MetadataValueType::Int32,
398            values,
399        }) => values
400            .iter()
401            .map(|value| match value {
402                MetadataValue::Int32(token_type) => Ok(*token_type),
403                other => Err(TokenizerError::new(format!(
404                    "{key} contains non-int32 array value {other:?}"
405                ))),
406            })
407            .collect::<Result<Vec<_>, _>>()
408            .map(Some),
409        Some(other) => Err(TokenizerError::new(format!(
410            "{key} must be an int32 array, found {other:?}"
411        ))),
412        None => Ok(None),
413    }
414}
415
416fn metadata_optional_usize(file: &GgufFile, key: &str) -> Result<Option<usize>, TokenizerError> {
417    let Some(value) = file.metadata.get(key) else {
418        return Ok(None);
419    };
420    let value = match value {
421        MetadataValue::UInt32(value) => u64::from(*value),
422        MetadataValue::UInt64(value) => *value,
423        other => {
424            return Err(TokenizerError::new(format!(
425                "{key} must be a uint32 or uint64, found {other:?}"
426            )));
427        }
428    };
429    usize::try_from(value)
430        .map(Some)
431        .map_err(|_error| TokenizerError::new(format!("{key} value {value} does not fit usize")))
432}