snaptokens 0.2.6

Fast BPE tokenization from local Hugging Face tokenizer.json files
Documentation
use serde::{Deserialize, Deserializer, de};
use serde_json::Value;

use crate::{models, pre_tokenizers};

/// An entry in the `added_tokens` array of `tokenizer.json`.
#[derive(Clone, Debug, Deserialize)]
pub struct AddedTokenConfig {
    /// Token ID.
    pub id: u32,
    /// Literal text content that triggers this token.
    pub content: String,
    /// Whether the token should only match as a whole word.
    #[serde(default)]
    pub single_word: bool,
    /// Whether to strip whitespace on the left when matching.
    #[serde(default)]
    pub lstrip: bool,
    /// Whether to strip whitespace on the right when matching.
    #[serde(default)]
    pub rstrip: bool,
    /// Whether the content should be matched against normalized
    /// text.
    #[serde(default)]
    pub normalized: bool,
    /// Whether this is a "special" token (e.g. BOS/EOS). All added
    /// tokens are matched regardless of this flag; it only affects
    /// post-processing.
    #[serde(default)]
    pub special: bool,
}

/// Parsed `tokenizer.json`.
#[derive(Debug, Deserialize)]
pub struct TokenizerJson {
    #[serde(default)]
    pub added_tokens: Vec<AddedTokenConfig>,
    pub normalizer: Option<NormalizerConfig>,
    pub pre_tokenizer: Option<PreTokenizerConfig>,
    pub model: ModelConfig,
    pub post_processor: Option<PostProcessorConfig>,
    pub decoder: Option<DecoderConfig>,
}

/// Normalizer pipeline step from `tokenizer.json`.
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "type")]
pub enum NormalizerConfig {
    Sequence {
        #[serde(default)]
        normalizers: Vec<NormalizerConfig>,
    },
    #[serde(rename = "NFC")]
    Nfc,
    Replace {
        pattern: Value,
        #[serde(default)]
        content: String,
    },
}

/// Pre-tokenizer pipeline step from `tokenizer.json`.
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "type")]
pub enum PreTokenizerConfig {
    Sequence {
        #[serde(default)]
        pretokenizers: Vec<PreTokenizerConfig>,
    },
    ByteLevel(pre_tokenizers::ByteLevel),
    Split(pre_tokenizers::Split),
}

/// Tokenization model from `tokenizer.json`.
#[derive(Clone, Debug)]
pub enum ModelConfig {
    Bpe(Box<models::bpe::Bpe>),
}

impl<'de> Deserialize<'de> for ModelConfig {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: Deserializer<'de>,
    {
        let value = Value::deserialize(deserializer)?;
        let model = value
            .as_object()
            .ok_or_else(|| de::Error::custom("model configuration must be an object"))?;

        match model.get("type") {
            Some(Value::String(model_type)) if model_type == "BPE" => {}
            Some(Value::String(model_type)) => {
                return Err(de::Error::unknown_variant(model_type, &["BPE"]));
            }
            Some(_) => return Err(de::Error::custom("model type must be a string")),
            None if model.contains_key("vocab") && model.contains_key("merges") => {}
            None => return Err(de::Error::missing_field("type")),
        }

        serde_json::from_value(value)
            .map(|bpe| Self::Bpe(Box::new(bpe)))
            .map_err(de::Error::custom)
    }
}

/// Post-processor pipeline step from `tokenizer.json`.
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "type")]
pub enum PostProcessorConfig {
    Sequence {
        #[serde(default)]
        processors: Vec<PostProcessorConfig>,
    },
    ByteLevel {
        #[serde(default)]
        add_prefix_space: bool,
        #[serde(default)]
        trim_offsets: bool,
        #[serde(default)]
        use_regex: bool,
    },
    TemplateProcessing {
        #[serde(default)]
        single: Value,
        #[serde(default)]
        pair: Value,
        #[serde(default)]
        special_tokens: Value,
    },
}

/// Decoder pipeline step from `tokenizer.json`.
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "type")]
pub enum DecoderConfig {
    Sequence {
        #[serde(default)]
        decoders: Vec<DecoderConfig>,
    },
    ByteLevel,
    Replace {
        pattern: Value,
        #[serde(default)]
        content: String,
    },
    Fuse,
    ByteFallback,
}

#[cfg(test)]
mod tests {
    use serde_json::json;

    use super::*;

    /// Ensures unimplemented pipeline types fail while parsing instead of
    /// masquerading as supported configuration.
    #[test]
    fn rejects_unimplemented_pipeline_types() {
        assert!(serde_json::from_value::<NormalizerConfig>(json!({"type": "Lowercase"})).is_err());
        assert!(
            serde_json::from_value::<PreTokenizerConfig>(json!({"type": "Whitespace"})).is_err()
        );
        assert!(serde_json::from_value::<ModelConfig>(json!({"type": "WordPiece"})).is_err());
        assert!(
            serde_json::from_value::<PostProcessorConfig>(json!({"type": "BertProcessing"}))
                .is_err()
        );
        assert!(serde_json::from_value::<DecoderConfig>(json!({"type": "WordPiece"})).is_err());
    }

    /// Legacy GPT-2 tokenizer files predate the tagged model schema but still
    /// identify BPE unambiguously through their vocabulary and merge table.
    #[test]
    fn accepts_legacy_untagged_bpe_model() {
        let model = json!({"vocab": {"a": 0}, "merges": []});
        assert!(serde_json::from_value::<ModelConfig>(model).is_ok());
    }
}