use serde::{Deserialize, Deserializer, de};
use serde_json::Value;
use crate::{models, pre_tokenizers};
#[derive(Clone, Debug, Deserialize)]
pub struct AddedTokenConfig {
pub id: u32,
pub content: String,
#[serde(default)]
pub single_word: bool,
#[serde(default)]
pub lstrip: bool,
#[serde(default)]
pub rstrip: bool,
#[serde(default)]
pub normalized: bool,
#[serde(default)]
pub special: bool,
}
#[derive(Debug, Deserialize)]
pub struct TokenizerJson {
#[serde(default)]
pub added_tokens: Vec<AddedTokenConfig>,
pub normalizer: Option<NormalizerConfig>,
pub pre_tokenizer: Option<PreTokenizerConfig>,
pub model: ModelConfig,
pub post_processor: Option<PostProcessorConfig>,
pub decoder: Option<DecoderConfig>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "type")]
pub enum NormalizerConfig {
Sequence {
#[serde(default)]
normalizers: Vec<NormalizerConfig>,
},
#[serde(rename = "NFC")]
Nfc,
Replace {
pattern: Value,
#[serde(default)]
content: String,
},
}
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "type")]
pub enum PreTokenizerConfig {
Sequence {
#[serde(default)]
pretokenizers: Vec<PreTokenizerConfig>,
},
ByteLevel(pre_tokenizers::ByteLevel),
Split(pre_tokenizers::Split),
}
#[derive(Clone, Debug)]
pub enum ModelConfig {
Bpe(Box<models::bpe::Bpe>),
}
impl<'de> Deserialize<'de> for ModelConfig {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = Value::deserialize(deserializer)?;
let model = value
.as_object()
.ok_or_else(|| de::Error::custom("model configuration must be an object"))?;
match model.get("type") {
Some(Value::String(model_type)) if model_type == "BPE" => {}
Some(Value::String(model_type)) => {
return Err(de::Error::unknown_variant(model_type, &["BPE"]));
}
Some(_) => return Err(de::Error::custom("model type must be a string")),
None if model.contains_key("vocab") && model.contains_key("merges") => {}
None => return Err(de::Error::missing_field("type")),
}
serde_json::from_value(value)
.map(|bpe| Self::Bpe(Box::new(bpe)))
.map_err(de::Error::custom)
}
}
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "type")]
pub enum PostProcessorConfig {
Sequence {
#[serde(default)]
processors: Vec<PostProcessorConfig>,
},
ByteLevel {
#[serde(default)]
add_prefix_space: bool,
#[serde(default)]
trim_offsets: bool,
#[serde(default)]
use_regex: bool,
},
TemplateProcessing {
#[serde(default)]
single: Value,
#[serde(default)]
pair: Value,
#[serde(default)]
special_tokens: Value,
},
}
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "type")]
pub enum DecoderConfig {
Sequence {
#[serde(default)]
decoders: Vec<DecoderConfig>,
},
ByteLevel,
Replace {
pattern: Value,
#[serde(default)]
content: String,
},
Fuse,
ByteFallback,
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
#[test]
fn rejects_unimplemented_pipeline_types() {
assert!(serde_json::from_value::<NormalizerConfig>(json!({"type": "Lowercase"})).is_err());
assert!(
serde_json::from_value::<PreTokenizerConfig>(json!({"type": "Whitespace"})).is_err()
);
assert!(serde_json::from_value::<ModelConfig>(json!({"type": "WordPiece"})).is_err());
assert!(
serde_json::from_value::<PostProcessorConfig>(json!({"type": "BertProcessing"}))
.is_err()
);
assert!(serde_json::from_value::<DecoderConfig>(json!({"type": "WordPiece"})).is_err());
}
#[test]
fn accepts_legacy_untagged_bpe_model() {
let model = json!({"vocab": {"a": 0}, "merges": []});
assert!(serde_json::from_value::<ModelConfig>(model).is_ok());
}
}