use crate::audio::AudioConfig;
use crate::image::ImageConfig;
use crate::model_settings::ModelSettingsBuilder;
use crate::special_tokens::SpecialTokenInfo;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TokenInfo {
pub rank: usize,
pub token_bytes: String,
pub token_str: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TekkenConfig {
pub pattern: String,
pub num_vocab_tokens: usize,
pub default_vocab_size: usize,
pub default_num_special_tokens: usize,
pub version: String,
}
#[derive(Debug, Clone, Serialize)]
pub struct ModelData {
pub vocab: Vec<TokenInfo>,
pub special_tokens: Option<Vec<SpecialTokenInfo>>,
pub config: TekkenConfig,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image: Option<ImageConfig>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub multimodal: Option<ImageConfig>,
pub audio: Option<AudioConfig>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model_settings_builder: Option<ModelSettingsBuilder>,
}
impl<'de> Deserialize<'de> for ModelData {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
struct ModelDataVisitor;
impl<'de> serde::de::Visitor<'de> for ModelDataVisitor {
type Value = ModelData;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("a tekken tokenizer document")
}
fn visit_map<A>(self, mut map: A) -> std::result::Result<ModelData, A::Error>
where
A: serde::de::MapAccess<'de>,
{
use serde::de::Error as _;
let mut vocab = None;
let mut special_tokens = None;
let mut config = None;
let mut image = None;
let mut multimodal = None;
let mut audio = None;
let mut model_settings_builder = None;
while let Some(key) = map.next_key::<String>()? {
match key.as_str() {
"vocab" => vocab = Some(map.next_value()?),
"special_tokens" => special_tokens = map.next_value()?,
"config" => config = Some(map.next_value()?),
"image" => image = map.next_value()?,
"multimodal" => multimodal = map.next_value()?,
"audio" => audio = map.next_value()?,
"model_settings_builder" => model_settings_builder = map.next_value()?,
_ => {
map.next_value::<serde::de::IgnoredAny>()?;
}
}
}
Ok(ModelData {
vocab: vocab.ok_or_else(|| A::Error::missing_field("vocab"))?,
special_tokens,
config: config.ok_or_else(|| A::Error::missing_field("config"))?,
image,
multimodal,
audio,
model_settings_builder,
})
}
}
deserializer.deserialize_map(ModelDataVisitor)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum TokenizerVersion {
V3,
V7,
V11,
V13,
V15,
}
impl TokenizerVersion {
#[must_use]
pub fn from_string(s: &str) -> Option<Self> {
match s {
"v3" => Some(Self::V3),
"v7" => Some(Self::V7),
"v11" => Some(Self::V11),
"v13" => Some(Self::V13),
"v15" => Some(Self::V15),
_ => None,
}
}
#[must_use]
pub const fn as_str(&self) -> &'static str {
match self {
Self::V3 => "v3",
Self::V7 => "v7",
Self::V11 => "v11",
Self::V13 => "v13",
Self::V15 => "v15",
}
}
#[must_use]
pub const fn version_num(&self) -> u32 {
match self {
Self::V3 => 3,
Self::V7 => 7,
Self::V11 => 11,
Self::V13 => 13,
Self::V15 => 15,
}
}
#[must_use]
pub const fn supports_model_settings(&self) -> bool {
self.version_num() >= 15
}
#[must_use]
pub const fn requires_explicit_special_tokens(&self) -> bool {
self.version_num() > 7
}
#[must_use]
pub const fn allows_deprecated_multimodal_key(&self) -> bool {
self.version_num() <= 11
}
}