Enum ModelConfig 

pub enum ModelConfig {
    Bpe(Box<Bpe>),
    WordPiece {
        vocab: HashMap<String, u32>,
        unk_token: String,
        continuing_subword_prefix: String,
        max_input_chars_per_word: Option<u64>,
    },
    WordLevel {
        vocab: HashMap<String, u32>,
        unk_token: String,
    },
    Unigram {
        vocab: Vec<(String, f64)>,
        unk_id: Option<u32>,
        byte_fallback: bool,
    },
    Other(Value),
}

Tokenization model from tokenizer.json.

Variants

Bpe(Box<Bpe>)

WordPiece

Fields

vocab: HashMap<String, u32>

unk_token: String

continuing_subword_prefix: String

max_input_chars_per_word: Option<u64>

WordLevel

Fields

vocab: HashMap<String, u32>

unk_token: String

Unigram

Fields

vocab: Vec<(String, f64)>

unk_id: Option<u32>

byte_fallback: bool

Other(Value)

Trait Implementations

impl Clone for ModelConfig

fn clone(&self) -> ModelConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ModelConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for ModelConfig

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl<'_enum> From<&'_enum ModelConfig> for ModelKind

fn from(val: &'_enum ModelConfig) -> ModelKind

Converts to this type from the input type.

impl From<ModelConfig> for ModelKind

fn from(val: ModelConfig) -> ModelKind

Converts to this type from the input type.

impl IntoDiscriminant for ModelConfig

type Discriminant = ModelKind

Enum listing the same variants as this enum but without any data fields

fn discriminant(&self) -> Self::Discriminant