Struct BinaryTokenizer 

pub struct BinaryTokenizer {
    pub vocab: HashMap<String, u32>,
    pub id_to_token: HashMap<u32, String>,
    pub special_tokens: HashMap<String, u32>,
    pub scores: Option<HashMap<u32, f32>>,
    pub merges: Option<Vec<(String, String)>>,
    pub config: HashMap<String, Value>,
    pub normalization_rules: Option<Vec<NormalizationRule>>,
    pub pre_tokenization_rules: Option<Vec<PreTokenizationRule>>,
}

Binary tokenizer representation

Fields

vocab: HashMap<String, u32>

Vocabulary mapping from tokens to IDs

id_to_token: HashMap<u32, String>

Reverse mapping from IDs to tokens

special_tokens: HashMap<String, u32>

Special tokens with their IDs

scores: Option<HashMap<u32, f32>>

Token scores for ranking (if applicable)

merges: Option<Vec<(String, String)>>

Merges for BPE tokenizers (if applicable)

config: HashMap<String, Value>

Additional tokenizer-specific configuration

normalization_rules: Option<Vec<NormalizationRule>>

Normalization rules

pre_tokenization_rules: Option<Vec<PreTokenizationRule>>

Pre-tokenization rules

Trait Implementations

impl Clone for BinaryTokenizer

fn clone(&self) -> BinaryTokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for BinaryTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for BinaryTokenizer

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for BinaryTokenizer

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.