[−][src]Trait rust_tokenizers::preprocessing::vocab::base_vocab::Vocab

pub trait Vocab {
    fn unknown_value() -> &'static str;

    fn values(&self) -> &HashMap<String, i64>;

    fn indices(&self) -> &HashMap<i64, String>;

    fn special_values(&self) -> &HashMap<String, i64>;

    fn special_indices(&self) -> &HashMap<i64, String>;

    fn from_file(path: &str) -> Self;

    fn token_to_id(&self, token: &str) -> i64;

    fn id_to_token(&self, id: &i64) -> String;

    fn read_vocab_file(path: &str) -> HashMap<String, i64> { ... }

    fn _token_to_id(
        &self, 
        token: &str, 
        values: &HashMap<String, i64>, 
        special_values: &HashMap<String, i64>, 
        unknown_value: &str
    ) -> Result<i64, Box<dyn Error>> { ... }

    fn _id_to_token(
        &self, 
        id: &i64, 
        indices: &HashMap<i64, String>, 
        special_indices: &HashMap<i64, String>, 
        unknown_value: &str
    ) -> Result<String, Box<dyn Error>> { ... }

    fn _register_as_special_value(
        token: &str, 
        values: &HashMap<String, i64>, 
        special_values: &mut HashMap<String, i64>
    ) { ... }

    fn convert_tokens_to_ids(&self, tokens: Vec<&str>) -> Vec<i64> { ... }
}

Required methods

fn unknown_value() -> &'static str

fn values(&self) -> &HashMap<String, i64>

fn indices(&self) -> &HashMap<i64, String>

fn special_values(&self) -> &HashMap<String, i64>

fn special_indices(&self) -> &HashMap<i64, String>

fn from_file(path: &str) -> Self

fn token_to_id(&self, token: &str) -> i64

fn id_to_token(&self, id: &i64) -> String

Loading content...

Provided methods

fn read_vocab_file(path: &str) -> HashMap<String, i64>

fn _token_to_id(
    &self,
    token: &str,
    values: &HashMap<String, i64>,
    special_values: &HashMap<String, i64>,
    unknown_value: &str
) -> Result<i64, Box<dyn Error>>

fn _id_to_token(
    &self,
    id: &i64,
    indices: &HashMap<i64, String>,
    special_indices: &HashMap<i64, String>,
    unknown_value: &str
) -> Result<String, Box<dyn Error>>

fn _register_as_special_value(
    token: &str,
    values: &HashMap<String, i64>,
    special_values: &mut HashMap<String, i64>
)

fn convert_tokens_to_ids(&self, tokens: Vec<&str>) -> Vec<i64>

Loading content...

Implementors

impl Vocab for BaseVocab[src]

impl Vocab for BertVocab[src]

impl Vocab for Gpt2Vocab[src]

impl Vocab for OpenAiGptVocab[src]

impl Vocab for RobertaVocab[src]

Loading content...