[−][src]Trait rust_tokenizers::preprocessing::vocab::base_vocab::Vocab
Required methods
fn unknown_value() -> &'static str
Associative function returning the unknown value
fn get_unknown_value(&self) -> &'static str
Returns the unknown value on an instance
fn values(&self) -> &HashMap<String, i64>
Return the map of token strings to IDs
fn indices(&self) -> &HashMap<i64, String>
Return the map of token IDs to strings
fn special_values(&self) -> &HashMap<String, i64>
Return the map of token strings to IDs
fn special_indices(&self) -> &HashMap<i64, String>
Return the map of token IDs to strings for special values
fn from_file(path: &str) -> Self
Read a vocabulary file from file
fn token_to_id(&self, token: &str) -> i64
fn id_to_token(&self, id: &i64) -> String
Provided methods
fn read_vocab_file(path: &str) -> HashMap<String, i64>
Read a Bert-style vocab.txt file (single column, one token per line)
fn _token_to_id(
&self,
token: &str,
values: &HashMap<String, i64>,
special_values: &HashMap<String, i64>,
unknown_value: &str
) -> Result<i64, Box<dyn Error>>
&self,
token: &str,
values: &HashMap<String, i64>,
special_values: &HashMap<String, i64>,
unknown_value: &str
) -> Result<i64, Box<dyn Error>>
fn _id_to_token(
&self,
id: &i64,
indices: &HashMap<i64, String>,
special_indices: &HashMap<i64, String>,
unknown_value: &str
) -> Result<String, Box<dyn Error>>
&self,
id: &i64,
indices: &HashMap<i64, String>,
special_indices: &HashMap<i64, String>,
unknown_value: &str
) -> Result<String, Box<dyn Error>>
fn _register_as_special_value(
token: &str,
values: &HashMap<String, i64>,
special_values: &mut HashMap<String, i64>
)
token: &str,
values: &HashMap<String, i64>,
special_values: &mut HashMap<String, i64>
)
fn convert_tokens_to_ids(&self, tokens: Vec<&str>) -> Vec<i64>
Implementors
impl Vocab for BaseVocab
[src]
fn unknown_value() -> &'static str
[src]
fn get_unknown_value(&self) -> &'static str
[src]
fn values(&self) -> &HashMap<String, i64>
[src]
fn indices(&self) -> &HashMap<i64, String>
[src]
fn special_values(&self) -> &HashMap<String, i64>
[src]
fn special_indices(&self) -> &HashMap<i64, String>
[src]
fn from_file(path: &str) -> BaseVocab
[src]
fn token_to_id(&self, token: &str) -> i64
[src]
fn id_to_token(&self, id: &i64) -> String
[src]
impl Vocab for BertVocab
[src]
fn unknown_value() -> &'static str
[src]
fn get_unknown_value(&self) -> &'static str
[src]
fn values(&self) -> &HashMap<String, i64>
[src]
fn indices(&self) -> &HashMap<i64, String>
[src]
fn special_values(&self) -> &HashMap<String, i64>
[src]
fn special_indices(&self) -> &HashMap<i64, String>
[src]
fn from_file(path: &str) -> BertVocab
[src]
fn token_to_id(&self, token: &str) -> i64
[src]
fn id_to_token(&self, id: &i64) -> String
[src]
impl Vocab for Gpt2Vocab
[src]
fn unknown_value() -> &'static str
[src]
fn get_unknown_value(&self) -> &'static str
[src]
fn values(&self) -> &HashMap<String, i64>
[src]
fn indices(&self) -> &HashMap<i64, String>
[src]
fn special_values(&self) -> &HashMap<String, i64>
[src]
fn special_indices(&self) -> &HashMap<i64, String>
[src]
fn from_file(path: &str) -> Gpt2Vocab
[src]
fn token_to_id(&self, token: &str) -> i64
[src]
fn id_to_token(&self, id: &i64) -> String
[src]
impl Vocab for OpenAiGptVocab
[src]
fn unknown_value() -> &'static str
[src]
fn get_unknown_value(&self) -> &'static str
[src]
fn values(&self) -> &HashMap<String, i64>
[src]
fn indices(&self) -> &HashMap<i64, String>
[src]
fn special_values(&self) -> &HashMap<String, i64>
[src]
fn special_indices(&self) -> &HashMap<i64, String>
[src]
fn from_file(path: &str) -> OpenAiGptVocab
[src]
fn token_to_id(&self, token: &str) -> i64
[src]
fn id_to_token(&self, id: &i64) -> String
[src]
impl Vocab for RobertaVocab
[src]
fn unknown_value() -> &'static str
[src]
fn get_unknown_value(&self) -> &'static str
[src]
fn values(&self) -> &HashMap<String, i64>
[src]
fn indices(&self) -> &HashMap<i64, String>
[src]
fn special_values(&self) -> &HashMap<String, i64>
[src]
fn special_indices(&self) -> &HashMap<i64, String>
[src]
fn from_file(path: &str) -> RobertaVocab
[src]
Read a Roberta-style vocab.json file