[−][src]Struct rust_tokenizers::preprocessing::vocab::base_vocab::BaseVocab
Fields
values: HashMap<String, i64>
A mapping of tokens as string to indices (i.e. the encoder base)
indices: HashMap<i64, String>
A mapping of token IDs to strings (i.e. the decoder base)
unknown_value: &'static str
The string to use for unknown (out of vocabulary) tokens
special_values: HashMap<String, i64>
A mapping of special value tokens as strings to IDs (i.e. the encoder base for special values), special values typically include things like BOS/EOS markers, class markers, mask markers and padding markers
special_indices: HashMap<i64, String>
A mapping of special value tokens as IDs to strings (i.e. the decoder base for special values)
Trait Implementations
impl Vocab for BaseVocab
[src]
fn unknown_value() -> &'static str
[src]
fn get_unknown_value(&self) -> &'static str
[src]
fn values(&self) -> &HashMap<String, i64>
[src]
fn indices(&self) -> &HashMap<i64, String>
[src]
fn special_values(&self) -> &HashMap<String, i64>
[src]
fn special_indices(&self) -> &HashMap<i64, String>
[src]
fn from_file(path: &str) -> BaseVocab
[src]
fn token_to_id(&self, token: &str) -> i64
[src]
fn id_to_token(&self, id: &i64) -> String
[src]
fn read_vocab_file(path: &str) -> HashMap<String, i64>
[src]
fn _token_to_id(
&self,
token: &str,
values: &HashMap<String, i64>,
special_values: &HashMap<String, i64>,
unknown_value: &str
) -> Result<i64, Box<dyn Error>>
[src]
&self,
token: &str,
values: &HashMap<String, i64>,
special_values: &HashMap<String, i64>,
unknown_value: &str
) -> Result<i64, Box<dyn Error>>
fn _id_to_token(
&self,
id: &i64,
indices: &HashMap<i64, String>,
special_indices: &HashMap<i64, String>,
unknown_value: &str
) -> Result<String, Box<dyn Error>>
[src]
&self,
id: &i64,
indices: &HashMap<i64, String>,
special_indices: &HashMap<i64, String>,
unknown_value: &str
) -> Result<String, Box<dyn Error>>
fn _register_as_special_value(
token: &str,
values: &HashMap<String, i64>,
special_values: &mut HashMap<String, i64>
)
[src]
token: &str,
values: &HashMap<String, i64>,
special_values: &mut HashMap<String, i64>
)
fn convert_tokens_to_ids(&self, tokens: Vec<&str>) -> Vec<i64>
[src]
Auto Trait Implementations
impl RefUnwindSafe for BaseVocab
impl Send for BaseVocab
impl Sync for BaseVocab
impl Unpin for BaseVocab
impl UnwindSafe for BaseVocab
Blanket Implementations
impl<T> Any for T where
T: 'static + ?Sized,
[src]
T: 'static + ?Sized,
impl<T> Borrow<T> for T where
T: ?Sized,
[src]
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
[src]
T: ?Sized,
fn borrow_mut(&mut self) -> &mut T
[src]
impl<T> From<T> for T
[src]
impl<T, U> Into<U> for T where
U: From<T>,
[src]
U: From<T>,
impl<T, U> TryFrom<U> for T where
U: Into<T>,
[src]
U: Into<T>,
type Error = Infallible
The type returned in the event of a conversion error.
fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>
[src]
impl<T, U> TryInto<U> for T where
U: TryFrom<T>,
[src]
U: TryFrom<T>,