[−][src]Struct tokenizers::models::wordlevel::WordLevel
Methods
impl WordLevel
[src]
pub fn from_files(vocab_path: &str, unk_token: String) -> Result<WordLevel>
[src]
Initialize a WordLevel model from vocab and merges file.
Trait Implementations
impl Default for WordLevel
[src]
impl Model for WordLevel
[src]
fn tokenize(&self, tokens: Vec<(String, (usize, usize))>) -> Result<Vec<Token>>
[src]
fn token_to_id(&self, token: &str) -> Option<u32>
[src]
fn id_to_token(&self, id: u32) -> Option<String>
[src]
fn get_vocab(&self) -> &HashMap<String, u32>
[src]
fn get_vocab_size(&self) -> usize
[src]
fn save(&self, folder: &Path, name: Option<&str>) -> Result<Vec<PathBuf>>
[src]
Auto Trait Implementations
impl RefUnwindSafe for WordLevel
impl Send for WordLevel
impl Sync for WordLevel
impl Unpin for WordLevel
impl UnwindSafe for WordLevel
Blanket Implementations
impl<T> Any for T where
T: 'static + ?Sized,
[src]
T: 'static + ?Sized,
impl<T> Borrow<T> for T where
T: ?Sized,
[src]
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
[src]
T: ?Sized,
fn borrow_mut(&mut self) -> &mut T
[src]
impl<T> From<T> for T
[src]
impl<T, U> Into<U> for T where
U: From<T>,
[src]
U: From<T>,
impl<T, U> TryFrom<U> for T where
U: Into<T>,
[src]
U: Into<T>,
type Error = Infallible
The type returned in the event of a conversion error.
fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>
[src]
impl<T, U> TryInto<U> for T where
U: TryFrom<T>,
[src]
U: TryFrom<T>,
type Error = <U as TryFrom<T>>::Error
The type returned in the event of a conversion error.
fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>
[src]
impl<V, T> VZip<V> for T where
V: MultiLane<T>,
V: MultiLane<T>,