pub struct HfTokenizer { /* private fields */ }Expand description
Hugging Face compatible tokenizer wrapper
Implementations§
Source§impl HfTokenizer
impl HfTokenizer
Sourcepub fn new(tokenizer: Box<dyn Tokenizer>, config: HfTokenizerConfig) -> Self
pub fn new(tokenizer: Box<dyn Tokenizer>, config: HfTokenizerConfig) -> Self
Create new HF-compatible tokenizer
Sourcepub fn encode(
&self,
text: &str,
add_specialtokens: bool,
) -> Result<HfEncodedInput>
pub fn encode( &self, text: &str, add_specialtokens: bool, ) -> Result<HfEncodedInput>
Tokenize text with HF-compatible output
Sourcepub fn encode_batch(
&self,
texts: &[&str],
add_special_tokens: bool,
) -> Result<Vec<HfEncodedInput>>
pub fn encode_batch( &self, texts: &[&str], add_special_tokens: bool, ) -> Result<Vec<HfEncodedInput>>
Batch encode multiple texts
Sourcepub fn decode(
&self,
token_ids: &[usize],
skip_specialtokens: bool,
) -> Result<String>
pub fn decode( &self, token_ids: &[usize], skip_specialtokens: bool, ) -> Result<String>
Decode token IDs back to text
Sourcepub fn vocab_size(&self) -> usize
pub fn vocab_size(&self) -> usize
Get vocabulary size
Auto Trait Implementations§
impl Freeze for HfTokenizer
impl !RefUnwindSafe for HfTokenizer
impl !Send for HfTokenizer
impl !Sync for HfTokenizer
impl Unpin for HfTokenizer
impl !UnwindSafe for HfTokenizer
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§impl<T> Pointable for T
impl<T> Pointable for T
Source§impl<SS, SP> SupersetOf<SS> for SPwhere
SS: SubsetOf<SP>,
impl<SS, SP> SupersetOf<SS> for SPwhere
SS: SubsetOf<SP>,
Source§fn to_subset(&self) -> Option<SS>
fn to_subset(&self) -> Option<SS>
The inverse inclusion map: attempts to construct
self from the equivalent element of its
superset. Read moreSource§fn is_in_subset(&self) -> bool
fn is_in_subset(&self) -> bool
Checks if
self is actually part of its subset T (and can be converted to it).Source§fn to_subset_unchecked(&self) -> SS
fn to_subset_unchecked(&self) -> SS
Use with care! Same as
self.to_subset but without any property checks. Always succeeds.Source§fn from_subset(element: &SS) -> SP
fn from_subset(element: &SS) -> SP
The inclusion map: converts
self to the equivalent element of its superset.