Struct mako::tokenization::hf_tokenizers::tokenizer::TokenizerImpl
source · [−]pub struct TokenizerImpl<M, N, PT, PP, D> { /* private fields */ }
Expand description
A Tokenizer
is capable of encoding/decoding any text.
Implementations
impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D> where
M: Model,
N: Normalizer,
PT: PreTokenizer,
PP: PostProcessor,
D: Decoder,
impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D> where
M: Model,
N: Normalizer,
PT: PreTokenizer,
PP: PostProcessor,
D: Decoder,
Set the normalizer
Get the normalizer
Set the pre tokenizer
Get the pre tokenizer
Set the post processor
Get the post processor
Set the decoder
Get the decoder
Set the model
Set the truncation parameters
Get the currently set truncation parameters
Get a mutable reference to the currently set truncation parameters
Set the padding parameters
Get the currently set padding parameters
Get a mutable reference to the currently set padding parameters
Get the size of the vocabulary
Converts a token in the corresponding id.
Converts an id to the corresponding token.
Encode the given input. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly:
pub fn encode_char_offsets<'s, E>(
&self,
input: E,
add_special_tokens: bool
) -> Result<Encoding> where
E: Into<EncodeInput<'s>>,
pub fn encode_char_offsets<'s, E>(
&self,
input: E,
add_special_tokens: bool
) -> Result<Encoding> where
E: Into<EncodeInput<'s>>,
Encode the given input, using offsets relative to chars instead of bytes. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly
Register the given tokens as special tokens. This is especially useful for removing these special tokens while decoding
Add the given tokens to the added vocabulary
impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D> where
M: Model + Send + Sync,
N: Normalizer + Send + Sync,
PT: PreTokenizer + Send + Sync,
PP: PostProcessor + Send + Sync,
D: Decoder + Send + Sync,
impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D> where
M: Model + Send + Sync,
N: Normalizer + Send + Sync,
PT: PreTokenizer + Send + Sync,
PP: PostProcessor + Send + Sync,
D: Decoder + Send + Sync,
pub fn encode_batch<'s, E>(
&self,
inputs: Vec<E>,
add_special_tokens: bool
) -> Result<Vec<Encoding>> where
E: Into<EncodeInput<'s>> + Send,
pub fn encode_batch<'s, E>(
&self,
inputs: Vec<E>,
add_special_tokens: bool
) -> Result<Vec<Encoding>> where
E: Into<EncodeInput<'s>> + Send,
Encode all the sentences in parallel, using multiple threads
pub fn encode_batch_char_offsets<'s, E>(
&self,
inputs: Vec<E>,
add_special_tokens: bool
) -> Result<Vec<Encoding>> where
E: Into<EncodeInput<'s>> + Send,
pub fn encode_batch_char_offsets<'s, E>(
&self,
inputs: Vec<E>,
add_special_tokens: bool
) -> Result<Vec<Encoding>> where
E: Into<EncodeInput<'s>> + Send,
Encode all the sentences in parallel, using multiple threads.
The offsets on each Encoding
will be relative to chars instead of bytes.
Decode all sentences in parallel
Train our Model from files
impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D> where
M: DeserializeOwned + Model,
N: DeserializeOwned + Normalizer,
PT: DeserializeOwned + PreTokenizer,
PP: DeserializeOwned + PostProcessor,
D: DeserializeOwned + Decoder,
impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D> where
M: DeserializeOwned + Model,
N: DeserializeOwned + Normalizer,
PT: DeserializeOwned + PreTokenizer,
PP: DeserializeOwned + PostProcessor,
D: DeserializeOwned + Decoder,
Trait Implementations
impl<'de, M, N, PT, PP, D> Deserialize<'de> for TokenizerImpl<M, N, PT, PP, D> where
M: Deserialize<'de> + Model,
N: Deserialize<'de> + Normalizer,
PT: Deserialize<'de> + PreTokenizer,
PP: Deserialize<'de> + PostProcessor,
D: Deserialize<'de> + Decoder,
impl<'de, M, N, PT, PP, D> Deserialize<'de> for TokenizerImpl<M, N, PT, PP, D> where
M: Deserialize<'de> + Model,
N: Deserialize<'de> + Normalizer,
PT: Deserialize<'de> + PreTokenizer,
PP: Deserialize<'de> + PostProcessor,
D: Deserialize<'de> + Decoder,
Deserialize this value from the given Serde deserializer. Read more
impl<M, N, PT, PP, D> From<TokenizerImpl<M, N, PT, PP, D>> for Tokenizer where
M: Into<ModelWrapper>,
N: Into<NormalizerWrapper>,
PT: Into<PreTokenizerWrapper>,
PP: Into<PostProcessorWrapper>,
D: Into<DecoderWrapper>,
impl<M, N, PT, PP, D> From<TokenizerImpl<M, N, PT, PP, D>> for Tokenizer where
M: Into<ModelWrapper>,
N: Into<NormalizerWrapper>,
PT: Into<PreTokenizerWrapper>,
PP: Into<PostProcessorWrapper>,
D: Into<DecoderWrapper>,
Performs the conversion.
impl<M, N, PT, PP, D> FromStr for TokenizerImpl<M, N, PT, PP, D> where
M: for<'de> Deserialize<'de> + Model,
N: for<'de> Deserialize<'de> + Normalizer,
PT: for<'de> Deserialize<'de> + PreTokenizer,
PP: for<'de> Deserialize<'de> + PostProcessor,
D: for<'de> Deserialize<'de> + Decoder,
impl<M, N, PT, PP, D> FromStr for TokenizerImpl<M, N, PT, PP, D> where
M: for<'de> Deserialize<'de> + Model,
N: for<'de> Deserialize<'de> + Normalizer,
PT: for<'de> Deserialize<'de> + PreTokenizer,
PP: for<'de> Deserialize<'de> + PostProcessor,
D: for<'de> Deserialize<'de> + Decoder,
Auto Trait Implementations
impl<M, N, PT, PP, D> RefUnwindSafe for TokenizerImpl<M, N, PT, PP, D> where
D: RefUnwindSafe,
M: RefUnwindSafe,
N: RefUnwindSafe,
PP: RefUnwindSafe,
PT: RefUnwindSafe,
impl<M, N, PT, PP, D> Send for TokenizerImpl<M, N, PT, PP, D> where
D: Send,
M: Send,
N: Send,
PP: Send,
PT: Send,
impl<M, N, PT, PP, D> Sync for TokenizerImpl<M, N, PT, PP, D> where
D: Sync,
M: Sync,
N: Sync,
PP: Sync,
PT: Sync,
impl<M, N, PT, PP, D> Unpin for TokenizerImpl<M, N, PT, PP, D> where
D: Unpin,
M: Unpin,
N: Unpin,
PP: Unpin,
PT: Unpin,
impl<M, N, PT, PP, D> UnwindSafe for TokenizerImpl<M, N, PT, PP, D> where
D: UnwindSafe,
M: UnwindSafe,
N: UnwindSafe,
PP: UnwindSafe,
PT: UnwindSafe,
Blanket Implementations
Mutably borrows from an owned value. Read more
fn fmt_binary(self) -> FmtBinary<Self> where
Self: Binary,
fn fmt_binary(self) -> FmtBinary<Self> where
Self: Binary,
Causes self
to use its Binary
implementation when Debug
-formatted.
fn fmt_display(self) -> FmtDisplay<Self> where
Self: Display,
fn fmt_display(self) -> FmtDisplay<Self> where
Self: Display,
Causes self
to use its Display
implementation when
Debug
-formatted. Read more
fn fmt_lower_exp(self) -> FmtLowerExp<Self> where
Self: LowerExp,
fn fmt_lower_exp(self) -> FmtLowerExp<Self> where
Self: LowerExp,
Causes self
to use its LowerExp
implementation when
Debug
-formatted. Read more
fn fmt_lower_hex(self) -> FmtLowerHex<Self> where
Self: LowerHex,
fn fmt_lower_hex(self) -> FmtLowerHex<Self> where
Self: LowerHex,
Causes self
to use its LowerHex
implementation when
Debug
-formatted. Read more
Causes self
to use its Octal
implementation when Debug
-formatted.
fn fmt_pointer(self) -> FmtPointer<Self> where
Self: Pointer,
fn fmt_pointer(self) -> FmtPointer<Self> where
Self: Pointer,
Causes self
to use its Pointer
implementation when
Debug
-formatted. Read more
fn fmt_upper_exp(self) -> FmtUpperExp<Self> where
Self: UpperExp,
fn fmt_upper_exp(self) -> FmtUpperExp<Self> where
Self: UpperExp,
Causes self
to use its UpperExp
implementation when
Debug
-formatted. Read more
fn fmt_upper_hex(self) -> FmtUpperHex<Self> where
Self: UpperHex,
fn fmt_upper_hex(self) -> FmtUpperHex<Self> where
Self: UpperHex,
Causes self
to use its UpperHex
implementation when
Debug
-formatted. Read more
impl<T> Pipe for T where
T: ?Sized,
impl<T> Pipe for T where
T: ?Sized,
Pipes by value. This is generally the method you want to use. Read more
Borrows self
and passes that borrow into the pipe function. Read more
fn pipe_ref_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> R where
R: 'a,
fn pipe_ref_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> R where
R: 'a,
Mutably borrows self
and passes that borrow into the pipe function. Read more
fn pipe_borrow<'a, B, R>(&'a self, func: impl FnOnce(&'a B) -> R) -> R where
Self: Borrow<B>,
B: 'a + ?Sized,
R: 'a,
fn pipe_borrow<'a, B, R>(&'a self, func: impl FnOnce(&'a B) -> R) -> R where
Self: Borrow<B>,
B: 'a + ?Sized,
R: 'a,
Borrows self
, then passes self.borrow()
into the pipe function. Read more
fn pipe_borrow_mut<'a, B, R>(
&'a mut self,
func: impl FnOnce(&'a mut B) -> R
) -> R where
Self: BorrowMut<B>,
B: 'a + ?Sized,
R: 'a,
fn pipe_borrow_mut<'a, B, R>(
&'a mut self,
func: impl FnOnce(&'a mut B) -> R
) -> R where
Self: BorrowMut<B>,
B: 'a + ?Sized,
R: 'a,
Mutably borrows self
, then passes self.borrow_mut()
into the pipe
function. Read more
fn pipe_as_ref<'a, U, R>(&'a self, func: impl FnOnce(&'a U) -> R) -> R where
Self: AsRef<U>,
U: 'a + ?Sized,
R: 'a,
fn pipe_as_ref<'a, U, R>(&'a self, func: impl FnOnce(&'a U) -> R) -> R where
Self: AsRef<U>,
U: 'a + ?Sized,
R: 'a,
Borrows self
, then passes self.as_ref()
into the pipe function.
fn pipe_as_mut<'a, U, R>(&'a mut self, func: impl FnOnce(&'a mut U) -> R) -> R where
Self: AsMut<U>,
U: 'a + ?Sized,
R: 'a,
fn pipe_as_mut<'a, U, R>(&'a mut self, func: impl FnOnce(&'a mut U) -> R) -> R where
Self: AsMut<U>,
U: 'a + ?Sized,
R: 'a,
Mutably borrows self
, then passes self.as_mut()
into the pipe
function. Read more
fn pipe_deref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Deref<Target = T>,
T: 'a + ?Sized,
R: 'a,
fn pipe_deref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Deref<Target = T>,
T: 'a + ?Sized,
R: 'a,
Borrows self
, then passes self.deref()
into the pipe function.
fn pipe_as_ref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: AsRef<T>,
T: 'a,
R: 'a,
fn pipe_as_ref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: AsRef<T>,
T: 'a,
R: 'a,
Pipes a trait borrow into a function that cannot normally be called in suffix position. Read more
fn pipe_as_mut<'a, T, R>(&'a mut self, func: impl FnOnce(&'a mut T) -> R) -> R where
Self: AsMut<T>,
T: 'a,
R: 'a,
fn pipe_as_mut<'a, T, R>(&'a mut self, func: impl FnOnce(&'a mut T) -> R) -> R where
Self: AsMut<T>,
T: 'a,
R: 'a,
Pipes a trait mutable borrow into a function that cannot normally be called in suffix position. Read more
fn pipe_borrow<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Borrow<T>,
T: 'a,
R: 'a,
fn pipe_borrow<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Borrow<T>,
T: 'a,
R: 'a,
Pipes a trait borrow into a function that cannot normally be called in suffix position. Read more
fn pipe_borrow_mut<'a, T, R>(
&'a mut self,
func: impl FnOnce(&'a mut T) -> R
) -> R where
Self: BorrowMut<T>,
T: 'a,
R: 'a,
fn pipe_borrow_mut<'a, T, R>(
&'a mut self,
func: impl FnOnce(&'a mut T) -> R
) -> R where
Self: BorrowMut<T>,
T: 'a,
R: 'a,
Pipes a trait mutable borrow into a function that cannot normally be called in suffix position. Read more
fn pipe_deref<'a, R>(&'a self, func: impl FnOnce(&'a Self::Target) -> R) -> R where
Self: Deref,
R: 'a,
fn pipe_deref<'a, R>(&'a self, func: impl FnOnce(&'a Self::Target) -> R) -> R where
Self: Deref,
R: 'a,
Pipes a dereference into a function that cannot normally be called in suffix position. Read more
fn pipe_deref_mut<'a, R>(
&'a mut self,
func: impl FnOnce(&'a mut Self::Target) -> R
) -> R where
Self: DerefMut,
R: 'a,
fn pipe_deref_mut<'a, R>(
&'a mut self,
func: impl FnOnce(&'a mut Self::Target) -> R
) -> R where
Self: DerefMut,
R: 'a,
Pipes a mutable dereference into a function that cannot normally be called in suffix position. Read more
Pipes a reference into a function that cannot ordinarily be called in suffix position. Read more
fn tap_borrow<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,
fn tap_borrow<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,
Immutable access to the Borrow<B>
of a value. Read more
fn tap_borrow_mut<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,
fn tap_borrow_mut<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,
Mutable access to the BorrowMut<B>
of a value. Read more
Immutable access to the AsRef<R>
view of a value. Read more
fn tap_ref_mut<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,
fn tap_ref_mut<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,
Mutable access to the AsMut<R>
view of a value. Read more
Immutable access to the Deref::Target
of a value. Read more
Mutable access to the Deref::Target
of a value. Read more
Calls .tap()
only in debug builds, and is erased in release builds.
fn tap_mut_dbg(self, func: impl FnOnce(&mut Self)) -> Self
fn tap_mut_dbg(self, func: impl FnOnce(&mut Self)) -> Self
Calls .tap_mut()
only in debug builds, and is erased in release
builds. Read more
fn tap_borrow_dbg<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,
fn tap_borrow_dbg<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,
Calls .tap_borrow()
only in debug builds, and is erased in release
builds. Read more
fn tap_borrow_mut_dbg<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,
fn tap_borrow_mut_dbg<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,
Calls .tap_borrow_mut()
only in debug builds, and is erased in release
builds. Read more
fn tap_ref_dbg<R>(self, func: impl FnOnce(&R)) -> Self where
Self: AsRef<R>,
R: ?Sized,
fn tap_ref_dbg<R>(self, func: impl FnOnce(&R)) -> Self where
Self: AsRef<R>,
R: ?Sized,
Calls .tap_ref()
only in debug builds, and is erased in release
builds. Read more
fn tap_ref_mut_dbg<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,
fn tap_ref_mut_dbg<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,
Calls .tap_ref_mut()
only in debug builds, and is erased in release
builds. Read more
Provides immutable access for inspection. Read more
Calls tap
in debug builds, and does nothing in release builds.
Provides mutable access for modification. Read more
fn tap_mut_dbg<F, R>(self, func: F) -> Self where
F: FnOnce(&mut Self) -> R,
fn tap_mut_dbg<F, R>(self, func: F) -> Self where
F: FnOnce(&mut Self) -> R,
Calls tap_mut
in debug builds, and does nothing in release builds.
impl<T, U> TapAsRef<U> for T where
U: ?Sized,
impl<T, U> TapAsRef<U> for T where
U: ?Sized,
Provides immutable access to the reference for inspection.
fn tap_ref_dbg<F, R>(self, func: F) -> Self where
Self: AsRef<T>,
F: FnOnce(&T) -> R,
fn tap_ref_dbg<F, R>(self, func: F) -> Self where
Self: AsRef<T>,
F: FnOnce(&T) -> R,
Calls tap_ref
in debug builds, and does nothing in release builds.
fn tap_ref_mut<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,
fn tap_ref_mut<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,
Provides mutable access to the reference for modification.
fn tap_ref_mut_dbg<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,
fn tap_ref_mut_dbg<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,
Calls tap_ref_mut
in debug builds, and does nothing in release builds.
impl<T, U> TapBorrow<U> for T where
U: ?Sized,
impl<T, U> TapBorrow<U> for T where
U: ?Sized,
fn tap_borrow<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,
fn tap_borrow<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,
Provides immutable access to the borrow for inspection. Read more
fn tap_borrow_dbg<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,
fn tap_borrow_dbg<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,
Calls tap_borrow
in debug builds, and does nothing in release builds.
fn tap_borrow_mut<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,
fn tap_borrow_mut<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,
Provides mutable access to the borrow for modification.
fn tap_borrow_mut_dbg<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,
fn tap_borrow_mut_dbg<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,
Calls tap_borrow_mut
in debug builds, and does nothing in release
builds. Read more
Immutably dereferences self
for inspection.
fn tap_deref_dbg<F, R>(self, func: F) -> Self where
Self: Deref,
F: FnOnce(&Self::Target) -> R,
fn tap_deref_dbg<F, R>(self, func: F) -> Self where
Self: Deref,
F: FnOnce(&Self::Target) -> R,
Calls tap_deref
in debug builds, and does nothing in release builds.
fn tap_deref_mut<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,
fn tap_deref_mut<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,
Mutably dereferences self
for modification.
fn tap_deref_mut_dbg<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,
fn tap_deref_mut_dbg<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,
Calls tap_deref_mut
in debug builds, and does nothing in release
builds. Read more