Struct mako::tokenization::hf_tokenizers::tokenizer::Tokenizer

Encode the given input. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly:

source

pub fn encode_char_offsets<'s, E>(
 &self,
 input: E,
 add_special_tokens: bool
) -> Result<Encoding> where
 E: Into<EncodeInput<'s>>,

Encode the given input, using offsets relative to chars instead of bytes. This method accepts both single sequences, as well as pair sequences. Also, a sequence can be a string, or already pre-tokenized input directly

source

pub fn decode(&self, ids: Vec<u32>, skip_special_tokens: bool) -> Result<String>

Decode the given ids, back to a String

source

pub fn add_special_tokens(&mut self, tokens: &[AddedToken ]) -> usize

Register the given tokens as special tokens. This is especially useful for removing these special tokens while decoding

source

pub fn add_tokens(&mut self, tokens: &[AddedToken ]) -> usize

Add the given tokens to the added vocabulary

source

pub fn post_process(
 &self,
 encoding: Encoding,
 pair_encoding: Option<Encoding>,
 add_special_tokens: bool
) -> Result<Encoding>

Post processing logic, handling the case where there is no PostProcessor set

source

pub fn encode_batch<'s, E>(
 &self,
 inputs: Vec<E>,
 add_special_tokens: bool
) -> Result<Vec<Encoding>> where
 E: Into<EncodeInput<'s>> + Send,

Encode all the sentences in parallel, using multiple threads

source

pub fn encode_batch_char_offsets<'s, E>(
 &self,
 inputs: Vec<E>,
 add_special_tokens: bool
) -> Result<Vec<Encoding>> where
 E: Into<EncodeInput<'s>> + Send,

Encode all the sentences in parallel, using multiple threads. The offsets on each Encoding will be relative to chars instead of bytes.

source

pub fn decode_batch(
 &self,
 sentences: Vec<Vec<u32>>,
 skip_special_tokens: bool
) -> Result<Vec<String>> where
 M: Send + Sync,

Decode all sentences in parallel

source

pub fn train_from_files<T>(
 &mut self,
 trainer: &mut T,
 files: Vec<String>
) -> Result<&mut Self> where
 T: Trainer<Model = M> + Sync,

Train our Model from files

source

pub fn train<T, I, S>(
 &mut self,
 trainer: &mut T,
 sequences: I
) -> Result<&mut Self> where
 T: Trainer<Model = M> + Sync,
 I: Iterator<Item = S> + Send,
 S: AsRef<str> + Send,

Train our Model, using the given Trainer and iterator

source

pub fn to_string(&self, pretty: bool) -> Result<String>

Serialize the current tokenizer as a String

source

pub fn save<P: AsRef<Path>>(&self, path: P, pretty: bool) -> Result<()>

Save the current tokenizer at the given path

Trait Implementations

source

impl Clone for Tokenizer

source

fn clone(&self) -> Tokenizer

Returns a copy of the value. Read more

1.0.0 · source

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more

source

impl Debug for Tokenizer

source

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more

source

impl Deref for Tokenizer

type Target = TokenizerImpl<ModelWrapper, NormalizerWrapper, PreTokenizerWrapper, PostProcessorWrapper, DecoderWrapper>

The resulting type after dereferencing.

source

fn deref(&self) -> &Self::Target

Dereferences the value.

source

impl DerefMut for Tokenizer

source

fn deref_mut(&mut self) -> &mut Self::Target

Mutably dereferences the value.

source

impl<'de> Deserialize<'de> for Tokenizer

source

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where
__D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer. Read more

source

impl<M, N, PT, PP, D> From<TokenizerImpl<M, N, PT, PP, D>> for Tokenizer where
 M: Into<ModelWrapper>,
 N: Into<NormalizerWrapper>,
 PT: Into<PreTokenizerWrapper>,
 PP: Into<PostProcessorWrapper>,
 D: Into<DecoderWrapper>,

source

fn from(t: TokenizerImpl<M, N, PT, PP, D>) -> Self

Performs the conversion.

source

impl FromStr for Tokenizer

type Err = Box<dyn Error + Send + Sync>

The associated error which can be returned from parsing.

source

fn from_str(s: &str) -> Result<Self>

Parses a string s to return a value of this type. Read more

source

impl Serialize for Tokenizer

source

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where
S: Serializer,

Serialize this value into the given Serde serializer. Read more

Auto Trait Implementations

impl RefUnwindSafe for Tokenizer

impl Send for Tokenizer

impl Sync for Tokenizer

impl Unpin for Tokenizer

impl UnwindSafe for Tokenizer

Blanket Implementations

source

impl<T> Any for T where
T: 'static + ?Sized,

source

pub fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

source

impl<T> Borrow<T> for T where
T: ?Sized,

source

pub fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

source

impl<T> BorrowMut<T> for T where
T: ?Sized,

source

pub fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

impl<T> Conv for T

fn conv<T>(self) -> T where
Self: Into<T>,

Converts self into T using Into<T>. Read more

impl<T> Conv for T

fn conv<T>(self) -> T where
Self: Into<T>,

Converts self into a target type. Read more

impl<T> FmtForward for T

fn fmt_binary(self) -> FmtBinary<Self> where
Self: Binary,

Causes self to use its Binary implementation when Debug-formatted.

fn fmt_display(self) -> FmtDisplay<Self> where
Self: Display,

Causes self to use its Display implementation when Debug-formatted. Read more

fn fmt_lower_exp(self) -> FmtLowerExp<Self> where
Self: LowerExp,

Causes self to use its LowerExp implementation when Debug-formatted. Read more

fn fmt_lower_hex(self) -> FmtLowerHex<Self> where
Self: LowerHex,

Causes self to use its LowerHex implementation when Debug-formatted. Read more

fn fmt_octal(self) -> FmtOctal<Self> where
Self: Octal,

Causes self to use its Octal implementation when Debug-formatted.

fn fmt_pointer(self) -> FmtPointer<Self> where
Self: Pointer,

Causes self to use its Pointer implementation when Debug-formatted. Read more

fn fmt_upper_exp(self) -> FmtUpperExp<Self> where
Self: UpperExp,

Causes self to use its UpperExp implementation when Debug-formatted. Read more

fn fmt_upper_hex(self) -> FmtUpperHex<Self> where
Self: UpperHex,

Causes self to use its UpperHex implementation when Debug-formatted. Read more

source

impl<T> From<T> for T

source

pub fn from(t: T) -> T

Performs the conversion.

source

impl<T, U> Into for T where
U: From<T>,

source

pub fn into(self) -> U

Performs the conversion.

impl<T> Pipe for T where
T: ?Sized,

fn pipe<R>(self, func: impl FnOnce(Self) -> R) -> R

Pipes by value. This is generally the method you want to use. Read more

fn pipe_ref<'a, R>(&'a self, func: impl FnOnce(&'a Self) -> R) -> R where
R: 'a,

Borrows self and passes that borrow into the pipe function. Read more

fn pipe_ref_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> R where
R: 'a,

Mutably borrows self and passes that borrow into the pipe function. Read more

fn pipe_borrow<'a, B, R>(&'a self, func: impl FnOnce(&'a B) -> R) -> R where
 Self: Borrow,
 B: 'a + ?Sized,
 R: 'a,

Borrows self, then passes self.borrow() into the pipe function. Read more

fn pipe_borrow_mut<'a, B, R>(
 &'a mut self,
 func: impl FnOnce(&'a mut B) -> R
) -> R where
 Self: BorrowMut,
 B: 'a + ?Sized,
 R: 'a,

Mutably borrows self, then passes self.borrow_mut() into the pipe function. Read more

fn pipe_as_ref<'a, U, R>(&'a self, func: impl FnOnce(&'a U) -> R) -> R where
 Self: AsRef,
 U: 'a + ?Sized,
 R: 'a,

Borrows self, then passes self.as_ref() into the pipe function.

fn pipe_as_mut<'a, U, R>(&'a mut self, func: impl FnOnce(&'a mut U) -> R) -> R where
 Self: AsMut,
 U: 'a + ?Sized,
 R: 'a,

Mutably borrows self, then passes self.as_mut() into the pipe function. Read more

fn pipe_deref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
 Self: Deref<Target = T>,
 T: 'a + ?Sized,
 R: 'a,

Borrows self, then passes self.deref() into the pipe function.

fn pipe_deref_mut<'a, T, R>(
 &'a mut self,
 func: impl FnOnce(&'a mut T) -> R
) -> R where
 Self: DerefMut<Target = T> + Deref,
 T: 'a + ?Sized,
 R: 'a,

Mutably borrows self, then passes self.deref_mut() into the pipe function. Read more

impl<T> Pipe for T

fn pipe<R>(self, func: impl FnOnce(Self) -> R) -> R

Pipes a value into a function that cannot ordinarily be called in suffix position. Read more

impl<T> PipeAsRef for T

fn pipe_as_ref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
 Self: AsRef<T>,
 T: 'a,
 R: 'a,

Pipes a trait borrow into a function that cannot normally be called in suffix position. Read more

fn pipe_as_mut<'a, T, R>(&'a mut self, func: impl FnOnce(&'a mut T) -> R) -> R where
 Self: AsMut<T>,
 T: 'a,
 R: 'a,

Pipes a trait mutable borrow into a function that cannot normally be called in suffix position. Read more

impl<T> PipeBorrow for T

fn pipe_borrow<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
 Self: Borrow<T>,
 T: 'a,
 R: 'a,

Pipes a trait borrow into a function that cannot normally be called in suffix position. Read more

fn pipe_borrow_mut<'a, T, R>(
 &'a mut self,
 func: impl FnOnce(&'a mut T) -> R
) -> R where
 Self: BorrowMut<T>,
 T: 'a,
 R: 'a,

Pipes a trait mutable borrow into a function that cannot normally be called in suffix position. Read more

impl<T> PipeDeref for T

fn pipe_deref<'a, R>(&'a self, func: impl FnOnce(&'a Self::Target) -> R) -> R where
Self: Deref,
R: 'a,

Pipes a dereference into a function that cannot normally be called in suffix position. Read more

fn pipe_deref_mut<'a, R>(
 &'a mut self,
 func: impl FnOnce(&'a mut Self::Target) -> R
) -> R where
 Self: DerefMut,
 R: 'a,

Pipes a mutable dereference into a function that cannot normally be called in suffix position. Read more

impl<T> PipeRef for T

fn pipe_ref<'a, R>(&'a self, func: impl FnOnce(&'a Self) -> R) -> R where
R: 'a,

Pipes a reference into a function that cannot ordinarily be called in suffix position. Read more

fn pipe_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> R where
R: 'a,

Pipes a mutable reference into a function that cannot ordinarily be called in suffix position. Read more

impl<T> Pointable for T

pub const ALIGN: usize

The alignment of pointer.

type Init = T

The type for initializers.

pub unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more

pub unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more

pub unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more

pub unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more

impl<T> Tap for T

fn tap(self, func: impl FnOnce(&Self)) -> Self

Immutable access to a value. Read more

fn tap_mut(self, func: impl FnOnce(&mut Self)) -> Self

Mutable access to a value. Read more

fn tap_borrow(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow,
B: ?Sized,

Immutable access to the Borrow of a value. Read more

fn tap_borrow_mut(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut,
B: ?Sized,

Mutable access to the BorrowMut of a value. Read more

fn tap_ref<R>(self, func: impl FnOnce(&R)) -> Self where
Self: AsRef<R>,
R: ?Sized,

Immutable access to the AsRef<R> view of a value. Read more

fn tap_ref_mut<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,

Mutable access to the AsMut<R> view of a value. Read more

fn tap_deref<T>(self, func: impl FnOnce(&T)) -> Self where
Self: Deref<Target = T>,
T: ?Sized,

Immutable access to the Deref::Target of a value. Read more

fn tap_deref_mut<T>(self, func: impl FnOnce(&mut T)) -> Self where
Self: DerefMut<Target = T> + Deref,
T: ?Sized,

Mutable access to the Deref::Target of a value. Read more

fn tap_dbg(self, func: impl FnOnce(&Self)) -> Self

Calls .tap() only in debug builds, and is erased in release builds.

fn tap_mut_dbg(self, func: impl FnOnce(&mut Self)) -> Self

Calls .tap_mut() only in debug builds, and is erased in release builds. Read more

fn tap_borrow_dbg(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow,
B: ?Sized,

Calls .tap_borrow() only in debug builds, and is erased in release builds. Read more

fn tap_borrow_mut_dbg(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut,
B: ?Sized,

Calls .tap_borrow_mut() only in debug builds, and is erased in release builds. Read more

fn tap_ref_dbg<R>(self, func: impl FnOnce(&R)) -> Self where
Self: AsRef<R>,
R: ?Sized,

Calls .tap_ref() only in debug builds, and is erased in release builds. Read more

fn tap_ref_mut_dbg<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,

Calls .tap_ref_mut() only in debug builds, and is erased in release builds. Read more

fn tap_deref_dbg<T>(self, func: impl FnOnce(&T)) -> Self where
Self: Deref<Target = T>,
T: ?Sized,

Calls .tap_deref() only in debug builds, and is erased in release builds. Read more

fn tap_deref_mut_dbg<T>(self, func: impl FnOnce(&mut T)) -> Self where
Self: DerefMut<Target = T> + Deref,
T: ?Sized,

Calls .tap_deref_mut() only in debug builds, and is erased in release builds. Read more

impl<T> Tap for T

fn tap<F, R>(self, func: F) -> Self where
F: FnOnce(&Self) -> R,

Provides immutable access for inspection. Read more

fn tap_dbg<F, R>(self, func: F) -> Self where
F: FnOnce(&Self) -> R,

Calls tap in debug builds, and does nothing in release builds.

fn tap_mut<F, R>(self, func: F) -> Self where
F: FnOnce(&mut Self) -> R,

Provides mutable access for modification. Read more

fn tap_mut_dbg<F, R>(self, func: F) -> Self where
F: FnOnce(&mut Self) -> R,

Calls tap_mut in debug builds, and does nothing in release builds.

impl<T, U> TapAsRef for T where
U: ?Sized,

fn tap_ref<F, R>(self, func: F) -> Self where
Self: AsRef<T>,
F: FnOnce(&T) -> R,

Provides immutable access to the reference for inspection.

fn tap_ref_dbg<F, R>(self, func: F) -> Self where
Self: AsRef<T>,
F: FnOnce(&T) -> R,

Calls tap_ref in debug builds, and does nothing in release builds.

fn tap_ref_mut<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,

Provides mutable access to the reference for modification.

fn tap_ref_mut_dbg<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,

Calls tap_ref_mut in debug builds, and does nothing in release builds.

impl<T, U> TapBorrow for T where
U: ?Sized,

fn tap_borrow<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,

Provides immutable access to the borrow for inspection. Read more

fn tap_borrow_dbg<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,

Calls tap_borrow in debug builds, and does nothing in release builds.

fn tap_borrow_mut<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,

Provides mutable access to the borrow for modification.

fn tap_borrow_mut_dbg<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,

Calls tap_borrow_mut in debug builds, and does nothing in release builds. Read more

impl<T> TapDeref for T

fn tap_deref<F, R>(self, func: F) -> Self where
Self: Deref,
F: FnOnce(&Self::Target) -> R,

Immutably dereferences self for inspection.

fn tap_deref_dbg<F, R>(self, func: F) -> Self where
Self: Deref,
F: FnOnce(&Self::Target) -> R,

Calls tap_deref in debug builds, and does nothing in release builds.

fn tap_deref_mut<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,

Mutably dereferences self for modification.

fn tap_deref_mut_dbg<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,

Calls tap_deref_mut in debug builds, and does nothing in release builds. Read more

source

impl<T> ToOwned for T where
T: Clone,

type Owned = T

The resulting type after obtaining ownership.

source

pub fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more

source

pub fn clone_into(&self, target: &mut T)

🔬 This is a nightly-only experimental API. (toowned_clone_into)

Uses borrowed data to replace owned data, usually by cloning. Read more

impl<T> TryConv for T

fn try_conv<T>(self) -> Result<T, Self::Error> where
Self: TryInto<T>,

Attempts to convert self into T using TryInto<T>. Read more

impl<T> TryConv for T

fn try_conv<T>(self) -> Result<T, Self::Error> where
Self: TryInto<T>,

Attempts to convert self into a target type. Read more

source

impl<T, U> TryFrom for T where
U: Into<T>,

type Error = Infallible

The type returned in the event of a conversion error.

source

pub fn try_from(value: U) -> Result<T, <T as TryFrom>::Error>

Performs the conversion.

source

impl<T, U> TryInto for T where
U: TryFrom<T>,

type Error = >::Error

The type returned in the event of a conversion error.

source

pub fn try_into(self) -> Result<U, >::Error>

Performs the conversion.

impl<V, T> VZip<V> for T where
V: MultiLane<T>,

pub fn vzip(self) -> V

source

Struct mako::tokenization::hf_tokenizers::tokenizer::Tokenizer

Implementations

impl Tokenizer

pub fn new(model: impl Into<ModelWrapper>) -> Self

pub fn into_inner( self) -> TokenizerImpl<ModelWrapper, NormalizerWrapper, PreTokenizerWrapper, PostProcessorWrapper, DecoderWrapper>

pub fn from_file<P: AsRef<Path>>(file: P) -> Result<Self>

Methods from Deref<Target = TokenizerImpl<ModelWrapper, NormalizerWrapper, PreTokenizerWrapper, PostProcessorWrapper, DecoderWrapper>>

pub fn with_normalizer(&mut self, normalizer: impl Into<N>) -> &mut Self

pub fn get_normalizer(&self) -> Option<&N>

pub fn with_pre_tokenizer(&mut self, pre_tokenizer: impl Into<PT>) -> &mut Self

pub fn get_pre_tokenizer(&self) -> Option<&PT>

pub fn with_post_processor( &mut self, post_processor: impl Into<PP>) -> &mut Self

pub fn get_post_processor(&self) -> Option<&PP>

pub fn with_decoder(&mut self, decoder: impl Into<D>) -> &mut Self

pub fn get_decoder(&self) -> Option<&D>

pub fn with_model(&mut self, model: impl Into<M>) -> &mut Self

pub fn get_model(&self) -> &M

pub fn with_truncation(&mut self, trunc: Option<TruncationParams>) -> &mut Self

pub fn get_truncation(&self) -> Option<&TruncationParams>

pub fn get_truncation_mut(&mut self) -> Option<&mut TruncationParams>

pub fn with_padding(&mut self, padding: Option<PaddingParams>) -> &mut Self

pub fn get_padding(&self) -> Option<&PaddingParams>

pub fn get_padding_mut(&mut self) -> Option<&mut PaddingParams>

pub fn get_vocab(&self, with_added_tokens: bool) -> HashMap<String, u32>

pub fn get_vocab_size(&self, with_added_tokens: bool) -> usize

pub fn token_to_id(&self, token: &str) -> Option<u32>

pub fn id_to_token(&self, id: u32) -> Option<String>

pub fn encode<'s, E>( &self, input: E, add_special_tokens: bool) -> Result<Encoding> where E: Into<EncodeInput<'s>>,

pub fn encode_char_offsets<'s, E>( &self, input: E, add_special_tokens: bool) -> Result<Encoding> where E: Into<EncodeInput<'s>>,

pub fn decode(&self, ids: Vec<u32>, skip_special_tokens: bool) -> Result<String>

pub fn add_special_tokens(&mut self, tokens: &[AddedToken]) -> usize

pub fn add_tokens(&mut self, tokens: &[AddedToken]) -> usize

pub fn post_process( &self, encoding: Encoding, pair_encoding: Option<Encoding>, add_special_tokens: bool) -> Result<Encoding>

pub fn encode_batch<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool) -> Result<Vec<Encoding>> where E: Into<EncodeInput<'s>> + Send,

pub fn encode_batch_char_offsets<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool) -> Result<Vec<Encoding>> where E: Into<EncodeInput<'s>> + Send,

pub fn decode_batch( &self, sentences: Vec<Vec<u32>>, skip_special_tokens: bool) -> Result<Vec<String>> where M: Send + Sync,

pub fn train_from_files<T>( &mut self, trainer: &mut T, files: Vec<String>) -> Result<&mut Self> where T: Trainer<Model = M> + Sync,

pub fn train<T, I, S>( &mut self, trainer: &mut T, sequences: I) -> Result<&mut Self> where T: Trainer<Model = M> + Sync, I: Iterator<Item = S> + Send, S: AsRef<str> + Send,

pub fn to_string(&self, pretty: bool) -> Result<String>

pub fn save<P: AsRef<Path>>(&self, path: P, pretty: bool) -> Result<()>

Trait Implementations

impl Clone for Tokenizer

fn clone(&self) -> Tokenizer

fn clone_from(&mut self, source: &Self)

impl Debug for Tokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

impl Deref for Tokenizer

type Target = TokenizerImpl<ModelWrapper, NormalizerWrapper, PreTokenizerWrapper, PostProcessorWrapper, DecoderWrapper>

fn deref(&self) -> &Self::Target

impl DerefMut for Tokenizer

fn deref_mut(&mut self) -> &mut Self::Target

impl<'de> Deserialize<'de> for Tokenizer

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error> where __D: Deserializer<'de>,

impl<M, N, PT, PP, D> From<TokenizerImpl<M, N, PT, PP, D>> for Tokenizer where M: Into<ModelWrapper>, N: Into<NormalizerWrapper>, PT: Into<PreTokenizerWrapper>, PP: Into<PostProcessorWrapper>, D: Into<DecoderWrapper>,

fn from(t: TokenizerImpl<M, N, PT, PP, D>) -> Self

impl FromStr for Tokenizer

type Err = Box<dyn Error + Send + Sync>

fn from_str(s: &str) -> Result<Self>

impl Serialize for Tokenizer

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error> where __S: Serializer,

Auto Trait Implementations

impl RefUnwindSafe for Tokenizer

impl Send for Tokenizer

impl Sync for Tokenizer

impl Unpin for Tokenizer

impl UnwindSafe for Tokenizer

Blanket Implementations

impl<T> Any for T where T: 'static + ?Sized,

pub fn type_id(&self) -> TypeId

impl<T> Borrow<T> for T where T: ?Sized,

pub fn borrow(&self) -> &T

impl<T> BorrowMut<T> for T where T: ?Sized,

pub fn borrow_mut(&mut self) -> &mut T

impl<T> Conv for T

fn conv<T>(self) -> T where Self: Into<T>,

impl<T> Conv for T

fn conv<T>(self) -> T where Self: Into<T>,

impl<T> FmtForward for T

fn fmt_binary(self) -> FmtBinary<Self> where Self: Binary,

fn fmt_display(self) -> FmtDisplay<Self> where Self: Display,

pub fn into_inner(
self
) -> TokenizerImpl<ModelWrapper, NormalizerWrapper, PreTokenizerWrapper, PostProcessorWrapper, DecoderWrapper>

pub fn with_post_processor(
&mut self,
post_processor: impl Into<PP>
) -> &mut Self

pub fn encode<'s, E>(
&self,
input: E,
add_special_tokens: bool
) -> Result<Encoding> where
E: Into<EncodeInput<'s>>,

pub fn encode_char_offsets<'s, E>(
&self,
input: E,
add_special_tokens: bool
) -> Result<Encoding> where
E: Into<EncodeInput<'s>>,

pub fn add_special_tokens(&mut self, tokens: &[AddedToken ]) -> usize

pub fn add_tokens(&mut self, tokens: &[AddedToken ]) -> usize

pub fn post_process(
&self,
encoding: Encoding,
pair_encoding: Option<Encoding>,
add_special_tokens: bool
) -> Result<Encoding>

pub fn encode_batch<'s, E>(
&self,
inputs: Vec<E>,
add_special_tokens: bool
) -> Result<Vec<Encoding>> where
E: Into<EncodeInput<'s>> + Send,

pub fn encode_batch_char_offsets<'s, E>(
&self,
inputs: Vec<E>,
add_special_tokens: bool
) -> Result<Vec<Encoding>> where
E: Into<EncodeInput<'s>> + Send,

pub fn decode_batch(
&self,
sentences: Vec<Vec<u32>>,
skip_special_tokens: bool
) -> Result<Vec<String>> where
M: Send + Sync,

pub fn train_from_files<T>(
&mut self,
trainer: &mut T,
files: Vec<String>
) -> Result<&mut Self> where
T: Trainer<Model = M> + Sync,

pub fn train<T, I, S>(
&mut self,
trainer: &mut T,
sequences: I
) -> Result<&mut Self> where
T: Trainer<Model = M> + Sync,
I: Iterator<Item = S> + Send,
S: AsRef<str> + Send,

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where
__D: Deserializer<'de>,

impl<M, N, PT, PP, D> From<TokenizerImpl<M, N, PT, PP, D>> for Tokenizer where
M: Into<ModelWrapper>,
N: Into<NormalizerWrapper>,
PT: Into<PreTokenizerWrapper>,
PP: Into<PostProcessorWrapper>,
D: Into<DecoderWrapper>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where
S: Serializer,

impl<T> Any for T where
T: 'static + ?Sized,

impl<T> Borrow<T> for T where
T: ?Sized,

impl<T> BorrowMut<T> for T where
T: ?Sized,

fn conv<T>(self) -> T where
Self: Into<T>,

fn conv<T>(self) -> T where
Self: Into<T>,

fn fmt_binary(self) -> FmtBinary<Self> where
Self: Binary,

fn fmt_display(self) -> FmtDisplay<Self> where
Self: Display,

fn fmt_lower_exp(self) -> FmtLowerExp<Self> where
Self: LowerExp,

fn fmt_lower_hex(self) -> FmtLowerHex<Self> where
Self: LowerHex,

fn fmt_octal(self) -> FmtOctal<Self> where
Self: Octal,

fn fmt_pointer(self) -> FmtPointer<Self> where
Self: Pointer,

fn fmt_upper_exp(self) -> FmtUpperExp<Self> where
Self: UpperExp,

fn fmt_upper_hex(self) -> FmtUpperHex<Self> where
Self: UpperHex,

impl<T, U> Into<U> for T where
U: From<T>,

impl<T> Pipe for T where
T: ?Sized,

fn pipe_ref<'a, R>(&'a self, func: impl FnOnce(&'a Self) -> R) -> R where
R: 'a,

fn pipe_ref_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> R where
R: 'a,

fn pipe_borrow<'a, B, R>(&'a self, func: impl FnOnce(&'a B) -> R) -> R where
Self: Borrow<B>,
B: 'a + ?Sized,
R: 'a,

fn pipe_borrow_mut<'a, B, R>(
&'a mut self,
func: impl FnOnce(&'a mut B) -> R
) -> R where
Self: BorrowMut<B>,
B: 'a + ?Sized,
R: 'a,

fn pipe_as_ref<'a, U, R>(&'a self, func: impl FnOnce(&'a U) -> R) -> R where
Self: AsRef<U>,
U: 'a + ?Sized,
R: 'a,

fn pipe_as_mut<'a, U, R>(&'a mut self, func: impl FnOnce(&'a mut U) -> R) -> R where
Self: AsMut<U>,
U: 'a + ?Sized,
R: 'a,

fn pipe_deref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Deref<Target = T>,
T: 'a + ?Sized,
R: 'a,

fn pipe_deref_mut<'a, T, R>(
&'a mut self,
func: impl FnOnce(&'a mut T) -> R
) -> R where
Self: DerefMut<Target = T> + Deref,
T: 'a + ?Sized,
R: 'a,

fn pipe_as_ref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: AsRef<T>,
T: 'a,
R: 'a,

fn pipe_as_mut<'a, T, R>(&'a mut self, func: impl FnOnce(&'a mut T) -> R) -> R where
Self: AsMut<T>,
T: 'a,
R: 'a,

fn pipe_borrow<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Borrow<T>,
T: 'a,
R: 'a,

fn pipe_borrow_mut<'a, T, R>(
&'a mut self,
func: impl FnOnce(&'a mut T) -> R
) -> R where
Self: BorrowMut<T>,
T: 'a,
R: 'a,

fn pipe_deref<'a, R>(&'a self, func: impl FnOnce(&'a Self::Target) -> R) -> R where
Self: Deref,
R: 'a,

fn pipe_deref_mut<'a, R>(
&'a mut self,
func: impl FnOnce(&'a mut Self::Target) -> R
) -> R where
Self: DerefMut,
R: 'a,

fn pipe_ref<'a, R>(&'a self, func: impl FnOnce(&'a Self) -> R) -> R where
R: 'a,

fn pipe_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> R where
R: 'a,

fn tap_borrow<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,

fn tap_borrow_mut<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,

fn tap_ref<R>(self, func: impl FnOnce(&R)) -> Self where
Self: AsRef<R>,
R: ?Sized,

fn tap_ref_mut<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,

fn tap_deref<T>(self, func: impl FnOnce(&T)) -> Self where
Self: Deref<Target = T>,
T: ?Sized,

fn tap_deref_mut<T>(self, func: impl FnOnce(&mut T)) -> Self where
Self: DerefMut<Target = T> + Deref,
T: ?Sized,

fn tap_borrow_dbg<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,

fn tap_borrow_mut_dbg<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,

fn tap_ref_dbg<R>(self, func: impl FnOnce(&R)) -> Self where
Self: AsRef<R>,
R: ?Sized,

fn tap_ref_mut_dbg<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,

fn tap_deref_dbg<T>(self, func: impl FnOnce(&T)) -> Self where
Self: Deref<Target = T>,
T: ?Sized,

fn tap_deref_mut_dbg<T>(self, func: impl FnOnce(&mut T)) -> Self where
Self: DerefMut<Target = T> + Deref,
T: ?Sized,

fn tap<F, R>(self, func: F) -> Self where
F: FnOnce(&Self) -> R,

fn tap_dbg<F, R>(self, func: F) -> Self where
F: FnOnce(&Self) -> R,

fn tap_mut<F, R>(self, func: F) -> Self where
F: FnOnce(&mut Self) -> R,