Struct mako::tokenization::hf_tokenizers::tokenizer::AddedToken
source · [−]pub struct AddedToken {
pub content: String,
pub single_word: bool,
pub lstrip: bool,
pub rstrip: bool,
pub normalized: bool,
}
Expand description
Represent a token added by the user on top of the existing Model vocabulary. AddedToken can be configured to specify the behavior they should have in various situations like:
- Whether they should only match single words
- Whether to include any whitespace on its left or right
Fields
content: String
The content of the added token
single_word: bool
Whether this token must be a single word or can break words
lstrip: bool
Whether this token should strip whitespaces on its left
rstrip: bool
Whether this token should strip whitespaces on its right
normalized: bool
Whether this token should be normalized
Implementations
Build this token from the given content, specifying if it is intented to be a special token. Special tokens are not normalized by default.
Specify whether this token should only match on whole single words, and never part of a word.
Specify whether this token should include all the whitespaces on its left, in order to strip them out.
Specify whether this token should include all the whitespaces on its right, in order to strip them out.
Specify whether this token should be normalized and match against its normalized version in the input text.
Retrive the pattern built for this token, according to all the specified parameters.
Trait Implementations
fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error> where
__D: Deserializer<'de>,
fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error> where
__D: Deserializer<'de>,
Deserialize this value from the given Serde deserializer. Read more
Auto Trait Implementations
impl RefUnwindSafe for AddedToken
impl Send for AddedToken
impl Sync for AddedToken
impl Unpin for AddedToken
impl UnwindSafe for AddedToken
Blanket Implementations
Mutably borrows from an owned value. Read more
fn fmt_binary(self) -> FmtBinary<Self> where
Self: Binary,
fn fmt_binary(self) -> FmtBinary<Self> where
Self: Binary,
Causes self
to use its Binary
implementation when Debug
-formatted.
fn fmt_display(self) -> FmtDisplay<Self> where
Self: Display,
fn fmt_display(self) -> FmtDisplay<Self> where
Self: Display,
Causes self
to use its Display
implementation when
Debug
-formatted. Read more
fn fmt_lower_exp(self) -> FmtLowerExp<Self> where
Self: LowerExp,
fn fmt_lower_exp(self) -> FmtLowerExp<Self> where
Self: LowerExp,
Causes self
to use its LowerExp
implementation when
Debug
-formatted. Read more
fn fmt_lower_hex(self) -> FmtLowerHex<Self> where
Self: LowerHex,
fn fmt_lower_hex(self) -> FmtLowerHex<Self> where
Self: LowerHex,
Causes self
to use its LowerHex
implementation when
Debug
-formatted. Read more
Causes self
to use its Octal
implementation when Debug
-formatted.
fn fmt_pointer(self) -> FmtPointer<Self> where
Self: Pointer,
fn fmt_pointer(self) -> FmtPointer<Self> where
Self: Pointer,
Causes self
to use its Pointer
implementation when
Debug
-formatted. Read more
fn fmt_upper_exp(self) -> FmtUpperExp<Self> where
Self: UpperExp,
fn fmt_upper_exp(self) -> FmtUpperExp<Self> where
Self: UpperExp,
Causes self
to use its UpperExp
implementation when
Debug
-formatted. Read more
fn fmt_upper_hex(self) -> FmtUpperHex<Self> where
Self: UpperHex,
fn fmt_upper_hex(self) -> FmtUpperHex<Self> where
Self: UpperHex,
Causes self
to use its UpperHex
implementation when
Debug
-formatted. Read more
impl<T> Pipe for T where
T: ?Sized,
impl<T> Pipe for T where
T: ?Sized,
Pipes by value. This is generally the method you want to use. Read more
Borrows self
and passes that borrow into the pipe function. Read more
fn pipe_ref_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> R where
R: 'a,
fn pipe_ref_mut<'a, R>(&'a mut self, func: impl FnOnce(&'a mut Self) -> R) -> R where
R: 'a,
Mutably borrows self
and passes that borrow into the pipe function. Read more
fn pipe_borrow<'a, B, R>(&'a self, func: impl FnOnce(&'a B) -> R) -> R where
Self: Borrow<B>,
B: 'a + ?Sized,
R: 'a,
fn pipe_borrow<'a, B, R>(&'a self, func: impl FnOnce(&'a B) -> R) -> R where
Self: Borrow<B>,
B: 'a + ?Sized,
R: 'a,
Borrows self
, then passes self.borrow()
into the pipe function. Read more
fn pipe_borrow_mut<'a, B, R>(
&'a mut self,
func: impl FnOnce(&'a mut B) -> R
) -> R where
Self: BorrowMut<B>,
B: 'a + ?Sized,
R: 'a,
fn pipe_borrow_mut<'a, B, R>(
&'a mut self,
func: impl FnOnce(&'a mut B) -> R
) -> R where
Self: BorrowMut<B>,
B: 'a + ?Sized,
R: 'a,
Mutably borrows self
, then passes self.borrow_mut()
into the pipe
function. Read more
fn pipe_as_ref<'a, U, R>(&'a self, func: impl FnOnce(&'a U) -> R) -> R where
Self: AsRef<U>,
U: 'a + ?Sized,
R: 'a,
fn pipe_as_ref<'a, U, R>(&'a self, func: impl FnOnce(&'a U) -> R) -> R where
Self: AsRef<U>,
U: 'a + ?Sized,
R: 'a,
Borrows self
, then passes self.as_ref()
into the pipe function.
fn pipe_as_mut<'a, U, R>(&'a mut self, func: impl FnOnce(&'a mut U) -> R) -> R where
Self: AsMut<U>,
U: 'a + ?Sized,
R: 'a,
fn pipe_as_mut<'a, U, R>(&'a mut self, func: impl FnOnce(&'a mut U) -> R) -> R where
Self: AsMut<U>,
U: 'a + ?Sized,
R: 'a,
Mutably borrows self
, then passes self.as_mut()
into the pipe
function. Read more
fn pipe_deref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Deref<Target = T>,
T: 'a + ?Sized,
R: 'a,
fn pipe_deref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Deref<Target = T>,
T: 'a + ?Sized,
R: 'a,
Borrows self
, then passes self.deref()
into the pipe function.
fn pipe_as_ref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: AsRef<T>,
T: 'a,
R: 'a,
fn pipe_as_ref<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: AsRef<T>,
T: 'a,
R: 'a,
Pipes a trait borrow into a function that cannot normally be called in suffix position. Read more
fn pipe_as_mut<'a, T, R>(&'a mut self, func: impl FnOnce(&'a mut T) -> R) -> R where
Self: AsMut<T>,
T: 'a,
R: 'a,
fn pipe_as_mut<'a, T, R>(&'a mut self, func: impl FnOnce(&'a mut T) -> R) -> R where
Self: AsMut<T>,
T: 'a,
R: 'a,
Pipes a trait mutable borrow into a function that cannot normally be called in suffix position. Read more
fn pipe_borrow<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Borrow<T>,
T: 'a,
R: 'a,
fn pipe_borrow<'a, T, R>(&'a self, func: impl FnOnce(&'a T) -> R) -> R where
Self: Borrow<T>,
T: 'a,
R: 'a,
Pipes a trait borrow into a function that cannot normally be called in suffix position. Read more
fn pipe_borrow_mut<'a, T, R>(
&'a mut self,
func: impl FnOnce(&'a mut T) -> R
) -> R where
Self: BorrowMut<T>,
T: 'a,
R: 'a,
fn pipe_borrow_mut<'a, T, R>(
&'a mut self,
func: impl FnOnce(&'a mut T) -> R
) -> R where
Self: BorrowMut<T>,
T: 'a,
R: 'a,
Pipes a trait mutable borrow into a function that cannot normally be called in suffix position. Read more
fn pipe_deref<'a, R>(&'a self, func: impl FnOnce(&'a Self::Target) -> R) -> R where
Self: Deref,
R: 'a,
fn pipe_deref<'a, R>(&'a self, func: impl FnOnce(&'a Self::Target) -> R) -> R where
Self: Deref,
R: 'a,
Pipes a dereference into a function that cannot normally be called in suffix position. Read more
fn pipe_deref_mut<'a, R>(
&'a mut self,
func: impl FnOnce(&'a mut Self::Target) -> R
) -> R where
Self: DerefMut,
R: 'a,
fn pipe_deref_mut<'a, R>(
&'a mut self,
func: impl FnOnce(&'a mut Self::Target) -> R
) -> R where
Self: DerefMut,
R: 'a,
Pipes a mutable dereference into a function that cannot normally be called in suffix position. Read more
Pipes a reference into a function that cannot ordinarily be called in suffix position. Read more
fn tap_borrow<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,
fn tap_borrow<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,
Immutable access to the Borrow<B>
of a value. Read more
fn tap_borrow_mut<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,
fn tap_borrow_mut<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,
Mutable access to the BorrowMut<B>
of a value. Read more
Immutable access to the AsRef<R>
view of a value. Read more
fn tap_ref_mut<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,
fn tap_ref_mut<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,
Mutable access to the AsMut<R>
view of a value. Read more
Immutable access to the Deref::Target
of a value. Read more
Mutable access to the Deref::Target
of a value. Read more
Calls .tap()
only in debug builds, and is erased in release builds.
fn tap_mut_dbg(self, func: impl FnOnce(&mut Self)) -> Self
fn tap_mut_dbg(self, func: impl FnOnce(&mut Self)) -> Self
Calls .tap_mut()
only in debug builds, and is erased in release
builds. Read more
fn tap_borrow_dbg<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,
fn tap_borrow_dbg<B>(self, func: impl FnOnce(&B)) -> Self where
Self: Borrow<B>,
B: ?Sized,
Calls .tap_borrow()
only in debug builds, and is erased in release
builds. Read more
fn tap_borrow_mut_dbg<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,
fn tap_borrow_mut_dbg<B>(self, func: impl FnOnce(&mut B)) -> Self where
Self: BorrowMut<B>,
B: ?Sized,
Calls .tap_borrow_mut()
only in debug builds, and is erased in release
builds. Read more
fn tap_ref_dbg<R>(self, func: impl FnOnce(&R)) -> Self where
Self: AsRef<R>,
R: ?Sized,
fn tap_ref_dbg<R>(self, func: impl FnOnce(&R)) -> Self where
Self: AsRef<R>,
R: ?Sized,
Calls .tap_ref()
only in debug builds, and is erased in release
builds. Read more
fn tap_ref_mut_dbg<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,
fn tap_ref_mut_dbg<R>(self, func: impl FnOnce(&mut R)) -> Self where
Self: AsMut<R>,
R: ?Sized,
Calls .tap_ref_mut()
only in debug builds, and is erased in release
builds. Read more
Provides immutable access for inspection. Read more
Calls tap
in debug builds, and does nothing in release builds.
Provides mutable access for modification. Read more
fn tap_mut_dbg<F, R>(self, func: F) -> Self where
F: FnOnce(&mut Self) -> R,
fn tap_mut_dbg<F, R>(self, func: F) -> Self where
F: FnOnce(&mut Self) -> R,
Calls tap_mut
in debug builds, and does nothing in release builds.
impl<T, U> TapAsRef<U> for T where
U: ?Sized,
impl<T, U> TapAsRef<U> for T where
U: ?Sized,
Provides immutable access to the reference for inspection.
fn tap_ref_dbg<F, R>(self, func: F) -> Self where
Self: AsRef<T>,
F: FnOnce(&T) -> R,
fn tap_ref_dbg<F, R>(self, func: F) -> Self where
Self: AsRef<T>,
F: FnOnce(&T) -> R,
Calls tap_ref
in debug builds, and does nothing in release builds.
fn tap_ref_mut<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,
fn tap_ref_mut<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,
Provides mutable access to the reference for modification.
fn tap_ref_mut_dbg<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,
fn tap_ref_mut_dbg<F, R>(self, func: F) -> Self where
Self: AsMut<T>,
F: FnOnce(&mut T) -> R,
Calls tap_ref_mut
in debug builds, and does nothing in release builds.
impl<T, U> TapBorrow<U> for T where
U: ?Sized,
impl<T, U> TapBorrow<U> for T where
U: ?Sized,
fn tap_borrow<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,
fn tap_borrow<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,
Provides immutable access to the borrow for inspection. Read more
fn tap_borrow_dbg<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,
fn tap_borrow_dbg<F, R>(self, func: F) -> Self where
Self: Borrow<T>,
F: FnOnce(&T) -> R,
Calls tap_borrow
in debug builds, and does nothing in release builds.
fn tap_borrow_mut<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,
fn tap_borrow_mut<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,
Provides mutable access to the borrow for modification.
fn tap_borrow_mut_dbg<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,
fn tap_borrow_mut_dbg<F, R>(self, func: F) -> Self where
Self: BorrowMut<T>,
F: FnOnce(&mut T) -> R,
Calls tap_borrow_mut
in debug builds, and does nothing in release
builds. Read more
Immutably dereferences self
for inspection.
fn tap_deref_dbg<F, R>(self, func: F) -> Self where
Self: Deref,
F: FnOnce(&Self::Target) -> R,
fn tap_deref_dbg<F, R>(self, func: F) -> Self where
Self: Deref,
F: FnOnce(&Self::Target) -> R,
Calls tap_deref
in debug builds, and does nothing in release builds.
fn tap_deref_mut<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,
fn tap_deref_mut<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,
Mutably dereferences self
for modification.
fn tap_deref_mut_dbg<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,
fn tap_deref_mut_dbg<F, R>(self, func: F) -> Self where
Self: DerefMut,
F: FnOnce(&mut Self::Target) -> R,
Calls tap_deref_mut
in debug builds, and does nothing in release
builds. Read more