[−][src]Struct tokenizers::pre_tokenizers::metaspace::Metaspace

pub struct Metaspace { /* fields omitted */ }

Replaces all the whitespaces by the provided meta character and then splits on this character

Methods

`impl Metaspace`[src]

`pub fn new(replacement: char, add_prefix_space: bool) -> Self`[src]

Trait Implementations

`impl Decoder for Metaspace`[src]

`fn decode(&self, tokens: Vec<String>) -> Result<String>`[src]

`impl Default for Metaspace`[src]

`fn default() -> Self`[src]

`impl PreTokenizer for Metaspace`[src]

`fn pre_tokenize( &self, normalized: &mut NormalizedString ) -> Result<Vec<(String, Offsets)>>`[src]

Auto Trait Implementations

`impl RefUnwindSafe for Metaspace`

`impl Send for Metaspace`

`impl Sync for Metaspace`

`impl Unpin for Metaspace`

`impl UnwindSafe for Metaspace`

Blanket Implementations

`impl<T> Any for T where T: 'static + ?Sized,` [src]

`fn type_id(&self) -> TypeId`[src]

`impl<T> Borrow<T> for T where T: ?Sized,` [src]

`fn borrow(&self) -> &T`[src]

`impl<T> BorrowMut<T> for T where T: ?Sized,` [src]

`fn borrow_mut(&mut self) -> &mut T`[src]

`impl<T> From<T> for T`[src]

`fn from(t: T) -> T`[src]

`impl<T, U> Into for T where U: From<T>,` [src]

`fn into(self) -> U`[src]

`impl<T, U> TryFrom for T where U: Into<T>,` [src]

`type Error = Infallible`

The type returned in the event of a conversion error.

`fn try_from(value: U) -> Result<T, <T as TryFrom>::Error>`[src]

`impl<T, U> TryInto for T where U: TryFrom<T>,` [src]

`type Error = >::Error`

The type returned in the event of a conversion error.

`fn try_into(self) -> Result<U, >::Error>`[src]

`impl<V, T> VZip<V> for T where V: MultiLane<T>,`

`fn vzip(self) -> V`

[−][src]Struct tokenizers::pre_tokenizers::metaspace::Metaspace

Methods

`impl Metaspace`[src]

`pub fn new(replacement: char, add_prefix_space: bool) -> Self`[src]

Trait Implementations

`impl Decoder for Metaspace`[src]

`fn decode(&self, tokens: Vec<String>) -> Result<String>`[src]

`impl Default for Metaspace`[src]

`fn default() -> Self`[src]

`impl PreTokenizer for Metaspace`[src]

`fn pre_tokenize( &self, normalized: &mut NormalizedString ) -> Result<Vec<(String, Offsets)>>`[src]

Auto Trait Implementations

`impl RefUnwindSafe for Metaspace`

`impl Send for Metaspace`

`impl Sync for Metaspace`

`impl Unpin for Metaspace`

`impl UnwindSafe for Metaspace`

Blanket Implementations

`impl<T> Any for T where T: 'static + ?Sized,` [src]

`fn type_id(&self) -> TypeId`[src]

`impl<T> Borrow<T> for T where T: ?Sized,` [src]

`fn borrow(&self) -> &T`[src]

`impl<T> BorrowMut<T> for T where T: ?Sized,` [src]

`fn borrow_mut(&mut self) -> &mut T`[src]

`impl<T> From<T> for T`[src]

`fn from(t: T) -> T`[src]

`impl<T, U> Into<U> for T where U: From<T>,` [src]

`fn into(self) -> U`[src]

`impl<T, U> TryFrom<U> for T where U: Into<T>,` [src]

`type Error = Infallible`

`fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>`[src]

`impl<T, U> TryInto<U> for T where U: TryFrom<T>,` [src]

`type Error = <U as TryFrom<T>>::Error`

`fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>`[src]

`impl<V, T> VZip<V> for T where V: MultiLane<T>,`

`fn vzip(self) -> V`

[−][src]Struct tokenizers::pre_tokenizers::metaspace::Metaspace

Methods

impl Metaspace[src]

pub fn new(replacement: char, add_prefix_space: bool) -> Self[src]

Trait Implementations

impl Decoder for Metaspace[src]

fn decode(&self, tokens: Vec<String>) -> Result<String>[src]

impl Default for Metaspace[src]

fn default() -> Self[src]

impl PreTokenizer for Metaspace[src]

fn pre_tokenize( &self, normalized: &mut NormalizedString) -> Result<Vec<(String, Offsets)>>[src]

Auto Trait Implementations

impl RefUnwindSafe for Metaspace

impl Send for Metaspace

impl Sync for Metaspace

impl Unpin for Metaspace

impl UnwindSafe for Metaspace

Blanket Implementations

impl<T> Any for T where T: 'static + ?Sized, [src]

fn type_id(&self) -> TypeId[src]

impl<T> Borrow<T> for T where T: ?Sized, [src]

fn borrow(&self) -> &T[src]

impl<T> BorrowMut<T> for T where T: ?Sized, [src]

fn borrow_mut(&mut self) -> &mut T[src]

impl<T> From<T> for T[src]

fn from(t: T) -> T[src]

impl<T, U> Into<U> for T where U: From<T>, [src]

fn into(self) -> U[src]

impl<T, U> TryFrom<U> for T where U: Into<T>, [src]

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>[src]

impl<T, U> TryInto<U> for T where U: TryFrom<T>, [src]

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>[src]

impl<V, T> VZip<V> for T where V: MultiLane<T>,

fn vzip(self) -> V

`impl Metaspace`[src]

`pub fn new(replacement: char, add_prefix_space: bool) -> Self`[src]

`impl Decoder for Metaspace`[src]

`fn decode(&self, tokens: Vec<String>) -> Result<String>`[src]

`impl Default for Metaspace`[src]

`fn default() -> Self`[src]

`impl PreTokenizer for Metaspace`[src]

`fn pre_tokenize( &self, normalized: &mut NormalizedString ) -> Result<Vec<(String, Offsets)>>`[src]

`impl RefUnwindSafe for Metaspace`

`impl Send for Metaspace`

`impl Sync for Metaspace`

`impl Unpin for Metaspace`

`impl UnwindSafe for Metaspace`

`impl<T> Any for T where T: 'static + ?Sized,` [src]

`fn type_id(&self) -> TypeId`[src]

`impl<T> Borrow<T> for T where T: ?Sized,` [src]

`fn borrow(&self) -> &T`[src]

`impl<T> BorrowMut<T> for T where T: ?Sized,` [src]

`fn borrow_mut(&mut self) -> &mut T`[src]

`impl<T> From<T> for T`[src]

`fn from(t: T) -> T`[src]

`impl<T, U> Into<U> for T where U: From<T>,` [src]

`fn into(self) -> U`[src]

`impl<T, U> TryFrom<U> for T where U: Into<T>,` [src]

`type Error = Infallible`

`fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>`[src]

`impl<T, U> TryInto<U> for T where U: TryFrom<T>,` [src]

`type Error = <U as TryFrom<T>>::Error`

`fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>`[src]

`impl<V, T> VZip<V> for T where V: MultiLane<T>,`

`fn vzip(self) -> V`