Struct dataflow::tokenization::AlphabetTokenizer
source · [−]pub struct AlphabetTokenizer {}
Trait Implementations
sourceimpl Clone for AlphabetTokenizer
impl Clone for AlphabetTokenizer
sourcefn clone(&self) -> AlphabetTokenizer
fn clone(&self) -> AlphabetTokenizer
Returns a copy of the value. Read more
1.0.0 · sourcefn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
Performs copy-assignment from source
. Read more
sourceimpl Debug for AlphabetTokenizer
impl Debug for AlphabetTokenizer
sourceimpl<'de> Deserialize<'de> for AlphabetTokenizer
impl<'de> Deserialize<'de> for AlphabetTokenizer
sourcefn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error> where
__D: Deserializer<'de>,
fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error> where
__D: Deserializer<'de>,
Deserialize this value from the given Serde deserializer. Read more
sourceimpl Serialize for AlphabetTokenizer
impl Serialize for AlphabetTokenizer
sourceimpl Tokenizer for AlphabetTokenizer
impl Tokenizer for AlphabetTokenizer
sourcefn tokenize(&self, string: &str) -> Vec<String>ⓘNotable traits for Vec<u8, A>impl<A> Write for Vec<u8, A> where
A: Allocator,
fn tokenize(&self, string: &str) -> Vec<String>ⓘNotable traits for Vec<u8, A>impl<A> Write for Vec<u8, A> where
A: Allocator,
A: Allocator,
Tokenize a single string
sourcefn batch_tokenize(&self, strings: Vec<String>) -> Vec<Vec<String>>ⓘNotable traits for Vec<u8, A>impl<A> Write for Vec<u8, A> where
A: Allocator,
fn batch_tokenize(&self, strings: Vec<String>) -> Vec<Vec<String>>ⓘNotable traits for Vec<u8, A>impl<A> Write for Vec<u8, A> where
A: Allocator,
A: Allocator,
Tokenize a batch of strings
sourcefn untokenize(&self, tokens: Vec<String>) -> String
fn untokenize(&self, tokens: Vec<String>) -> String
Untokenize a single string
Auto Trait Implementations
impl RefUnwindSafe for AlphabetTokenizer
impl Send for AlphabetTokenizer
impl Sync for AlphabetTokenizer
impl Unpin for AlphabetTokenizer
impl UnwindSafe for AlphabetTokenizer
Blanket Implementations
sourceimpl<T> BorrowMut<T> for T where
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
const: unstable · sourcefn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
sourceimpl<T> Instrument for T
impl<T> Instrument for T
sourcefn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
sourcefn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
impl<T> Pointable for T
impl<T> Pointable for T
sourceimpl<T> ToOwned for T where
T: Clone,
impl<T> ToOwned for T where
T: Clone,
type Owned = T
type Owned = T
The resulting type after obtaining ownership.
sourcefn clone_into(&self, target: &mut T)
fn clone_into(&self, target: &mut T)
🔬 This is a nightly-only experimental API. (
toowned_clone_into
)Uses borrowed data to replace owned data, usually by cloning. Read more
impl<V, T> VZip<V> for T where
V: MultiLane<T>,
impl<V, T> VZip<V> for T where
V: MultiLane<T>,
fn vzip(self) -> V
sourceimpl<T> WithSubscriber for T
impl<T> WithSubscriber for T
sourcefn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> where
S: Into<Dispatch>,
fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> where
S: Into<Dispatch>,
Attaches the provided Subscriber
to this type, returning a
WithDispatch
wrapper. Read more
sourcefn with_current_subscriber(self) -> WithDispatch<Self>
fn with_current_subscriber(self) -> WithDispatch<Self>
Attaches the current default Subscriber
to this type, returning a
WithDispatch
wrapper. Read more