Struct AlphabetTokenizer

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more

Source §

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more

Source §

impl<T> Pointable for T

Source §

const ALIGN: usize

The alignment of pointer.

Source §

type Init = T

The type for initializers.

Source §

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more

Source §

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more

Source §

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more

Source §

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more

Source §

impl<T> Same for T

Source §

type Output = T

Should always be Self

Source §

impl<T> ToOwned for T
where T: Clone,

Source §

type Owned = T

The resulting type after obtaining ownership.

Source §

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more

Source §

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more

Source §

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source §

type Error = Infallible

The type returned in the event of a conversion error.

Source §

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

Performs the conversion.

Source §

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source §

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.

Source §

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.

Source §

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source §

fn vzip(self) -> V

Source §

impl<T> WithSubscriber for T

Source §

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a WithDispatch wrapper. Read more

Source §

fn with_current_subscriber(self) -> WithDispatch<Self>

Attaches the current default Subscriber to this type, returning a WithDispatch wrapper. Read more

Source §

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,

Source §

AlphabetTokenizer

Struct AlphabetTokenizer Copy item path

Trait Implementations§

impl Clone for AlphabetTokenizer

fn clone(&self) -> AlphabetTokenizer

fn clone_from(&mut self, source: &Self)

impl Debug for AlphabetTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

impl<'de> Deserialize<'de> for AlphabetTokenizer

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where __D: Deserializer<'de>,

impl Node<&str> for AlphabetTokenizer

type Output = Vec<String>

fn process(&mut self, input: &str) -> Self::Output

fn reset(&mut self)

fn data_remaining(&self, before: usize) -> usize

impl Node<String> for AlphabetTokenizer

type Output = Vec<String>

fn process(&mut self, input: String) -> Self::Output

fn reset(&mut self)

fn data_remaining(&self, before: usize) -> usize

impl Node<Vec<String>> for AlphabetTokenizer

type Output = Vec<Vec<String>>

fn process(&mut self, input: Vec<String>) -> Self::Output

fn reset(&mut self)

fn data_remaining(&self, before: usize) -> usize

impl Serialize for AlphabetTokenizer

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where __S: Serializer,

impl Tokenizer for AlphabetTokenizer

fn load() -> Self

fn tokenize(&self, string: &str) -> Vec<String>

fn batch_tokenize(&self, strings: Vec<String>) -> Vec<Vec<String>>

fn untokenize(&self, tokens: Vec<String>) -> String

fn batch_untokenize(&self, tokens: Vec<Vec<String>>) -> Vec<String>

Auto Trait Implementations§

impl Freeze for AlphabetTokenizer

impl RefUnwindSafe for AlphabetTokenizer

impl Send for AlphabetTokenizer

impl Sync for AlphabetTokenizer

impl Unpin for AlphabetTokenizer

impl UnwindSafe for AlphabetTokenizer

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> CloneToUninit for Twhere T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

impl<Input, Output, E> ExtendNode<Input, Output, E> for Ewhere E: Node<Input, Output = Output>,

fn chain<O, N>(self, node: N) -> (E, N)where N: Node<Output, Output = O>,

impl<Input, Output, E> ExtendNodeMap<Input, Output, E> for Ewhere E: Node<Input, Output = Vec<Output>>,

fn map<O, N>(self, node: N) -> (E, Map<Output, N>)where N: Node<Output, Output = O>,

fn filter_map<O, N>(self, node: N) -> (E, FilterMap<Output, N>)where N: Node<Output, Output = Option<O>>,

fn filter<F>(self, function: F) -> (E, Filter<Output, F>)where F: FnMut(&Output) -> bool,

impl<Input, Out1, Out2, E> ExtendNodePair<Input, Out1, Out2, E> for Ewhere E: Node<Input, Output = (Out1, Out2)>,

fn pair<F1, F2, N1, N2>( self, node1: N1, node2: N2, ) -> (E, Pair<Out1, Out2, N1, N2>)where N1: Node<Out1, Output = F1>, N2: Node<Out2, Output = F2>,

impl<Input, Output, E> ExtendNodeSplit<Input, Output, E> for Ewhere Output: Clone, E: Node<Input, Output = Output>,

fn split<O1, O2, E1, E2>( self, node1: E1, node2: E2, ) -> (E, Duplicator<Output>, Pair<Output, Output, E1, E2>)where E1: Node<Output, Output = O1>, E2: Node<Output, Output = O2>,

impl<T> From<T> for T

fn from(t: T) -> T

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

fn in_current_span(self) -> Instrumented<Self>

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self>

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>where F: FnOnce(&Self) -> bool,

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> Same for T

type Output = T

impl<T> ToOwned for Twhere T: Clone,

type Owned = T

Struct AlphabetTokenizer

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where __D: Deserializer<'de>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: Serializer,

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> CloneToUninit for T
where T: Clone,

impl<Input, Output, E> ExtendNode<Input, Output, E> for E
where E: Node<Input, Output = Output>,

fn chain<O, N>(self, node: N) -> (E, N)
where N: Node<Output, Output = O>,

impl<Input, Output, E> ExtendNodeMap<Input, Output, E> for E
where E: Node<Input, Output = Vec<Output>>,

fn map<O, N>(self, node: N) -> (E, Map<Output, N>)
where N: Node<Output, Output = O>,

fn filter_map<O, N>(self, node: N) -> (E, FilterMap<Output, N>)
where N: Node<Output, Output = Option<O>>,

fn filter<F>(self, function: F) -> (E, Filter<Output, F>)
where F: FnMut(&Output) -> bool,

impl<Input, Out1, Out2, E> ExtendNodePair<Input, Out1, Out2, E> for E
where E: Node<Input, Output = (Out1, Out2)>,

fn pair<F1, F2, N1, N2>( self, node1: N1, node2: N2, ) -> (E, Pair<Out1, Out2, N1, N2>)
where N1: Node<Out1, Output = F1>, N2: Node<Out2, Output = F2>,

impl<Input, Output, E> ExtendNodeSplit<Input, Output, E> for E
where Output: Clone, E: Node<Input, Output = Output>,

fn split<O1, O2, E1, E2>( self, node1: E1, node2: E2, ) -> (E, Duplicator<Output>, Pair<Output, Output, E1, E2>)
where E1: Node<Output, Output = O1>, E2: Node<Output, Output = O2>,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

impl<T> ToOwned for T
where T: Clone,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,

impl<T> ErasedDestructor for T
where T: 'static,