Enum text_tokenizer::Token

pub enum Token<T> {
    Word(String),
    StrangeWord(String),
    Hashtag(String),
    Mention(String),
    Punctuation(String),
    Numerical(Numerical),
    Number(Number),
    Emoji(&'static str),
    Unicode(String),
    Separator(Separator),
    UnicodeFormatter(Formatter),
    UnicodeModifier(char),
    Url(String),
    BBCode {
        left: Vec<T>,
        right: Vec<T>,
    },
}

Variants

Word(String)

StrangeWord(String)

Hashtag(String)

Mention(String)

Punctuation(String)

Numerical(Numerical)

Number(Number)

Emoji(&'static str)

Unicode(String)

Separator(Separator)

UnicodeFormatter(Formatter)

UnicodeModifier(char)

Url(String)

BBCode

Fields

left: Vec<T>

right: Vec<T>

Trait Implementations

impl<T: Clone> Clone for Token<T>

fn clone(&self) -> Token<T>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<T: Debug> Debug for Token<T>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<T: PartialEq> PartialEq for Token<T>

fn eq(&self, other: &Token<T>) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl<T: PartialOrd> PartialOrd for Token<T>

fn partial_cmp(&self, other: &Token<T>) -> Option<Ordering>

This method returns an ordering between self and other values if one exists.

fn lt(&self, other: &Rhs) -> bool

This method tests less than (for self and other) and is used by the < operator.

fn le(&self, other: &Rhs) -> bool

This method tests less than or equal to (for self and other) and is used by the <= operator.

fn gt(&self, other: &Rhs) -> bool

This method tests greater than (for self and other) and is used by the > operator.

fn ge(&self, other: &Rhs) -> bool

This method tests greater than or equal to (for self and other) and is used by the >= operator.

impl<T> StructuralPartialEq for Token<T>