Enum TokenKind 

pub enum TokenKind {
    Word(Option<WordMetadata>),
    Punctuation(Punctuation),
    Decade,
    Number(Number),
    Space(usize),
    Newline(usize),
    EmailAddress,
    Url,
    Hostname,
    Unlintable,
    ParagraphBreak,
    Regexish,
}

The parsed value of a Token. Has a variety of queries available. If there is a query missing, it may be easy to implement by just calling the delegate_to_metadata macro.

Variants

Word(Option<WordMetadata>)

None if the word does not exist in the dictionary.

Punctuation(Punctuation)

Decade

Number(Number)

Space(usize)

A sequence of “ “ spaces.

Newline(usize)

A sequence of “\n” newlines

EmailAddress

Url

Hostname

Unlintable

A special token used for things like inline code blocks that should be ignored by all linters.

ParagraphBreak

Regexish

Implementations

impl TokenKind

pub const fn is_word(&self) -> bool

Returns true if self is of variant Word.

pub fn as_word(&self) -> Option<&Option<WordMetadata>>

Returns Some if self is a reference of variant Word, and None otherwise.

pub fn as_mut_word(&mut self) -> Option<&mut Option<WordMetadata>>

Returns Some if self is a mutable reference of variant Word, and None otherwise.

pub fn expect_word(self) -> Option<WordMetadata>

where Self: Debug,

Unwraps the value, yielding the content of Word.

Panics

Panics if the value is not Word, with a panic message including the content of self.

pub fn word(self) -> Option<Option<WordMetadata>>

Returns Some if self is of variant Word, and None otherwise.

pub const fn is_punctuation(&self) -> bool

Returns true if self is of variant Punctuation.

pub fn as_punctuation(&self) -> Option<&Punctuation>

Returns Some if self is a reference of variant Punctuation, and None otherwise.

pub fn as_mut_punctuation(&mut self) -> Option<&mut Punctuation>

Returns Some if self is a mutable reference of variant Punctuation, and None otherwise.

pub fn expect_punctuation(self) -> Punctuation

where Self: Debug,

Unwraps the value, yielding the content of Punctuation.

Panics

Panics if the value is not Punctuation, with a panic message including the content of self.

pub fn punctuation(self) -> Option<Punctuation>

Returns Some if self is of variant Punctuation, and None otherwise.

pub const fn is_decade(&self) -> bool

Returns true if self is of variant Decade.

pub const fn is_number(&self) -> bool

Returns true if self is of variant Number.

pub fn as_number(&self) -> Option<&Number>

Returns Some if self is a reference of variant Number, and None otherwise.

pub fn as_mut_number(&mut self) -> Option<&mut Number>

Returns Some if self is a mutable reference of variant Number, and None otherwise.

pub fn expect_number(self) -> Number

where Self: Debug,

Unwraps the value, yielding the content of Number.

Panics

Panics if the value is not Number, with a panic message including the content of self.

pub fn number(self) -> Option<Number>

Returns Some if self is of variant Number, and None otherwise.

pub const fn is_space(&self) -> bool

Returns true if self is of variant Space.

pub fn as_space(&self) -> Option<&usize>

Returns Some if self is a reference of variant Space, and None otherwise.

pub fn as_mut_space(&mut self) -> Option<&mut usize>

Returns Some if self is a mutable reference of variant Space, and None otherwise.

pub fn expect_space(self) -> usize

where Self: Debug,

Unwraps the value, yielding the content of Space.

Panics

Panics if the value is not Space, with a panic message including the content of self.

pub fn space(self) -> Option<usize>

Returns Some if self is of variant Space, and None otherwise.

pub const fn is_newline(&self) -> bool

Returns true if self is of variant Newline.

pub fn as_newline(&self) -> Option<&usize>

Returns Some if self is a reference of variant Newline, and None otherwise.

pub fn as_mut_newline(&mut self) -> Option<&mut usize>

Returns Some if self is a mutable reference of variant Newline, and None otherwise.

pub fn expect_newline(self) -> usize

where Self: Debug,

Unwraps the value, yielding the content of Newline.

Panics

Panics if the value is not Newline, with a panic message including the content of self.

pub fn newline(self) -> Option<usize>

Returns Some if self is of variant Newline, and None otherwise.

pub const fn is_email_address(&self) -> bool

Returns true if self is of variant EmailAddress.

pub const fn is_url(&self) -> bool

Returns true if self is of variant Url.

pub const fn is_hostname(&self) -> bool

Returns true if self is of variant Hostname.

pub const fn is_unlintable(&self) -> bool

Returns true if self is of variant Unlintable.

pub const fn is_paragraph_break(&self) -> bool

Returns true if self is of variant ParagraphBreak.

pub const fn is_regexish(&self) -> bool

Returns true if self is of variant Regexish.

impl TokenKind

pub fn is_nominal(&self) -> bool

Delegates to WordMetadata::is_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_noun(&self) -> bool

Delegates to WordMetadata::is_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_pronoun(&self) -> bool

Delegates to WordMetadata::is_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_proper_noun(&self) -> bool

Delegates to WordMetadata::is_proper_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_singular_nominal(&self) -> bool

Delegates to WordMetadata::is_singular_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_plural_nominal(&self) -> bool

Delegates to WordMetadata::is_plural_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_possessive_nominal(&self) -> bool

Delegates to WordMetadata::is_possessive_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_non_plural_nominal(&self) -> bool

Delegates to WordMetadata::is_non_plural_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_singular_noun(&self) -> bool

Delegates to WordMetadata::is_singular_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_plural_noun(&self) -> bool

Delegates to WordMetadata::is_plural_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_non_plural_noun(&self) -> bool

Delegates to WordMetadata::is_non_plural_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_countable_noun(&self) -> bool

Delegates to WordMetadata::is_countable_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_non_countable_noun(&self) -> bool

Delegates to WordMetadata::is_non_countable_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_mass_noun(&self) -> bool

Delegates to WordMetadata::is_mass_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_mass_noun_only(&self) -> bool

Delegates to WordMetadata::is_mass_noun_only when this token is a word.

Returns false if the token is not a word.

pub fn is_non_mass_noun(&self) -> bool

Delegates to WordMetadata::is_non_mass_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_singular_pronoun(&self) -> bool

Delegates to WordMetadata::is_singular_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_plural_pronoun(&self) -> bool

Delegates to WordMetadata::is_plural_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_non_plural_pronoun(&self) -> bool

Delegates to WordMetadata::is_non_plural_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_reflexive_pronoun(&self) -> bool

Delegates to WordMetadata::is_reflexive_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_first_person_singular_pronoun(&self) -> bool

Delegates to WordMetadata::is_first_person_singular_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_first_person_plural_pronoun(&self) -> bool

Delegates to WordMetadata::is_first_person_plural_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_second_person_pronoun(&self) -> bool

Delegates to WordMetadata::is_second_person_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_third_person_pronoun(&self) -> bool

Delegates to WordMetadata::is_third_person_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_third_person_singular_pronoun(&self) -> bool

Delegates to WordMetadata::is_third_person_singular_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_third_person_plural_pronoun(&self) -> bool

Delegates to WordMetadata::is_third_person_plural_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_object_pronoun(&self) -> bool

Delegates to WordMetadata::is_object_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_possessive_noun(&self) -> bool

Delegates to WordMetadata::is_possessive_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_possessive_pronoun(&self) -> bool

Delegates to WordMetadata::is_possessive_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_verb(&self) -> bool

Delegates to WordMetadata::is_verb when this token is a word.

Returns false if the token is not a word.

pub fn is_auxiliary_verb(&self) -> bool

Delegates to WordMetadata::is_auxiliary_verb when this token is a word.

Returns false if the token is not a word.

pub fn is_linking_verb(&self) -> bool

Delegates to WordMetadata::is_linking_verb when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_lemma(&self) -> bool

Delegates to WordMetadata::is_verb_lemma when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_past_form(&self) -> bool

Delegates to WordMetadata::is_verb_past_form when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_progressive_form(&self) -> bool

Delegates to WordMetadata::is_verb_progressive_form when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_third_person_singular_present_form(&self) -> bool

Delegates to WordMetadata::is_verb_third_person_singular_present_form when this token is a word.

Returns false if the token is not a word.

pub fn is_adjective(&self) -> bool

Delegates to WordMetadata::is_adjective when this token is a word.

Returns false if the token is not a word.

pub fn is_comparative_adjective(&self) -> bool

Delegates to WordMetadata::is_comparative_adjective when this token is a word.

Returns false if the token is not a word.

pub fn is_superlative_adjective(&self) -> bool

Delegates to WordMetadata::is_superlative_adjective when this token is a word.

Returns false if the token is not a word.

pub fn is_positive_adjective(&self) -> bool

Delegates to WordMetadata::is_positive_adjective when this token is a word.

Returns false if the token is not a word.

pub fn is_adverb(&self) -> bool

Delegates to WordMetadata::is_adverb when this token is a word.

Returns false if the token is not a word.

pub fn is_determiner(&self) -> bool

Delegates to WordMetadata::is_determiner when this token is a word.

Returns false if the token is not a word.

pub fn is_demonstrative_determiner(&self) -> bool

Delegates to WordMetadata::is_demonstrative_determiner when this token is a word.

Returns false if the token is not a word.

pub fn is_possessive_determiner(&self) -> bool

Delegates to WordMetadata::is_possessive_determiner when this token is a word.

Returns false if the token is not a word.

pub fn is_quantifier(&self) -> bool

Delegates to WordMetadata::is_quantifier when this token is a word.

Returns false if the token is not a word.

pub fn is_non_quantifier_determiner(&self) -> bool

Delegates to WordMetadata::is_non_quantifier_determiner when this token is a word.

Returns false if the token is not a word.

pub fn is_conjunction(&self) -> bool

Delegates to WordMetadata::is_conjunction when this token is a word.

Returns false if the token is not a word.

pub fn is_swear(&self) -> bool

Delegates to WordMetadata::is_swear when this token is a word.

Returns false if the token is not a word.

pub fn is_likely_homograph(&self) -> bool

Delegates to WordMetadata::is_likely_homograph when this token is a word.

Returns false if the token is not a word.

pub fn is_lowercase(&self) -> bool

Delegates to WordMetadata::is_lowercase when this token is a word.

Returns false if the token is not a word.

pub fn is_titlecase(&self) -> bool

Delegates to WordMetadata::is_titlecase when this token is a word.

Returns false if the token is not a word.

pub fn is_allcaps(&self) -> bool

Delegates to WordMetadata::is_allcaps when this token is a word.

Returns false if the token is not a word.

pub fn is_lower_camel(&self) -> bool

Delegates to WordMetadata::is_lower_camel when this token is a word.

Returns false if the token is not a word.

pub fn is_upper_camel(&self) -> bool

Delegates to WordMetadata::is_upper_camel when this token is a word.

Returns false if the token is not a word.

pub fn is_preposition(&self) -> bool

pub fn is_common_word(&self) -> bool

pub fn is_oov(&self) -> bool

Checks whether a word token is out-of-vocabulary (not found in the dictionary).

Returns true if the token is a word that was not found in the dictionary, false if the token is a word found in the dictionary or is not a word token.

pub fn is_open_square(&self) -> bool

pub fn is_close_square(&self) -> bool

pub fn is_open_round(&self) -> bool

pub fn is_close_round(&self) -> bool

pub fn is_pipe(&self) -> bool

pub fn is_currency(&self) -> bool

pub fn is_ellipsis(&self) -> bool

pub fn is_hyphen(&self) -> bool

pub fn is_quote(&self) -> bool

pub fn is_apostrophe(&self) -> bool

pub fn is_period(&self) -> bool

pub fn is_at(&self) -> bool

pub fn is_comma(&self) -> bool

pub fn is_semicolon(&self) -> bool

pub fn is_ampersand(&self) -> bool

pub fn is_word_like(&self) -> bool

Checks whether a token is word-like–meaning it is more complex than punctuation and can hold semantic meaning in the way a word does.

pub fn is_case_separator(&self) -> bool

Used by crate::parsers::CollapseIdentifiers TODO: Separate this into two functions and add OR functionality to pattern matching

pub fn is_whitespace(&self) -> bool

Checks whether the token is whitespace.

pub fn is_upos(&self, upos: UPOS) -> bool

pub fn matches_variant_of(&self, other: &Self) -> bool

Checks that self is the same enum variant as other, regardless of whether the inner metadata is also equal.

pub fn with_default_data(&self) -> Self

Produces a copy of self with any inner data replaced with its default value. Useful for making comparisons on just the variant of the enum.

pub fn blank_word() -> Self

Construct a TokenKind::Word with no metadata.

pub fn as_mut_quote(&mut self) -> Option<&mut Quote>

pub fn as_quote(&self) -> Option<&Quote>

Trait Implementations

impl Clone for TokenKind

fn clone(&self) -> TokenKind

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenKind

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TokenKind

fn default() -> TokenKind

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for TokenKind

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Hash for TokenKind

fn hash<__H: Hasher>(&self, state: &mut __H)

Feeds this value into the given Hasher.

fn hash_slice<H>(data: &[Self], state: &mut H)

where H: Hasher, Self: Sized,

Feeds a slice of this type into the given Hasher.

impl PartialEq for TokenKind

fn eq(&self, other: &TokenKind) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl PartialOrd for TokenKind

fn partial_cmp(&self, other: &TokenKind) -> Option<Ordering>

This method returns an ordering between self and other values if one exists.

fn lt(&self, other: &Rhs) -> bool

Tests less than (for self and other) and is used by the < operator.

fn le(&self, other: &Rhs) -> bool

Tests less than or equal to (for self and other) and is used by the <= operator.

fn gt(&self, other: &Rhs) -> bool

Tests greater than (for self and other) and is used by the > operator.

fn ge(&self, other: &Rhs) -> bool

Tests greater than or equal to (for self and other) and is used by the >= operator.

impl Serialize for TokenKind

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Eq for TokenKind

impl StructuralPartialEq for TokenKind