Enum TokenKind 

pub enum TokenKind {
    Word(Option<DictWordMetadata>),
    Punctuation(Punctuation),
    Decade,
    Number(Number),
    Space(usize),
    Newline(usize),
    EmailAddress,
    Url,
    Hostname,
    Unlintable,
    ParagraphBreak,
    Regexish,
    HeadingStart,
}

The parsed value of a Token. Has a variety of queries available. If there is a query missing, it may be easy to implement by just calling the delegate_to_metadata macro.

Variants

Word(Option<DictWordMetadata>)

None if the word does not exist in the dictionary.

Punctuation(Punctuation)

Decade

Number(Number)

Space(usize)

A sequence of “ “ spaces.

Newline(usize)

A sequence of “\n” newlines

EmailAddress

Url

Hostname

Unlintable

A special token used for things like inline code blocks that should be ignored by all linters.

ParagraphBreak

Regexish

HeadingStart

Implementations

impl TokenKind

pub const fn is_word(&self) -> bool

Returns true if self is of variant Word.

pub fn as_word(&self) -> Option<&Option<DictWordMetadata>>

Returns Some if self is a reference of variant Word, and None otherwise.

pub fn as_mut_word(&mut self) -> Option<&mut Option<DictWordMetadata>>

Returns Some if self is a mutable reference of variant Word, and None otherwise.

pub fn expect_word(self) -> Option<DictWordMetadata>

where Self: Debug,

Unwraps the value, yielding the content of Word.

Panics

Panics if the value is not Word. In debug builds the panic message includes the content of self.

pub fn word(self) -> Option<Option<DictWordMetadata>>

Returns Some if self is of variant Word, and None otherwise.

pub const fn is_punctuation(&self) -> bool

Returns true if self is of variant Punctuation.

pub fn as_punctuation(&self) -> Option<&Punctuation>

Returns Some if self is a reference of variant Punctuation, and None otherwise.

pub fn as_mut_punctuation(&mut self) -> Option<&mut Punctuation>

Returns Some if self is a mutable reference of variant Punctuation, and None otherwise.

pub fn expect_punctuation(self) -> Punctuation

where Self: Debug,

Unwraps the value, yielding the content of Punctuation.

Panics

Panics if the value is not Punctuation. In debug builds the panic message includes the content of self.

pub fn punctuation(self) -> Option<Punctuation>

Returns Some if self is of variant Punctuation, and None otherwise.

pub const fn is_decade(&self) -> bool

Returns true if self is of variant Decade.

pub const fn is_number(&self) -> bool

Returns true if self is of variant Number.

pub fn as_number(&self) -> Option<&Number>

Returns Some if self is a reference of variant Number, and None otherwise.

pub fn as_mut_number(&mut self) -> Option<&mut Number>

Returns Some if self is a mutable reference of variant Number, and None otherwise.

pub fn expect_number(self) -> Number

where Self: Debug,

Unwraps the value, yielding the content of Number.

Panics

Panics if the value is not Number. In debug builds the panic message includes the content of self.

pub fn number(self) -> Option<Number>

Returns Some if self is of variant Number, and None otherwise.

pub const fn is_space(&self) -> bool

Returns true if self is of variant Space.

pub fn as_space(&self) -> Option<&usize>

Returns Some if self is a reference of variant Space, and None otherwise.

pub fn as_mut_space(&mut self) -> Option<&mut usize>

Returns Some if self is a mutable reference of variant Space, and None otherwise.

pub fn expect_space(self) -> usize

where Self: Debug,

Unwraps the value, yielding the content of Space.

Panics

Panics if the value is not Space. In debug builds the panic message includes the content of self.

pub fn space(self) -> Option<usize>

Returns Some if self is of variant Space, and None otherwise.

pub const fn is_newline(&self) -> bool

Returns true if self is of variant Newline.

pub fn as_newline(&self) -> Option<&usize>

Returns Some if self is a reference of variant Newline, and None otherwise.

pub fn as_mut_newline(&mut self) -> Option<&mut usize>

Returns Some if self is a mutable reference of variant Newline, and None otherwise.

pub fn expect_newline(self) -> usize

where Self: Debug,

Unwraps the value, yielding the content of Newline.

Panics

Panics if the value is not Newline. In debug builds the panic message includes the content of self.

pub fn newline(self) -> Option<usize>

Returns Some if self is of variant Newline, and None otherwise.

pub const fn is_email_address(&self) -> bool

Returns true if self is of variant EmailAddress.

pub const fn is_url(&self) -> bool

Returns true if self is of variant Url.

pub const fn is_hostname(&self) -> bool

Returns true if self is of variant Hostname.

pub const fn is_unlintable(&self) -> bool

Returns true if self is of variant Unlintable.

pub const fn is_paragraph_break(&self) -> bool

Returns true if self is of variant ParagraphBreak.

pub const fn is_regexish(&self) -> bool

Returns true if self is of variant Regexish.

pub const fn is_heading_start(&self) -> bool

Returns true if self is of variant HeadingStart.

impl TokenKind

pub fn is_nominal(&self) -> bool

Delegates to DictWordMetadata::is_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_noun(&self) -> bool

Delegates to DictWordMetadata::is_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_proper_noun(&self) -> bool

Delegates to DictWordMetadata::is_proper_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_singular_nominal(&self) -> bool

Delegates to DictWordMetadata::is_singular_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_plural_nominal(&self) -> bool

Delegates to DictWordMetadata::is_plural_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_possessive_nominal(&self) -> bool

Delegates to DictWordMetadata::is_possessive_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_non_plural_nominal(&self) -> bool

Delegates to DictWordMetadata::is_non_plural_nominal when this token is a word.

Returns false if the token is not a word.

pub fn is_singular_noun(&self) -> bool

Delegates to DictWordMetadata::is_singular_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_singular_noun_only(&self) -> bool

Delegates to DictWordMetadata::is_singular_noun_only when this token is a word.

Returns false if the token is not a word.

pub fn is_plural_noun(&self) -> bool

Delegates to DictWordMetadata::is_plural_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_plural_noun_only(&self) -> bool

Delegates to DictWordMetadata::is_plural_noun_only when this token is a word.

Returns false if the token is not a word.

pub fn is_non_plural_noun(&self) -> bool

Delegates to DictWordMetadata::is_non_plural_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_non_possessive_noun(&self) -> bool

Delegates to DictWordMetadata::is_non_possessive_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_countable_noun(&self) -> bool

Delegates to DictWordMetadata::is_countable_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_non_countable_noun(&self) -> bool

Delegates to DictWordMetadata::is_non_countable_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_mass_noun(&self) -> bool

Delegates to DictWordMetadata::is_mass_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_mass_noun_only(&self) -> bool

Delegates to DictWordMetadata::is_mass_noun_only when this token is a word.

Returns false if the token is not a word.

pub fn is_non_mass_noun(&self) -> bool

Delegates to DictWordMetadata::is_non_mass_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_singular_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_singular_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_plural_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_plural_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_non_plural_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_non_plural_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_reflexive_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_reflexive_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_personal_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_personal_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_first_person_singular_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_first_person_singular_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_first_person_plural_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_first_person_plural_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_second_person_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_second_person_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_third_person_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_third_person_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_third_person_singular_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_third_person_singular_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_third_person_plural_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_third_person_plural_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_subject_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_subject_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_object_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_object_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_possessive_noun(&self) -> bool

Delegates to DictWordMetadata::is_possessive_noun when this token is a word.

Returns false if the token is not a word.

pub fn is_possessive_pronoun(&self) -> bool

Delegates to DictWordMetadata::is_possessive_pronoun when this token is a word.

Returns false if the token is not a word.

pub fn is_verb(&self) -> bool

Delegates to DictWordMetadata::is_verb when this token is a word.

Returns false if the token is not a word.

pub fn is_auxiliary_verb(&self) -> bool

Delegates to DictWordMetadata::is_auxiliary_verb when this token is a word.

Returns false if the token is not a word.

pub fn is_linking_verb(&self) -> bool

Delegates to DictWordMetadata::is_linking_verb when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_lemma(&self) -> bool

Delegates to DictWordMetadata::is_verb_lemma when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_past_form(&self) -> bool

Delegates to DictWordMetadata::is_verb_past_form when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_preterite_and_participle_form(&self) -> bool

Delegates to DictWordMetadata::is_verb_preterite_and_participle_form when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_simple_past_form(&self) -> bool

Delegates to DictWordMetadata::is_verb_simple_past_form when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_past_participle_form(&self) -> bool

Delegates to DictWordMetadata::is_verb_past_participle_form when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_simple_past_only(&self) -> bool

Delegates to DictWordMetadata::is_verb_simple_past_only when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_past_participle_only(&self) -> bool

Delegates to DictWordMetadata::is_verb_past_participle_only when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_progressive_form(&self) -> bool

Delegates to DictWordMetadata::is_verb_progressive_form when this token is a word.

Returns false if the token is not a word.

pub fn is_verb_third_person_singular_present_form(&self) -> bool

Delegates to DictWordMetadata::is_verb_third_person_singular_present_form when this token is a word.

Returns false if the token is not a word.

pub fn is_adjective(&self) -> bool

Delegates to DictWordMetadata::is_adjective when this token is a word.

Returns false if the token is not a word.

pub fn is_comparative_adjective(&self) -> bool

Delegates to DictWordMetadata::is_comparative_adjective when this token is a word.

Returns false if the token is not a word.

pub fn is_superlative_adjective(&self) -> bool

Delegates to DictWordMetadata::is_superlative_adjective when this token is a word.

Returns false if the token is not a word.

pub fn is_positive_adjective(&self) -> bool

Delegates to DictWordMetadata::is_positive_adjective when this token is a word.

Returns false if the token is not a word.

pub fn is_adverb(&self) -> bool

Delegates to DictWordMetadata::is_adverb when this token is a word.

Returns false if the token is not a word.

pub fn is_manner_adverb(&self) -> bool

Delegates to DictWordMetadata::is_manner_adverb when this token is a word.

Returns false if the token is not a word.

pub fn is_frequency_adverb(&self) -> bool

Delegates to DictWordMetadata::is_frequency_adverb when this token is a word.

Returns false if the token is not a word.

pub fn is_degree_adverb(&self) -> bool

Delegates to DictWordMetadata::is_degree_adverb when this token is a word.

Returns false if the token is not a word.

pub fn is_determiner(&self) -> bool

Delegates to DictWordMetadata::is_determiner when this token is a word.

Returns false if the token is not a word.

pub fn is_demonstrative_determiner(&self) -> bool

Delegates to DictWordMetadata::is_demonstrative_determiner when this token is a word.

Returns false if the token is not a word.

pub fn is_possessive_determiner(&self) -> bool

Delegates to DictWordMetadata::is_possessive_determiner when this token is a word.

Returns false if the token is not a word.

pub fn is_quantifier(&self) -> bool

Delegates to DictWordMetadata::is_quantifier when this token is a word.

Returns false if the token is not a word.

pub fn is_non_quantifier_determiner(&self) -> bool

Delegates to DictWordMetadata::is_non_quantifier_determiner when this token is a word.

Returns false if the token is not a word.

pub fn is_non_demonstrative_determiner(&self) -> bool

Delegates to DictWordMetadata::is_non_demonstrative_determiner when this token is a word.

Returns false if the token is not a word.

pub fn is_conjunction(&self) -> bool

Delegates to DictWordMetadata::is_conjunction when this token is a word.

Returns false if the token is not a word.

pub fn is_swear(&self) -> bool

Delegates to DictWordMetadata::is_swear when this token is a word.

Returns false if the token is not a word.

pub fn is_abbreviation(&self) -> bool

Delegates to DictWordMetadata::is_abbreviation when this token is a word.

Returns false if the token is not a word.

pub fn is_likely_homograph(&self) -> bool

Delegates to DictWordMetadata::is_likely_homograph when this token is a word.

Returns false if the token is not a word.

pub fn is_lowercase(&self) -> bool

Delegates to DictWordMetadata::is_lowercase when this token is a word.

Returns false if the token is not a word.

pub fn is_titlecase(&self) -> bool

Delegates to DictWordMetadata::is_titlecase when this token is a word.

Returns false if the token is not a word.

pub fn is_allcaps(&self) -> bool

Delegates to DictWordMetadata::is_allcaps when this token is a word.

Returns false if the token is not a word.

pub fn is_lower_camel(&self) -> bool

Delegates to DictWordMetadata::is_lower_camel when this token is a word.

Returns false if the token is not a word.

pub fn is_upper_camel(&self) -> bool

Delegates to DictWordMetadata::is_upper_camel when this token is a word.

Returns false if the token is not a word.

pub fn is_apostrophized(&self) -> bool

Delegates to DictWordMetadata::is_apostrophized when this token is a word.

Returns false if the token is not a word.

pub fn is_roman_numerals(&self) -> bool

Delegates to DictWordMetadata::is_roman_numerals when this token is a word.

Returns false if the token is not a word.

pub fn get_pronoun_person(&self) -> Option<Person>

pub fn is_preposition(&self) -> bool

pub fn is_common_word(&self) -> bool

pub fn is_np_member(&self) -> bool

Checks whether the token is a member of a nominal phrase.

pub fn is_oov(&self) -> bool

Checks whether a word token is out-of-vocabulary (not found in the dictionary).

Returns true if the token is a word that was not found in the dictionary, false if the token is a word found in the dictionary or is not a word token.

pub fn is_cardinal_number(&self) -> bool

pub fn is_ordinal_number(&self) -> bool

pub fn is_open_square(&self) -> bool

pub fn is_close_square(&self) -> bool

pub fn is_less_than(&self) -> bool

pub fn is_greater_than(&self) -> bool

pub fn is_open_round(&self) -> bool

pub fn is_close_round(&self) -> bool

pub fn is_pipe(&self) -> bool

pub fn is_currency(&self) -> bool

pub fn is_ellipsis(&self) -> bool

pub fn is_hyphen(&self) -> bool

pub fn is_plus(&self) -> bool

pub fn is_quote(&self) -> bool

pub fn is_apostrophe(&self) -> bool

pub fn is_period(&self) -> bool

pub fn is_at(&self) -> bool

pub fn is_comma(&self) -> bool

pub fn is_semicolon(&self) -> bool

pub fn is_acute(&self) -> bool

pub fn is_ampersand(&self) -> bool

pub fn is_backslash(&self) -> bool

pub fn is_slash(&self) -> bool

pub fn is_percent(&self) -> bool

pub fn is_degree(&self) -> bool

pub fn is_open_single(&self) -> bool

pub fn is_single_prime(&self) -> bool

pub fn is_double_prime(&self) -> bool

pub fn is_backtick(&self) -> bool

pub fn is_word_like(&self) -> bool

Checks whether a token is word-like–meaning it is more complex than punctuation and can hold semantic meaning in the way a word does.

pub fn is_sentence_terminator(&self) -> bool

pub fn is_case_separator(&self) -> bool

Used by crate::parsers::CollapseIdentifiers TODO: Separate this into two functions and add OR functionality to pattern matching

pub fn is_whitespace(&self) -> bool

Checks whether the token is whitespace.

pub fn is_upos(&self, upos: UPOS) -> bool

pub fn matches_variant_of(&self, other: &Self) -> bool

Checks that self is the same enum variant as other, regardless of whether the inner metadata is also equal.

pub fn with_default_data(&self) -> Self

Produces a copy of self with any inner data replaced with its default value. Useful for making comparisons on just the variant of the enum.

pub fn blank_word() -> Self

Construct a TokenKind::Word with no metadata.

pub fn as_mut_quote(&mut self) -> Option<&mut Quote>

pub fn as_quote(&self) -> Option<&Quote>

Trait Implementations

impl Clone for TokenKind

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenKind

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TokenKind

fn default() -> Self

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for TokenKind

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Eq for TokenKind

impl Hash for TokenKind

fn hash<__H: Hasher>(&self, state: &mut __H)

Feeds this value into the given Hasher.

fn hash_slice<H>(data: &[Self], state: &mut H)

where H: Hasher, Self: Sized,

Feeds a slice of this type into the given Hasher.

impl PartialEq for TokenKind

fn eq(&self, other: &Self) -> bool

Equality operator ==.

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=.

impl PartialOrd for TokenKind

fn partial_cmp(&self, other: &Self) -> Option<Ordering>

This method returns an ordering between self and other values if one exists.

fn lt(&self, other: &Rhs) -> bool

Tests less than (for self and other) and is used by the < operator.

fn le(&self, other: &Rhs) -> bool

Tests less than or equal to (for self and other) and is used by the <= operator.

fn gt(&self, other: &Rhs) -> bool

Tests greater than (for self and other) and is used by the > operator.

fn ge(&self, other: &Rhs) -> bool

Tests greater than or equal to (for self and other) and is used by the >= operator.

impl Serialize for TokenKind

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl StructuralPartialEq for TokenKind