Trait TokenStringExt 

pub trait TokenStringExt: Sealed {
    // Required methods
    fn tokens(&self) -> &[Token];
    fn tokens_mut(&mut self) -> &mut [Token];

    // Provided methods
    fn first_adjective(&self) -> Option<&Token> { ... }
    fn last_adjective(&self) -> Option<&Token> { ... }
    fn last_adjective_index(&self) -> Option<usize> { ... }
    fn iter_adjective_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_adjectives(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_apostrophe(&self) -> Option<&Token> { ... }
    fn last_apostrophe(&self) -> Option<&Token> { ... }
    fn last_apostrophe_index(&self) -> Option<usize> { ... }
    fn iter_apostrophe_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_apostrophes(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_at(&self) -> Option<&Token> { ... }
    fn last_at(&self) -> Option<&Token> { ... }
    fn last_at_index(&self) -> Option<usize> { ... }
    fn iter_at_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_ats(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_comma(&self) -> Option<&Token> { ... }
    fn last_comma(&self) -> Option<&Token> { ... }
    fn last_comma_index(&self) -> Option<usize> { ... }
    fn iter_comma_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_commas(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_conjunction(&self) -> Option<&Token> { ... }
    fn last_conjunction(&self) -> Option<&Token> { ... }
    fn last_conjunction_index(&self) -> Option<usize> { ... }
    fn iter_conjunction_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_conjunctions(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_chunk_terminator(&self) -> Option<&Token> { ... }
    fn last_chunk_terminator(&self) -> Option<&Token> { ... }
    fn last_chunk_terminator_index(&self) -> Option<usize> { ... }
    fn iter_chunk_terminator_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_chunk_terminators(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_currency(&self) -> Option<&Token> { ... }
    fn last_currency(&self) -> Option<&Token> { ... }
    fn last_currency_index(&self) -> Option<usize> { ... }
    fn iter_currency_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_currencys(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_ellipsis(&self) -> Option<&Token> { ... }
    fn last_ellipsis(&self) -> Option<&Token> { ... }
    fn last_ellipsis_index(&self) -> Option<usize> { ... }
    fn iter_ellipsis_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_ellipsiss(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_hostname(&self) -> Option<&Token> { ... }
    fn last_hostname(&self) -> Option<&Token> { ... }
    fn last_hostname_index(&self) -> Option<usize> { ... }
    fn iter_hostname_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_hostnames(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_likely_homograph(&self) -> Option<&Token> { ... }
    fn last_likely_homograph(&self) -> Option<&Token> { ... }
    fn last_likely_homograph_index(&self) -> Option<usize> { ... }
    fn iter_likely_homograph_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_likely_homographs(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_number(&self) -> Option<&Token> { ... }
    fn last_number(&self) -> Option<&Token> { ... }
    fn last_number_index(&self) -> Option<usize> { ... }
    fn iter_number_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_numbers(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_noun(&self) -> Option<&Token> { ... }
    fn last_noun(&self) -> Option<&Token> { ... }
    fn last_noun_index(&self) -> Option<usize> { ... }
    fn iter_noun_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_nouns(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_paragraph_break(&self) -> Option<&Token> { ... }
    fn last_paragraph_break(&self) -> Option<&Token> { ... }
    fn last_paragraph_break_index(&self) -> Option<usize> { ... }
    fn iter_paragraph_break_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_paragraph_breaks(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_pipe(&self) -> Option<&Token> { ... }
    fn last_pipe(&self) -> Option<&Token> { ... }
    fn last_pipe_index(&self) -> Option<usize> { ... }
    fn iter_pipe_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_pipes(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_preposition(&self) -> Option<&Token> { ... }
    fn last_preposition(&self) -> Option<&Token> { ... }
    fn last_preposition_index(&self) -> Option<usize> { ... }
    fn iter_preposition_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_prepositions(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_punctuation(&self) -> Option<&Token> { ... }
    fn last_punctuation(&self) -> Option<&Token> { ... }
    fn last_punctuation_index(&self) -> Option<usize> { ... }
    fn iter_punctuation_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_punctuations(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_quote(&self) -> Option<&Token> { ... }
    fn last_quote(&self) -> Option<&Token> { ... }
    fn last_quote_index(&self) -> Option<usize> { ... }
    fn iter_quote_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_quotes(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_sentence_terminator(&self) -> Option<&Token> { ... }
    fn last_sentence_terminator(&self) -> Option<&Token> { ... }
    fn last_sentence_terminator_index(&self) -> Option<usize> { ... }
    fn iter_sentence_terminator_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_sentence_terminators(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_space(&self) -> Option<&Token> { ... }
    fn last_space(&self) -> Option<&Token> { ... }
    fn last_space_index(&self) -> Option<usize> { ... }
    fn iter_space_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_spaces(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_unlintable(&self) -> Option<&Token> { ... }
    fn last_unlintable(&self) -> Option<&Token> { ... }
    fn last_unlintable_index(&self) -> Option<usize> { ... }
    fn iter_unlintable_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_unlintables(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_verb(&self) -> Option<&Token> { ... }
    fn last_verb(&self) -> Option<&Token> { ... }
    fn last_verb_index(&self) -> Option<usize> { ... }
    fn iter_verb_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_verbs(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_word(&self) -> Option<&Token> { ... }
    fn last_word(&self) -> Option<&Token> { ... }
    fn last_word_index(&self) -> Option<usize> { ... }
    fn iter_word_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_words(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_word_like(&self) -> Option<&Token> { ... }
    fn last_word_like(&self) -> Option<&Token> { ... }
    fn last_word_like_index(&self) -> Option<usize> { ... }
    fn iter_word_like_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_word_likes(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_heading_start(&self) -> Option<&Token> { ... }
    fn last_heading_start(&self) -> Option<&Token> { ... }
    fn last_heading_start_index(&self) -> Option<usize> { ... }
    fn iter_heading_start_indices(
        &self,
    ) -> impl DoubleEndedIterator<Item = usize> + '_ { ... }
    fn iter_heading_starts(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn first_sentence_word(&self) -> Option<&Token> { ... }
    fn first_non_whitespace(&self) -> Option<&Token> { ... }
    fn span(&self) -> Option<Span<char>> { ... }
    fn get_rel(&self, index: isize) -> Option<&Token>
       where Self: AsRef<[Token]> { ... }
    fn get_rel_slice(
        &self,
        rel_start: isize,
        inclusive_end: isize,
    ) -> Option<&[Token]>
       where Self: AsRef<[Token]> { ... }
    fn get_ch<'a>(&self, src: &'a [char]) -> Option<&'a [char]> { ... }
    fn get_str(&self, src: &[char]) -> Option<String> { ... }
    fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_ { ... }
    fn iter_linking_verbs(&self) -> impl Iterator<Item = &Token> + '_ { ... }
    fn iter_chunks(&self) -> impl Iterator<Item = &[Token]> + '_ { ... }
    fn iter_paragraphs(&self) -> impl Iterator<Item = &[Token]> + '_ { ... }
    fn iter_headings(&self) -> impl Iterator<Item = &[Token]> + '_ { ... }
    fn iter_sentences(&self) -> impl Iterator<Item = &[Token]> + '_ { ... }
    fn iter_sentences_mut(&mut self) -> impl Iterator<Item = &mut [Token]> + '_ { ... }
}

Extension methods for Token sequences that make them easier to wrangle and query.

Required Methods

fn tokens(&self) -> &[Token]

fn tokens_mut(&mut self) -> &mut [Token]

Provided Methods

fn first_adjective(&self) -> Option<&Token>

fn last_adjective(&self) -> Option<&Token>

fn last_adjective_index(&self) -> Option<usize>

fn iter_adjective_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_adjectives(&self) -> impl Iterator<Item = &Token> + '_

fn first_apostrophe(&self) -> Option<&Token>

fn last_apostrophe(&self) -> Option<&Token>

fn last_apostrophe_index(&self) -> Option<usize>

fn iter_apostrophe_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_apostrophes(&self) -> impl Iterator<Item = &Token> + '_

fn first_at(&self) -> Option<&Token>

fn last_at(&self) -> Option<&Token>

fn last_at_index(&self) -> Option<usize>

fn iter_at_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_ats(&self) -> impl Iterator<Item = &Token> + '_

fn first_comma(&self) -> Option<&Token>

fn last_comma(&self) -> Option<&Token>

fn last_comma_index(&self) -> Option<usize>

fn iter_comma_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_commas(&self) -> impl Iterator<Item = &Token> + '_

fn first_conjunction(&self) -> Option<&Token>

fn last_conjunction(&self) -> Option<&Token>

fn last_conjunction_index(&self) -> Option<usize>

fn iter_conjunction_indices( &self, ) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_conjunctions(&self) -> impl Iterator<Item = &Token> + '_

fn first_chunk_terminator(&self) -> Option<&Token>

fn last_chunk_terminator(&self) -> Option<&Token>

fn last_chunk_terminator_index(&self) -> Option<usize>

fn iter_chunk_terminator_indices( &self, ) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_chunk_terminators(&self) -> impl Iterator<Item = &Token> + '_

fn first_currency(&self) -> Option<&Token>

fn last_currency(&self) -> Option<&Token>

fn last_currency_index(&self) -> Option<usize>

fn iter_currency_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_currencys(&self) -> impl Iterator<Item = &Token> + '_

fn first_ellipsis(&self) -> Option<&Token>

fn last_ellipsis(&self) -> Option<&Token>

fn last_ellipsis_index(&self) -> Option<usize>

fn iter_ellipsis_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_ellipsiss(&self) -> impl Iterator<Item = &Token> + '_

fn first_hostname(&self) -> Option<&Token>

fn last_hostname(&self) -> Option<&Token>

fn last_hostname_index(&self) -> Option<usize>

fn iter_hostname_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_hostnames(&self) -> impl Iterator<Item = &Token> + '_

fn first_likely_homograph(&self) -> Option<&Token>

fn last_likely_homograph(&self) -> Option<&Token>

fn last_likely_homograph_index(&self) -> Option<usize>

fn iter_likely_homograph_indices( &self, ) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_likely_homographs(&self) -> impl Iterator<Item = &Token> + '_

fn first_number(&self) -> Option<&Token>

fn last_number(&self) -> Option<&Token>

fn last_number_index(&self) -> Option<usize>

fn iter_number_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_numbers(&self) -> impl Iterator<Item = &Token> + '_

fn first_noun(&self) -> Option<&Token>

fn last_noun(&self) -> Option<&Token>

fn last_noun_index(&self) -> Option<usize>

fn iter_noun_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_nouns(&self) -> impl Iterator<Item = &Token> + '_

fn first_paragraph_break(&self) -> Option<&Token>

fn last_paragraph_break(&self) -> Option<&Token>

fn last_paragraph_break_index(&self) -> Option<usize>

fn iter_paragraph_break_indices( &self, ) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_paragraph_breaks(&self) -> impl Iterator<Item = &Token> + '_

fn first_pipe(&self) -> Option<&Token>

fn last_pipe(&self) -> Option<&Token>

fn last_pipe_index(&self) -> Option<usize>

fn iter_pipe_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_pipes(&self) -> impl Iterator<Item = &Token> + '_

fn first_preposition(&self) -> Option<&Token>

fn last_preposition(&self) -> Option<&Token>

fn last_preposition_index(&self) -> Option<usize>

fn iter_preposition_indices( &self, ) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_prepositions(&self) -> impl Iterator<Item = &Token> + '_

fn first_punctuation(&self) -> Option<&Token>

fn last_punctuation(&self) -> Option<&Token>

fn last_punctuation_index(&self) -> Option<usize>

fn iter_punctuation_indices( &self, ) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_punctuations(&self) -> impl Iterator<Item = &Token> + '_

fn first_quote(&self) -> Option<&Token>

fn last_quote(&self) -> Option<&Token>

fn last_quote_index(&self) -> Option<usize>

fn iter_quote_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_quotes(&self) -> impl Iterator<Item = &Token> + '_

fn first_sentence_terminator(&self) -> Option<&Token>

fn last_sentence_terminator(&self) -> Option<&Token>

fn last_sentence_terminator_index(&self) -> Option<usize>

fn iter_sentence_terminator_indices( &self, ) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_sentence_terminators(&self) -> impl Iterator<Item = &Token> + '_

fn first_space(&self) -> Option<&Token>

fn last_space(&self) -> Option<&Token>

fn last_space_index(&self) -> Option<usize>

fn iter_space_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_spaces(&self) -> impl Iterator<Item = &Token> + '_

fn first_unlintable(&self) -> Option<&Token>

fn last_unlintable(&self) -> Option<&Token>

fn last_unlintable_index(&self) -> Option<usize>

fn iter_unlintable_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_unlintables(&self) -> impl Iterator<Item = &Token> + '_

fn first_verb(&self) -> Option<&Token>

fn last_verb(&self) -> Option<&Token>

fn last_verb_index(&self) -> Option<usize>

fn iter_verb_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_verbs(&self) -> impl Iterator<Item = &Token> + '_

fn first_word(&self) -> Option<&Token>

fn last_word(&self) -> Option<&Token>

fn last_word_index(&self) -> Option<usize>

fn iter_word_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_words(&self) -> impl Iterator<Item = &Token> + '_

fn first_word_like(&self) -> Option<&Token>

fn last_word_like(&self) -> Option<&Token>

fn last_word_like_index(&self) -> Option<usize>

fn iter_word_like_indices(&self) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_word_likes(&self) -> impl Iterator<Item = &Token> + '_

fn first_heading_start(&self) -> Option<&Token>

fn last_heading_start(&self) -> Option<&Token>

fn last_heading_start_index(&self) -> Option<usize>

fn iter_heading_start_indices( &self, ) -> impl DoubleEndedIterator<Item = usize> + '_

fn iter_heading_starts(&self) -> impl Iterator<Item = &Token> + '_

fn first_sentence_word(&self) -> Option<&Token>

fn first_non_whitespace(&self) -> Option<&Token>

fn span(&self) -> Option<Span<char>>

Grab the span that represents the beginning of the first element and the end of the last element.

fn get_rel(&self, index: isize) -> Option<&Token>

where Self: AsRef<[Token]>,

Get a reference to a token by index, with negative numbers counting from the end.

Examples

let source = "The cat sat on the mat.".chars().collect::<Vec<_>>();
let tokens = PlainEnglish.parse(&source);
assert_eq!(tokens.get_rel(0).unwrap().get_str(&source), "The");
assert_eq!(tokens.get_rel(1).unwrap().kind.is_whitespace(), true);
assert_eq!(tokens.get_rel(-1).unwrap().kind.is_punctuation(), true);
assert_eq!(tokens.get_rel(-2).unwrap().get_str(&source), "mat");

Returns

• Some(&Token) - If the index is in bounds
• None - If the index is out of bounds

fn get_rel_slice( &self, rel_start: isize, inclusive_end: isize, ) -> Option<&[Token]>

where Self: AsRef<[Token]>,

Get a slice of tokens using relative indices.

Examples

let source = "The cat sat on the mat.".chars().collect::<Vec<_>>();
let tokens = PlainEnglish.parse(&source);
assert_eq!(tokens.get_rel_slice(0, 2).unwrap().span().unwrap().get_content_string(&source), "The cat");
assert_eq!(tokens.get_rel_slice(-3, -1).unwrap().span().unwrap().get_content_string(&source), " mat.");

fn get_ch<'a>(&self, src: &'a [char]) -> Option<&'a [char]>

fn get_str(&self, src: &[char]) -> Option<String>

fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_

fn iter_linking_verbs(&self) -> impl Iterator<Item = &Token> + '_

fn iter_chunks(&self) -> impl Iterator<Item = &[Token]> + '_

Iterate over chunks.

For example, the following sentence contains two chunks separated by a comma:

Here is an example, it is short.

fn iter_paragraphs(&self) -> impl Iterator<Item = &[Token]> + '_

Get an iterator over token slices that represent the individual paragraphs in a document.

fn iter_headings(&self) -> impl Iterator<Item = &[Token]> + '_

Get an iterator over token slices that represent headings.

A heading begins with a TokenKind::HeadingStart token and ends with the next TokenKind::ParagraphBreak.

fn iter_sentences(&self) -> impl Iterator<Item = &[Token]> + '_

Get an iterator over token slices that represent the individual sentences in a document.

fn iter_sentences_mut(&mut self) -> impl Iterator<Item = &mut [Token]> + '_

Get an iterator over mutable token slices that represent the individual sentences in a document.

Dyn Compatibility

This trait is not dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementations on Foreign Types

impl TokenStringExt for [Token]

fn tokens(&self) -> &[Token]

fn tokens_mut(&mut self) -> &mut [Token]

Implementors

impl TokenStringExt for Document