Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Text tokenizer for splitting text into tokens

Implementations

impl Tokenizer

pub fn new() -> LangExtractResult<Self>

Create a new tokenizer

pub fn tokenize(&self, text: &str) -> LangExtractResult<TokenizedText>

Tokenize text into tokens

pub fn tokens_text( &self, tokenized_text: &TokenizedText, token_interval: &TokenInterval, ) -> LangExtractResult<String>

Reconstruct text from a token interval

pub fn is_end_of_sentence_token( &self, text: &str, tokens: &[Token], current_idx: usize, ) -> bool

Check if a punctuation token ends a sentence

pub fn is_sentence_break_after_newline( &self, text: &str, tokens: &[Token], current_idx: usize, ) -> bool

Check if there’s a sentence break after a newline

pub fn find_sentence_range( &self, text: &str, tokens: &[Token], start_token_index: usize, ) -> LangExtractResult<TokenInterval>

Find sentence range starting from a given token index

Trait Implementations

impl Default for Tokenizer

fn default() -> Self

Returns the “default value” for a type.