Struct TextTokenizer

pub struct TextTokenizer {
    pub tokenizer: Tokenizer,
}

Fields

tokenizer: Tokenizer

Implementations

impl TextTokenizer

pub fn new(tokenizer: Tokenizer) -> TextTokenizer

pub fn from_config<P: AsRef<Path>>(path: P) -> Result<TextTokenizer, Error>

pub fn tokenize_question_answer( &self, question: &str, context: &str, ) -> Result<Vec<EncodedText>, PreprocessingError>

pub fn decode( &self, encoding: &EncodedText, ) -> Result<String, PreprocessingError>

pub fn get_mask_input_for( &self, text: &str, shape: &Shape, ) -> Result<Tensor, PreprocessingError>

pub fn get_input_for( &self, text: &str, shape: &Shape, ) -> Result<Tensor, PreprocessingError>