Struct wonnx_preprocessing::text::TextTokenizer

pub struct TextTokenizer {
    pub tokenizer: Tokenizer,
}

Fields

tokenizer: Tokenizer

Implementations

impl TextTokenizer

pub fn new(tokenizer: Tokenizer) -> TextTokenizer

pub fn from_config<P: AsRef<Path>>(path: P) -> Result<TextTokenizer, Error>

pub fn tokenize_question_answer( &self, question: &str, context: &str ) -> Result<Vec<EncodedText>, PreprocessingError>

pub fn decode( &self, encoding: &EncodedText ) -> Result<String, PreprocessingError>

pub fn get_mask_input_for( &self, text: &str, shape: &Shape ) -> Result<Tensor, PreprocessingError>

pub fn get_input_for( &self, text: &str, shape: &Shape ) -> Result<Tensor, PreprocessingError>