Trait sbert::tokenizers::Tokenizer

pub trait Tokenizer {
    // Required methods
    fn new<P: Into<PathBuf>>(path: P) -> Result<Self, Error>
       where Self: Sized;
    fn pre_tokenize<S: AsRef<str>>(&self, input: &[S]) -> Vec<Vec<String>>;
    fn tokenize<S: AsRef<str>>(&self, input: &[S]) -> (Vec<Tensor>, Vec<Tensor>);
}

Required Methods

fn new<P: Into<PathBuf>>(path: P) -> Result<Self, Error>where Self: Sized,

fn pre_tokenize<S: AsRef<str>>(&self, input: &[S]) -> Vec<Vec<String>>

fn tokenize<S: AsRef<str>>(&self, input: &[S]) -> (Vec<Tensor>, Vec<Tensor>)

Implementors

impl Tokenizer for HFTokenizer

impl Tokenizer for RustTokenizers

impl Tokenizer for RustTokenizersSentencePiece