[−][src]Trait sbert::Tokenizer

pub trait Tokenizer {
    fn new<P: Into<PathBuf>>(path: P) -> Result<Self, Error>
    where
        Self: Sized;

    fn pre_tokenize<S: AsRef<str>>(&self, input: &[S]) -> Vec<Vec<String>>;

    fn tokenize<S: AsRef<str>>(&self, input: &[S]) -> (Vec<Tensor>, Vec<Tensor>);
}

Required methods

fn new<P: Into<PathBuf>>(path: P) -> Result<Self, Error> where
    Self: Sized, 

fn pre_tokenize<S: AsRef<str>>(&self, input: &[S]) -> Vec<Vec<String>>

fn tokenize<S: AsRef<str>>(&self, input: &[S]) -> (Vec<Tensor>, Vec<Tensor>)

Loading content...

Implementors

impl Tokenizer for HFTokenizer[src]

impl Tokenizer for RustTokenizers[src]

Loading content...