Trait finalfusion::vocab::Vocab

pub trait Vocab {
    fn idx(&self, word: &str) -> Option<WordIndex>;
    fn words_len(&self) -> usize;
    fn vocab_len(&self) -> usize;
    fn words(&self) -> &[String];
}

Embedding vocabularies.

Required methods

fn idx(&self, word: &str) -> Option<WordIndex>

Get the index of a token.

fn words_len(&self) -> usize

Get the number of words in the vocabulary.

fn vocab_len(&self) -> usize

Get the total length of this vocabulary, including possible subword indices.

fn words(&self) -> &[String]

Get the words in the vocabulary.

Loading content...

Implementors

impl Vocab for VocabWrap

fn idx(&self, word: &str) -> Option<WordIndex>

fn words_len(&self) -> usize

Get the vocabulary size.

fn vocab_len(&self) -> usize

fn words(&self) -> &[String]

Get the words in the vocabulary.

impl Vocab for SimpleVocab

fn idx(&self, word: &str) -> Option<WordIndex>

fn words_len(&self) -> usize

fn vocab_len(&self) -> usize

fn words(&self) -> &[String]

impl<I> Vocab for SubwordVocab<I> where
    I: Indexer, 

fn idx(&self, word: &str) -> Option<WordIndex>

fn words_len(&self) -> usize

fn vocab_len(&self) -> usize

fn words(&self) -> &[String]

Loading content...