Struct finalfrontier::SubwordVocab

pub struct SubwordVocab { /* fields omitted */ }

A corpus vocabulary with subword lookup.

Methods

impl SubwordVocab

pub fn word(&self, word: &str) -> Option<&Word>

Get the given word.

pub fn subword_indices(&self, word: &str) -> Cow<[u64]>

Get the subword indices of a word.

pub fn indices(&self, word: &str) -> Vec<u64>

Get all indices of a word, both regular and subword.

This method copies the subword list for known words into a new Vec.

Trait Implementations

impl Vocab for SubwordVocab

type VocabType = String

type Config = SubwordVocabConfig

fn config(&self) -> SubwordVocabConfig

Return this vocabulary's config.

fn idx<Q: ?Sized>(&self, key: &Q) -> Option<usize> where
    Self::VocabType: Borrow<Q>,
    Q: Hash + Eq, 

Get the index of the entry, will return None if the item is not present.

fn discard(&self, idx: usize) -> f32

Get the discard probability of the entry with the given index.

fn types(&self) -> &[Word]

Get all types in the vocabulary.

fn n_types(&self) -> usize

Get the number of types in the corpus.

fn is_empty(&self) -> bool

fn len(&self) -> usize

Get the number of entries in the vocabulary.

impl From<SubwordVocab> for VocabWrap

fn from(vocab: SubwordVocab) -> VocabWrap

Performs the conversion.

impl<T> From<VocabBuilder<SubwordVocabConfig, T>> for SubwordVocab where
    T: Hash + Eq + Into<String>, 

Constructs a SubwordVocab from a VocabBuilder<T> where T: Into<String>.

fn from(builder: VocabBuilder<SubwordVocabConfig, T>) -> Self

Performs the conversion.

impl Clone for SubwordVocab

fn clone(&self) -> SubwordVocab

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.