Struct finalfusion::embeddings::Embeddings

pub struct Embeddings<V, S> { /* fields omitted */ }

Word embeddings.

This data structure stores word embeddings (also known as word vectors) and provides some useful methods on the embeddings, such as similarity and analogy queries.

Methods

impl<V, S> Embeddings<V, S> where
    V: Vocab, 

pub fn new(
    metadata: Option<Metadata>,
    vocab: V,
    storage: S,
    norms: NdNorms
) -> Self

Construct an embeddings from a vocabulary, storage, and norms.

The embeddings for known words must be normalized. However, this is not verified due to the high computational cost.

impl<V, S> Embeddings<V, S>

pub fn into_parts(self) -> (Option<Metadata>, V, S, Option<NdNorms>)

Decompose embeddings in its vocabulary, storage, and optionally norms.

pub fn metadata(&self) -> Option<&Metadata>

Get metadata.

pub fn metadata_mut(&mut self) -> Option<&mut Metadata>

Get metadata mutably.

pub fn norms(&self) -> Option<&NdNorms>

Get embedding norms.

pub fn set_metadata(&mut self, metadata: Option<Metadata>) -> Option<Metadata>

Set metadata.

Returns the previously-stored metadata.

pub fn storage(&self) -> &S

Get the embedding storage.

pub fn vocab(&self) -> &V

Get the vocabulary.

impl<V, S> Embeddings<V, S> where
    V: Vocab,
    S: Storage, 

pub fn dims(&self) -> usize

Return the length (in vector components) of the word embeddings.

pub fn embedding(&self, word: &str) -> Option<CowArray1<f32>>

Get the embedding of a word.

pub fn embedding_with_norm(&self, word: &str) -> Option<EmbeddingWithNorm>

Get the embedding and original norm of a word.

Returns for a word:

• The word embedding.
• The norm of the embedding before normalization to a unit vector.

The original embedding can be reconstructed by multiplying all embedding components by the original norm.

If the model does not have associated norms, 1 will be returned as the norm for vocabulary words.

Important traits for Iter<'a>

impl<'a> Iterator for Iter<'a> type Item = (&'a str, CowArray1<'a, f32>);

pub fn iter(&self) -> Iter

Get an iterator over pairs of words and the corresponding embeddings.

Important traits for IterWithNorms<'a>

impl<'a> Iterator for IterWithNorms<'a> type Item = (&'a str, EmbeddingWithNorm<'a>);

pub fn iter_with_norms(&self) -> IterWithNorms

Get an iterator over triples of words, embeddings, and norms.

Returns an iterator that returns triples of:

• A word.
• Its word embedding.
• The original norm of the embedding before normalization to a unit vector.

The original embedding can be reconstructed by multiplying all embedding components by the original norm.

If the model does not have associated norms, the norm is always 1.

pub fn len(&self) -> usize

Get the vocabulary size.

The vocabulary size excludes subword units.

Trait Implementations

impl ReadFastText for Embeddings<FastTextSubwordVocab, NdArray>

fn read_fasttext(reader: &mut impl BufRead) -> Result<Self>

Read embeddings in the fastText format.

fn read_fasttext_lossy(reader: &mut impl BufRead) -> Result<Self>

Read embeddings in the fastText format lossily.

impl<R> ReadText<R> for Embeddings<SimpleVocab, NdArray> where
    R: BufRead, 

fn read_text(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

fn read_text_lossy(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

impl<R> ReadTextDims<R> for Embeddings<SimpleVocab, NdArray> where
    R: BufRead, 

fn read_text_dims(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

fn read_text_dims_lossy(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

impl<W, V, S> WriteText<W> for Embeddings<V, S> where
    W: Write,
    V: Vocab,
    S: Storage, 

fn write_text(&self, write: &mut W, unnormalize: bool) -> Result<()>

Read the embeddings from the given buffered reader.

impl<W, V, S> WriteTextDims<W> for Embeddings<V, S> where
    W: Write,
    V: Vocab,
    S: Storage, 

fn write_text_dims(&self, write: &mut W, unnormalize: bool) -> Result<()>

Write the embeddings to the given writer.

impl<R> ReadWord2Vec<R> for Embeddings<SimpleVocab, NdArray> where
    R: BufRead, 

fn read_word2vec_binary(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

fn read_word2vec_binary_lossy(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

impl<W, V, S> WriteWord2Vec<W> for Embeddings<V, S> where
    W: Write,
    V: Vocab,
    S: Storage, 

fn write_word2vec_binary(&self, w: &mut W, unnormalize: bool) -> Result<()> where
    W: Write, 

Write the embeddings from the given writer.

impl<V, S> Quantize<V> for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn quantize_using<T, R>(
    &self,
    n_subquantizers: usize,
    n_subquantizer_bits: u32,
    n_iterations: usize,
    n_attempts: usize,
    normalize: bool,
    rng: &mut R
) -> Embeddings<V, QuantizedArray> where
    T: TrainPQ<f32>,
    R: Rng, 

Quantize the embedding matrix using the provided RNG.

fn quantize<T>(
    &self,
    n_subquantizers: usize,
    n_subquantizer_bits: u32,
    n_iterations: usize,
    n_attempts: usize,
    normalize: bool
) -> Embeddings<V, QuantizedArray> where
    T: TrainPQ<f32>, 

Quantize the embedding matrix.

impl<V, S> ReadEmbeddings for Embeddings<V, S> where
    V: ReadChunk,
    S: ReadChunk, 

fn read_embeddings<R>(read: &mut R) -> Result<Self> where
    R: Read + Seek, 

Read the embeddings.

impl<V, S> MmapEmbeddings for Embeddings<V, S> where
    Self: Sized,
    V: ReadChunk,
    S: MmapChunk, 

fn mmap_embeddings(read: &mut BufReader<File>) -> Result<Self>

impl<V, S> WriteEmbeddings for Embeddings<V, S> where
    V: WriteChunk,
    S: WriteChunk, 

fn write_embeddings<W>(&self, write: &mut W) -> Result<()> where
    W: Write + Seek, 

impl<V, S> Analogy for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn analogy_masked(
    &self,
    query: [&str; 3],
    remove: [bool; 3],
    limit: usize
) -> Result<Vec<WordSimilarityResult>, [bool; 3]>

Perform an analogy query.

fn analogy(
    &self,
    query: [&str; 3],
    limit: usize
) -> Result<Vec<WordSimilarityResult>, [bool; 3]>

Perform an analogy query.

impl<V, S> AnalogyBy for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn analogy_by_masked<F>(
    &self,
    query: [&str; 3],
    remove: [bool; 3],
    limit: usize,
    similarity: F
) -> Result<Vec<WordSimilarityResult>, [bool; 3]> where
    F: FnMut(ArrayView2<f32>, ArrayView1<f32>) -> Array1<f32>, 

Perform an analogy query using the given similarity function.

fn analogy_by<F>(
    &self,
    query: [&str; 3],
    limit: usize,
    similarity: F
) -> Result<Vec<WordSimilarityResult>, [bool; 3]> where
    F: FnMut(ArrayView2<f32>, ArrayView1<f32>) -> Array1<f32>, 

Perform an analogy query using the given similarity function.

impl<V, S> WordSimilarity for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn word_similarity(
    &self,
    word: &str,
    limit: usize
) -> Option<Vec<WordSimilarityResult>>

Find words that are similar to the query word.

fn word_similarity_by<F>(
    &self,
    word: &str,
    limit: usize,
    similarity: F
) -> Option<Vec<WordSimilarityResult>> where
    F: FnMut(ArrayView2<f32>, ArrayView1<f32>) -> Array1<f32>, 

Find words that are similar to the query word using the given similarity function.

impl<V, S> EmbeddingSimilarity for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn embedding_similarity_masked(
    &self,
    query: ArrayView1<f32>,
    limit: usize,
    skip: &HashSet<&str>
) -> Option<Vec<WordSimilarityResult>>

Find words that are similar to the query embedding while skipping certain words.

fn embedding_similarity_by<F>(
    &self,
    query: ArrayView1<f32>,
    limit: usize,
    skip: &HashSet<&str>,
    similarity: F
) -> Option<Vec<WordSimilarityResult>> where
    F: FnMut(ArrayView2<f32>, ArrayView1<f32>) -> Array1<f32>, 

Find words that are similar to the query embedding using the given similarity function.

fn embedding_similarity(
    &self,
    query: ArrayView1<f32>,
    limit: usize
) -> Option<Vec<WordSimilarityResult>>

Find words that are similar to the query embedding.

impl From<Embeddings<SimpleVocab, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SimpleVocab, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<SimpleVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SimpleVocab, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SimpleVocab, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<SimpleVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SimpleVocab, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, QuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FinalfusionSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FinalfusionSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FinalfusionSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FinalfusionSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FinalfusionSubwordVocab, QuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FastTextSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FastTextSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, QuantizedArray>) -> Self

Performs the conversion.

impl<'a, V, S> IntoIterator for &'a Embeddings<V, S> where
    V: Vocab,
    S: Storage, 

type Item = (&'a str, CowArray1<'a, f32>)

The type of the elements being iterated over.

type IntoIter = Iter<'a>

Which kind of iterator are we turning this into?

fn into_iter(self) -> Self::IntoIter

Creates an iterator from a value.

impl<V: Debug, S: Debug> Debug for Embeddings<V, S>

fn fmt(&self, f: &mut Formatter) -> Result

Formats the value using the given formatter.