Struct finalfusion::embeddings::Embeddings

pub struct Embeddings<V, S> { /* fields omitted */ }

Word embeddings.

This data structure stores word embeddings (also known as word vectors) and provides some useful methods on the embeddings, such as similarity and analogy queries.

Methods

impl<V, S> Embeddings<V, S> where
    V: Vocab,
    S: Storage, 

pub fn new(
    metadata: Option<Metadata>,
    vocab: V,
    storage: S,
    norms: NdNorms
) -> Self

Construct an embeddings from a vocabulary, storage, and norms.

The embeddings for known words must be normalized. However, this is not verified due to the high computational cost.

impl<V, S> Embeddings<V, S>

pub fn into_parts(self) -> (Option<Metadata>, V, S, Option<NdNorms>)

Decompose embeddings in its vocabulary, storage, and optionally norms.

pub fn metadata(&self) -> Option<&Metadata>

Get metadata.

pub fn metadata_mut(&mut self) -> Option<&mut Metadata>

Get metadata mutably.

pub fn norms(&self) -> Option<&NdNorms>

Get embedding norms.

pub fn set_metadata(&mut self, metadata: Option<Metadata>) -> Option<Metadata>

Set metadata.

Returns the previously-stored metadata.

pub fn storage(&self) -> &S

Get the embedding storage.

pub fn vocab(&self) -> &V

Get the vocabulary.

impl<V, S> Embeddings<V, S> where
    V: Vocab,
    S: Storage, 

pub fn dims(&self) -> usize

Return the length (in vector components) of the word embeddings.

pub fn embedding(&self, word: &str) -> Option<CowArray<f32, Ix1>>

Get the embedding of a word.

pub fn embedding_into(&self, word: &str, target: ArrayViewMut1<f32>) -> bool

Realize the embedding of a word into the given vector.

This variant of embedding realizes the embedding into the given vector. This makes it possible to look up embeddings without any additional allocations. This method returns false and does not modify the vector if no embedding could be found.

Panics when then the vector does not have the same dimensionality as the word embeddings.

pub fn embedding_with_norm(&self, word: &str) -> Option<EmbeddingWithNorm>

Get the embedding and original norm of a word.

Returns for a word:

• The word embedding.
• The norm of the embedding before normalization to a unit vector.

The original embedding can be reconstructed by multiplying all embedding components by the original norm.

If the model does not have associated norms, 1 will be returned as the norm for vocabulary words.

pub fn iter(&self) -> Iter

Get an iterator over pairs of words and the corresponding embeddings.

pub fn iter_with_norms(&self) -> IterWithNorms

Get an iterator over triples of words, embeddings, and norms.

Returns an iterator that returns triples of:

• A word.
• Its word embedding.
• The original norm of the embedding before normalization to a unit vector.

The original embedding can be reconstructed by multiplying all embedding components by the original norm.

If the model does not have associated norms, the norm is always 1.

pub fn len(&self) -> usize

Get the vocabulary size.

The vocabulary size excludes subword units.

Trait Implementations

impl<V, S> Analogy for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn analogy_masked(
    &self,
    query: [&str; 3],
    remove: [bool; 3],
    limit: usize
) -> Result<Vec<WordSimilarityResult>, [bool; 3]>

Perform an analogy query.

fn analogy(
    &self,
    query: [&str; 3],
    limit: usize
) -> Result<Vec<WordSimilarityResult>, [bool; 3]>

Perform an analogy query.

impl<V, S> AnalogyBy for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn analogy_by_masked<F>(
    &self,
    query: [&str; 3],
    remove: [bool; 3],
    limit: usize,
    similarity: F
) -> Result<Vec<WordSimilarityResult>, [bool; 3]> where
    F: FnMut(ArrayView2<f32>, ArrayView1<f32>) -> Array1<f32>, 

Perform an analogy query using the given similarity function.

fn analogy_by<F>(
    &self,
    query: [&str; 3],
    limit: usize,
    similarity: F
) -> Result<Vec<WordSimilarityResult>, [bool; 3]> where
    F: FnMut(ArrayView2<f32>, ArrayView1<f32>) -> Array1<f32>, 

Perform an analogy query using the given similarity function.

impl<V: Clone, S: Clone> Clone for Embeddings<V, S>

fn clone(&self) -> Embeddings<V, S>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<V: Debug, S: Debug> Debug for Embeddings<V, S>

fn fmt(&self, f: &mut Formatter) -> Result

Formats the value using the given formatter.

impl<V, S> EmbeddingSimilarity for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn embedding_similarity_masked(
    &self,
    query: ArrayView1<f32>,
    limit: usize,
    skip: &HashSet<&str>
) -> Option<Vec<WordSimilarityResult>>

Find words that are similar to the query embedding while skipping certain words.

fn embedding_similarity(
    &self,
    query: ArrayView1<f32>,
    limit: usize
) -> Option<Vec<WordSimilarityResult>>

Find words that are similar to the query embedding.

impl<V, S> EmbeddingSimilarityBy for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn embedding_similarity_by<F>(
    &self,
    query: ArrayView1<f32>,
    limit: usize,
    skip: &HashSet<&str>,
    similarity: F
) -> Option<Vec<WordSimilarityResult>> where
    F: FnMut(ArrayView2<f32>, ArrayView1<f32>) -> Array1<f32>, 

Find words that are similar to the query embedding using the given similarity function.

impl From<Embeddings<SimpleVocab, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SimpleVocab, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<SimpleVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SimpleVocab, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, MmapQuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<SimpleVocab, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SimpleVocab, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<SimpleVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SimpleVocab, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, QuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, MmapQuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, QuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FastTextSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, MmapQuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FastTextSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, QuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<BucketSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<BucketSubwordVocab, MmapArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<BucketSubwordVocab, MmapQuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<BucketSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<BucketSubwordVocab, NdArray>) -> Self

Performs the conversion.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<BucketSubwordVocab, QuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<VocabWrap, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<VocabWrap, MmapQuantizedArray>) -> Self

Performs the conversion.

impl From<Embeddings<VocabWrap, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<VocabWrap, QuantizedArray>) -> Self

Performs the conversion.

impl<'a, V, S> IntoIterator for &'a Embeddings<V, S> where
    V: Vocab,
    S: Storage, 

type Item = (&'a str, CowArray<'a, f32, Ix1>)

The type of the elements being iterated over.

type IntoIter = Iter<'a>

Which kind of iterator are we turning this into?

fn into_iter(self) -> Self::IntoIter

Creates an iterator from a value.

impl<V, S> MmapEmbeddings for Embeddings<V, S> where
    Self: Sized,
    V: ReadChunk,
    S: MmapChunk, 

fn mmap_embeddings(read: &mut BufReader<File>) -> Result<Self>

impl<V, S> Quantize<V> for Embeddings<V, S> where
    V: Vocab + Clone,
    S: StorageView, 

fn quantize_using<T, R>(
    &self,
    n_subquantizers: usize,
    n_subquantizer_bits: u32,
    n_iterations: usize,
    n_attempts: usize,
    normalize: bool,
    rng: R
) -> Embeddings<V, QuantizedArray> where
    T: TrainPQ<f32>,
    R: RngCore + SeedableRng + Send, 

Quantize the embedding matrix using the provided RNG.

fn quantize<T>(
    &self,
    n_subquantizers: usize,
    n_subquantizer_bits: u32,
    n_iterations: usize,
    n_attempts: usize,
    normalize: bool
) -> Embeddings<V, QuantizedArray> where
    T: TrainPQ<f32>, 

Quantize the embedding matrix.

impl<V, S> ReadEmbeddings for Embeddings<V, S> where
    V: ReadChunk,
    S: ReadChunk, 

fn read_embeddings<R>(read: &mut R) -> Result<Self> where
    R: Read + Seek, 

Read the embeddings.

impl ReadFastText for Embeddings<FastTextSubwordVocab, NdArray>

fn read_fasttext(reader: &mut impl BufRead) -> Result<Self>

Read embeddings in the fastText format.

fn read_fasttext_lossy(reader: &mut impl BufRead) -> Result<Self>

Read embeddings in the fastText format lossily.

impl<R> ReadText<R> for Embeddings<SimpleVocab, NdArray> where
    R: BufRead, 

fn read_text(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

fn read_text_lossy(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

impl<R> ReadTextDims<R> for Embeddings<SimpleVocab, NdArray> where
    R: BufRead, 

fn read_text_dims(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

fn read_text_dims_lossy(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

impl<R> ReadWord2Vec<R> for Embeddings<SimpleVocab, NdArray> where
    R: BufRead, 

fn read_word2vec_binary(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

fn read_word2vec_binary_lossy(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

impl<V, S> WordSimilarity for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn word_similarity(
    &self,
    word: &str,
    limit: usize
) -> Option<Vec<WordSimilarityResult>>

Find words that are similar to the query word.

impl<V, S> WordSimilarityBy for Embeddings<V, S> where
    V: Vocab,
    S: StorageView, 

fn word_similarity_by<F>(
    &self,
    word: &str,
    limit: usize,
    similarity: F
) -> Option<Vec<WordSimilarityResult>> where
    F: FnMut(ArrayView2<f32>, ArrayView1<f32>) -> Array1<f32>, 

Find words that are similar to the query word using the given similarity function.

impl<V, S> WriteEmbeddings for Embeddings<V, S> where
    V: WriteChunk,
    S: WriteChunk, 

fn write_embeddings<W>(&self, write: &mut W) -> Result<()> where
    W: Write + Seek, 

impl<W, V, S> WriteText<W> for Embeddings<V, S> where
    W: Write,
    V: Vocab,
    S: Storage, 

fn write_text(&self, write: &mut W, unnormalize: bool) -> Result<()>

Read the embeddings from the given buffered reader.

impl<W, V, S> WriteTextDims<W> for Embeddings<V, S> where
    W: Write,
    V: Vocab,
    S: Storage, 

fn write_text_dims(&self, write: &mut W, unnormalize: bool) -> Result<()>

Write the embeddings to the given writer.

impl<W, V, S> WriteWord2Vec<W> for Embeddings<V, S> where
    W: Write,
    V: Vocab,
    S: Storage, 

fn write_word2vec_binary(&self, w: &mut W, unnormalize: bool) -> Result<()> where
    W: Write, 

Write the embeddings from the given writer.