Struct finalfusion::embeddings::Embeddings

pub struct Embeddings<V, S> { /* private fields */ }

Word embeddings.

This data structure stores word embeddings (also known as word vectors) and provides some useful methods on the embeddings, such as similarity and analogy queries.

Implementations

impl<V, S> Embeddings<V, S>where V: Vocab, S: Storage,

pub fn new( metadata: Option<Metadata>, vocab: V, storage: S, norms: NdNorms ) -> Self

Construct an embeddings from a vocabulary, storage, and norms.

The embeddings for known words must be normalized. However, this is not verified due to the high computational cost.

impl<V, S> Embeddings<V, S>

pub fn into_parts(self) -> (Option<Metadata>, V, S, Option<NdNorms>)

Decompose embeddings in its vocabulary, storage, and optionally norms.

pub fn metadata(&self) -> Option<&Metadata>

Get metadata.

pub fn metadata_mut(&mut self) -> Option<&mut Metadata>

Get metadata mutably.

pub fn norms(&self) -> Option<&NdNorms>

Get embedding norms.

pub fn set_metadata(&mut self, metadata: Option<Metadata>) -> Option<Metadata>

Set metadata.

Returns the previously-stored metadata.

pub fn storage(&self) -> &S

Get the embedding storage.

pub fn vocab(&self) -> &V

Get the vocabulary.

impl<V, S> Embeddings<V, S>where V: Vocab, S: Storage,

pub fn dims(&self) -> usize

Return the length (in vector components) of the word embeddings.

pub fn embedding(&self, word: &str) -> Option<CowArray<'_, f32, Ix1>>

Get the embedding of a word.

pub fn embedding_into(&self, word: &str, target: ArrayViewMut1<'_, f32>) -> bool

Realize the embedding of a word into the given vector.

This variant of embedding realizes the embedding into the given vector. This makes it possible to look up embeddings without any additional allocations. This method returns false and does not modify the vector if no embedding could be found.

Panics when then the vector does not have the same dimensionality as the word embeddings.

pub fn embedding_batch( &self, words: &[impl AsRef<str>] ) -> (Array2<f32>, Vec<bool>)

Get a batch of embeddings.

The embeddings of all words are computed and returned. This method also return a Vec indicating for each word if an embedding could be found.

pub fn embedding_batch_into( &self, words: &[impl AsRef<str>], output: ArrayViewMut2<'_, f32> ) -> Vec<bool>

Get a batch of embeddings.

The embeddings of all words are computed and written to output. A Vec is returned that indicates for each word if an embedding could be found.

This method panics when output does not have the correct shape.

pub fn embedding_with_norm(&self, word: &str) -> Option<EmbeddingWithNorm<'_>>

Get the embedding and original norm of a word.

Returns for a word:

• The word embedding.
• The norm of the embedding before normalization to a unit vector.

The original embedding can be reconstructed by multiplying all embedding components by the original norm.

If the model does not have associated norms, 1 will be returned as the norm for vocabulary words.

pub fn iter(&self) -> Iter<'_>

Get an iterator over pairs of words and the corresponding embeddings.

pub fn iter_with_norms(&self) -> IterWithNorms<'_>

Get an iterator over triples of words, embeddings, and norms.

Returns an iterator that returns triples of:

• A word.
• Its word embedding.
• The original norm of the embedding before normalization to a unit vector.

The original embedding can be reconstructed by multiplying all embedding components by the original norm.

If the model does not have associated norms, the norm is always 1.

pub fn len(&self) -> usize

Get the vocabulary size.

The vocabulary size excludes subword units.

impl<I, S> Embeddings<SubwordVocab<I>, S>where I: BucketIndexer, S: Storage + CloneFromMapping,

pub fn to_explicit(&self) -> Result<Embeddings<ExplicitSubwordVocab, S::Result>>

Convert to explicitly indexed subword Embeddings.

impl<S> Embeddings<VocabWrap, S>where S: Storage + CloneFromMapping,

pub fn try_to_explicit( &self ) -> Result<Embeddings<ExplicitSubwordVocab, S::Result>>

Try to convert to explicitly indexed subword embeddings.

Conversion fails if the wrapped vocabulary is SimpleVocab, FloretSubwordVocab or already an ExplicitSubwordVocab.

Trait Implementations

impl<V, S> Analogy for Embeddings<V, S>where V: Vocab, S: StorageView,

fn analogy_masked( &self, query: [&str; 3], remove: [bool; 3], limit: usize, batch_size: Option<usize> ) -> Result<Vec<WordSimilarityResult<'_>>, [bool; 3]>

Perform an analogy query.

fn analogy( &self, query: [&str; 3], limit: usize, batch_size: Option<usize> ) -> Result<Vec<WordSimilarityResult<'_>>, [bool; 3]>

Perform an analogy query.

impl<V: Clone, S: Clone> Clone for Embeddings<V, S>

fn clone(&self) -> Embeddings<V, S>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<V: Debug, S: Debug> Debug for Embeddings<V, S>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<V, S> EmbeddingSimilarity for Embeddings<V, S>where V: Vocab, S: StorageView,

fn embedding_similarity_masked( &self, query: ArrayView1<'_, f32>, limit: usize, skip: &HashSet<&str>, batch_size: Option<usize> ) -> Option<Vec<WordSimilarityResult<'_>>>

Find words that are similar to the query embedding while skipping certain words.

fn embedding_similarity( &self, query: ArrayView1<'_, f32>, limit: usize, batch_size: Option<usize> ) -> Option<Vec<WordSimilarityResult<'_>>>

Find words that are similar to the query embedding.

impl From<Embeddings<SimpleVocab, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<SimpleVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SimpleVocab, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SimpleVocab, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, MmapQuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SimpleVocab, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<SimpleVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SimpleVocab, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SimpleVocab, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<SimpleVocab, QuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, MmapQuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<ExplicitIndexer>, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<ExplicitSubwordVocab, QuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FastTextSubwordVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, MmapQuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FastTextSubwordVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FastTextIndexer>, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FastTextSubwordVocab, QuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FloretIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FloretSubwordVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FloretIndexer>, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FloretSubwordVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FloretIndexer>, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FloretSubwordVocab, MmapQuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FloretIndexer>, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<FloretSubwordVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FloretIndexer>, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FloretSubwordVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<FloretIndexer>, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<FloretSubwordVocab, QuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, MmapArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<BucketSubwordVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, MmapArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<BucketSubwordVocab, MmapArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<BucketSubwordVocab, MmapQuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, NdArray>> for Embeddings<VocabWrap, StorageViewWrap>

fn from(from: Embeddings<BucketSubwordVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, NdArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<BucketSubwordVocab, NdArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<SubwordVocab<HashIndexer<FnvHasher>>, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<BucketSubwordVocab, QuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<VocabWrap, MmapQuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<VocabWrap, MmapQuantizedArray>) -> Self

Converts to this type from the input type.

impl From<Embeddings<VocabWrap, QuantizedArray>> for Embeddings<VocabWrap, StorageWrap>

fn from(from: Embeddings<VocabWrap, QuantizedArray>) -> Self

Converts to this type from the input type.

impl<'a, V, S> IntoIterator for &'a Embeddings<V, S>where V: Vocab, S: Storage,

type Item = (&'a str, ArrayBase<CowRepr<'a, f32>, Dim<[usize; 1]>>)

The type of the elements being iterated over.

type IntoIter = Iter<'a>

Which kind of iterator are we turning this into?

fn into_iter(self) -> Self::IntoIter

Creates an iterator from a value.

impl<V, S> MmapEmbeddings for Embeddings<V, S>where Self: Sized, V: ReadChunk, S: MmapChunk,

fn mmap_embeddings(read: &mut BufReader<File>) -> Result<Self>

impl<V, S> Quantize<V> for Embeddings<V, S>where V: Vocab + Clone, S: StorageView,

fn quantize_using<T, R>( &self, n_subquantizers: usize, n_subquantizer_bits: u32, n_iterations: usize, n_attempts: usize, normalize: bool, rng: R ) -> Result<Embeddings<V, QuantizedArray>>where T: TrainPq<f32>, R: CryptoRng + RngCore + SeedableRng + Send,

Quantize the embedding matrix using the provided RNG.

fn quantize<T>( &self, n_subquantizers: usize, n_subquantizer_bits: u32, n_iterations: usize, n_attempts: usize, normalize: bool ) -> Result<Embeddings<V, QuantizedArray>>where T: TrainPq<f32>,

Quantize the embedding matrix.

impl<V, S> ReadEmbeddings for Embeddings<V, S>where V: ReadChunk, S: ReadChunk,

fn read_embeddings<R>(read: &mut R) -> Result<Self>where R: Read + Seek,

Read the embeddings.

impl ReadFastText for Embeddings<FastTextSubwordVocab, NdArray>

fn read_fasttext(reader: &mut impl BufRead) -> Result<Self>

Read embeddings in the fastText format.

fn read_fasttext_lossy(reader: &mut impl BufRead) -> Result<Self>

Read embeddings in the fastText format lossily.

impl ReadFloretText for Embeddings<FloretSubwordVocab, NdArray>

fn read_floret_text(reader: &mut impl BufRead) -> Result<Self>

Read embeddings in the floret format.

impl<R> ReadText<R> for Embeddings<SimpleVocab, NdArray>where R: BufRead,

fn read_text(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

fn read_text_lossy(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

impl<R> ReadTextDims<R> for Embeddings<SimpleVocab, NdArray>where R: BufRead,

fn read_text_dims(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

fn read_text_dims_lossy(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

impl<R> ReadWord2Vec<R> for Embeddings<SimpleVocab, NdArray>where R: BufRead,

fn read_word2vec_binary(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

fn read_word2vec_binary_lossy(reader: &mut R) -> Result<Self>

Read the embeddings from the given buffered reader.

impl<V, S> WordSimilarity for Embeddings<V, S>where V: Vocab, S: StorageView,

fn word_similarity( &self, word: &str, limit: usize, batch_size: Option<usize> ) -> Option<Vec<WordSimilarityResult<'_>>>

Find words that are similar to the query word.

impl<V, S> WriteEmbeddings for Embeddings<V, S>where V: WriteChunk, S: WriteChunk,

fn write_embeddings<W>(&self, write: &mut W) -> Result<()>where W: Write + Seek,

fn write_embeddings_len(&self, offset: u64) -> u64

impl<W, S> WriteFastText<W> for Embeddings<FastTextSubwordVocab, S>where W: Write, S: Storage,

fn write_fasttext(&self, write: &mut W) -> Result<()>

Write the embeddings to the given writer in fastText format.

impl WriteFloretText for Embeddings<FloretSubwordVocab, NdArray>

fn write_floret_text(&self, write: &mut dyn Write) -> Result<()>

Read embeddings in the floret format.

impl<W, V, S> WriteText<W> for Embeddings<V, S>where W: Write, V: Vocab, S: Storage,

fn write_text(&self, write: &mut W, unnormalize: bool) -> Result<()>

Read the embeddings from the given buffered reader.

impl<W, V, S> WriteTextDims<W> for Embeddings<V, S>where W: Write, V: Vocab, S: Storage,

fn write_text_dims(&self, write: &mut W, unnormalize: bool) -> Result<()>

Write the embeddings to the given writer.

impl<W, V, S> WriteWord2Vec<W> for Embeddings<V, S>where W: Write, V: Vocab, S: Storage,

fn write_word2vec_binary(&self, w: &mut W, unnormalize: bool) -> Result<()>where W: Write,

Write the embeddings from the given writer.