finalfusion/
lib.rs

1//! A library for reading, writing, and using word embeddings.
2//!
3//! finalfusion allows you to read, write, and use
4//! word2vec/[GloVe](https://nlp.stanford.edu/projects/glove/)
5//! embeddings and read [fastText](https://fasttext.cc/) embeddings.
6//! finalfusion uses *finalfusion* as its native data format, which
7//! has several benefits over the word2vec, GloVe, and fastText
8//! formats.
9//!
10//! ## Reading finalfusion embeddings
11//!
12//! finalfusion embeddings can be read with the `read_embeddings`
13//! method, which expects a reader that implements the `BufRead`
14//! trait.
15//!
16//! Since finalfusion supports various types of vocabularies and
17//! embedding matrix (storage) formats, these should be specified
18//! as type parameters of the `Embeddings` type. However, typically
19//! one would want to read finalfusion embeddings with any type of
20//! vocabulary or embedding matrix. For this purpose, the `VocabWrap`
21//! and `StorageWrap` types are provided, which wrap any type of
22//! vocabulary and embeddung matrix.
23//!
24//! We can thus load a finalfusion format and retrieve an embedding
25//! as follows:
26//!
27//! ```
28//! use std::fs::File;
29//! use std::io::BufReader;
30//!
31//! use finalfusion::prelude::*;
32//!
33//! let mut reader = BufReader::new(File::open("testdata/similarity.fifu").unwrap());
34//!
35//! // Read the embeddings.
36//! let embeddings: Embeddings<VocabWrap, StorageWrap> =
37//!     Embeddings::read_embeddings(&mut reader)
38//!     .unwrap();
39//!
40//! // Look up an embedding.
41//! let embedding = embeddings.embedding("Berlin");
42//! ```
43//!
44//! For performing analogy/similarity queries on the embedding
45//! matrix, we need an embedding matrix which can act as a view.
46//! In that case one should use `StorageViewWrap` in place of
47//! `StorageWrap`. `StorageViewWrap` is only supported for a
48//! subset of embedding matrix types -- in particular, quantized
49//! matrices cannot be used as a view.
50//!
51//! ## Reading other embedding formats
52//!
53//! Consult the documentation of the `fasttext`, `text` and
54//! `word2vec` modules for information on how to read fastText,
55//! GloVe, and word2vec embeddings.
56
57mod chunks;
58pub use chunks::{metadata, norms, storage, vocab};
59
60pub mod compat;
61
62pub mod embeddings;
63
64pub mod error;
65
66pub mod io;
67
68pub mod prelude;
69
70pub mod similarity;
71
72pub mod subword;
73
74pub(crate) mod util;
finalfusion/lib.rs

finalfusion/
lib.rs
