bm25-vectorizer 1.0.0

A minimal Rust library for creating sparse vector representations (embeddings) using the BM25 algorithm for information retrieval.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

mod bm25_tokenizer;
mod bm25_vectorizer;
mod mocking;
mod bm25_token_indexer;

mod example;

pub use bm25_token_indexer::Bm25TokenIndexer;
pub use bm25_tokenizer::Bm25Tokenizer;
pub use bm25_vectorizer::AverageDocumentLength;
pub use bm25_vectorizer::Bm25Vectorizer;
pub use bm25_vectorizer::Bm25VectorizerBuilder;
pub use bm25_vectorizer::LengthNormalisation;
pub use bm25_vectorizer::SparseRepresentation;
pub use bm25_vectorizer::TermFrequencyLowerBound;
pub use bm25_vectorizer::TermRelevanceSaturation;
pub use bm25_vectorizer::TokenIndexValue;
pub use mocking::MockCasePreservingTokenizer;
pub use mocking::MockDictionaryTokenIndexer;
pub use mocking::MockHashTokenIndexer;
pub use mocking::MockPunctuationTokenizer;
pub use mocking::MockStringTokenIndexer;
pub use mocking::MockWhitespaceTokenizer;