candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

//! A recognized word gets a dense index (one-based id times four plus the component); any other word a hash of
//! its stem, reduced and shifted, unlike `Qdrant/bm25`'s plain hash despite the model's README.

use crate::models::sparse::hash::murmur_abs_hash;
use crate::models::sparse::stemmer::stem;

/// Sparse entries per recognized word, one per component of the `(19125, 512, 4)` head matrix.
pub const EMBEDDING_SIZE: u32 = 4;

const GAP: u32 = 32_000;

/// The reference's vocabulary size, 19124 real words plus the reserved unrecognized id 0; `SHIFT` is built from it.
pub(super) const VOCAB_SIZE_INCLUDING_UNRECOGNIZED: u32 = 19_125;

/// `((vocab_size * embedding_size) // GAP + 2) * GAP` in the reference, 128000 here: the hashed range starts above
/// every dense index. Fixed, so `MiniCoilSparse::new` checks a loaded matrix against it.
pub(super) const SHIFT: u32 = (VOCAB_SIZE_INCLUDING_UNRECOGNIZED * EMBEDDING_SIZE / GAP + 2) * GAP;

/// `INT32_MAX - shift` in the reference: the modulus the hashed value is reduced into before `SHIFT` is added.
const RANGE: u32 = i32::MAX as u32 - SHIFT;

/// A recognized word's index from its one-based `word_id`, in `4..=76499` for this vocabulary.
pub fn dense_index(word_id: u32, component: u32) -> u32 {
  word_id * EMBEDDING_SIZE + component
}

/// `SHIFT + abs(murmur3_x86_32(stem(word))) % RANGE`; stems `word` itself, unlike `bm25_hash`.
pub fn hashed_index(word: &str) -> u32 {
  SHIFT + murmur_abs_hash(&stem(word)) % RANGE
}

#[cfg(test)]
mod tests;