candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

use super::*;
use crate::models::sparse::hash::bm25_hash;

/// `word_id` 19125 maps to 76503 and the real last id, 19124, to 76499.
#[test]
fn dense_index_matches_the_designs_superseded_draft_boundary_and_the_corrected_one() {
  assert_eq!(dense_index(1, 0), 4);
  assert_eq!(dense_index(19125, 3), 76503);
  assert_eq!(dense_index(19124, 3), 76499);
}

/// The README's worked example: "fruit" is word id 2248 and "bat" 4708 in the published vocabulary, with dense
/// indices `8992..=8995` and `18832..=18835`.
#[test]
fn dense_index_reproduces_the_readme_worked_example() {
  let fruit: Vec<u32> = (0..4).map(|c| dense_index(2248, c)).collect();
  assert_eq!(fruit, vec![8992, 8993, 8994, 8995]);
  let bat: Vec<u32> = (0..4).map(|c| dense_index(4708, c)).collect();
  assert_eq!(bat, vec![18832, 18833, 18834, 18835]);
}

/// The README's fallback index for "baseball": it stems to `basebal`, whose `abs(mmh3.hash)` is `2068025269`,
/// and `128000 + 2068025269 % 2147355647 == 2068153269`.
#[test]
fn hashed_index_reproduces_the_readme_worked_example() {
  assert_eq!(hashed_index("baseball"), 2_068_153_269);
}

/// `bm25_hash` applies no modulo and no shift; the stem is computed by hand because `bm25_hash` does not stem.
#[test]
fn the_two_hash_functions_disagree_on_the_same_word() {
  let word = "baseball";
  let stemmed = stem(word);
  assert_eq!(bm25_hash(&stemmed), 2_068_025_269);
  assert_ne!(hashed_index(word), bm25_hash(&stemmed));
}

#[test]
fn shift_and_range_match_the_reference_derivation() {
  assert_eq!(SHIFT, 128_000);
  assert_eq!(RANGE, 2_147_355_647);
}