candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

//! `Qdrant/bm25`'s term-frequency saturation, `count·(k+1) / (count + k·(1 - b + b·doc_len/avg_len))`,
//! with its own constants; `doc_len` counts the surviving stems.

/// Declared apart from `minicoil::term_frequency::K`, which only happens to match.
pub(crate) const K: f32 = 1.2;

/// Declared apart from `minicoil::term_frequency::B`, like `K`.
pub(crate) const B: f32 = 0.75;

/// `Bm25.__init__`'s default average length in surviving stems, not miniCOIL's 150.
pub(crate) const AVERAGE_LENGTH: f32 = 256.0;

/// Callers pass `count >= 1` and `doc_len >= 1`, so nothing divides by zero.
pub(crate) fn saturation(count: u32, doc_len: u32) -> f32 {
  let count = count as f32;
  let doc_len = doc_len as f32;
  count * (K + 1.0) / (count + K * (1.0 - B + B * doc_len / AVERAGE_LENGTH))
}

#[cfg(test)]
mod tests {
  use super::*;

  /// At `doc_len == AVERAGE_LENGTH` and `count == 1` the ratio is exactly `(k+1)/(1+k) == 1`, no tolerance needed.
  #[test]
  fn a_single_occurrence_at_the_average_length_weighs_exactly_one() {
    assert_eq!(saturation(1, 256), 1.0);
  }

  /// `fastembed`'s `_term_frequency` at `k = 1.2`, `b = 0.75`, `avg_len = 256`: one occurrence in six tokens weighs 1.6652868.
  #[test]
  fn matches_the_closed_form_on_the_short_fixture() {
    let got = saturation(1, 6);
    assert!((got - 1.665_286_8).abs() < 1e-5, "{got}");
  }

  #[test]
  fn a_shorter_than_average_document_weighs_a_single_occurrence_higher() {
    let short = saturation(1, 1);
    let at_average = saturation(1, 256);
    assert!(short > at_average, "{short} vs {at_average}");
  }

  #[test]
  fn differs_from_minicoils_own_average_length_at_the_same_input() {
    let bm25 = saturation(1, 200);
    let minicoil = crate::models::sparse::minicoil::term_frequency::saturation(1, 200);
    assert_ne!(bm25, minicoil);
  }
}