candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

//! BM25's term-frequency half, `count·(k+1) / (count + k·(1 - b + b·doc_len/avg_len))`, at miniCOIL's constants;
//! `count` and `doc_len` count reconstructed words, not subword tokens.

/// BM25's tf-saturation curvature; neither model makes it configurable.
pub const K: f32 = 1.2;

/// BM25's length-normalization strength.
pub const B: f32 = 0.75;

/// miniCOIL's average document length in resolved words: the reference's default of 150, not BM25's 256.
pub const AVERAGE_LENGTH: f32 = 150.0;

/// Callers pass `count >= 1` and `doc_len >= 1`, so nothing divides by zero.
pub fn saturation(count: u32, doc_len: u32) -> f32 {
  let count = count as f32;
  let doc_len = doc_len as f32;
  count * (K + 1.0) / (count + K * (1.0 - B + B * doc_len / AVERAGE_LENGTH))
}

#[cfg(test)]
mod tests {
  use super::*;

  /// At `doc_len == AVERAGE_LENGTH` and `count == 1` the ratio is exactly `(k+1)/(1+k) == 1`, no tolerance needed.
  #[test]
  fn a_single_occurrence_at_the_average_length_weighs_exactly_one() {
    assert_eq!(saturation(1, 150), 1.0);
  }

  /// `3·2.2/4.2` is not exact in f32, hence the tolerance.
  #[test]
  fn repeated_occurrences_at_the_average_length_scale_by_the_bm25_ratio() {
    let got = saturation(3, 150);
    let want = 3.0 * 2.2 / 4.2;
    assert!((got - want).abs() < 1e-6, "{got} vs {want}");
  }

  #[test]
  fn a_shorter_than_average_document_weighs_a_single_occurrence_higher() {
    let short = saturation(1, 1);
    let at_average = saturation(1, 150);
    assert!(short > at_average, "{short} vs {at_average}");
  }

  /// One occurrence in five words: `1 - 0.75 + 0.75·5/150 = 0.275`, so the result is `2.2/1.33`.
  #[test]
  fn matches_hand_arithmetic_at_a_short_row() {
    let got = saturation(1, 5);
    let want = 2.2 / 1.33;
    assert!((got - want).abs() < 1e-5, "{got} vs {want}");
  }
}