candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

//! `Qdrant/bm25`'s hash: the absolute murmur3 of an already-stemmed word, no modulo and no shift.
//! miniCOIL's fallback adds both, so `hashed_index` and `bm25_hash` stay separate call sites.

/// `abs(mmh3.hash(word, seed=0, signed=True))`: Python's hash is signed, so the high-bit case differs from the
/// raw unsigned output; `unsigned_abs` also avoids `i32::abs`'s panic on `i32::MIN`.
pub(crate) fn murmur_abs_hash(word: &str) -> u32 {
  let raw = murmur3::murmur3_32(&mut std::io::Cursor::new(word.as_bytes()), 0)
    .expect("reading from an in-memory byte slice cannot fail");
  (raw as i32).unsigned_abs()
}

/// `compute_token_id`: `abs(mmh3.hash(token))`. Callers stem first; this never stems.
pub(crate) fn bm25_hash(word: &str) -> u32 {
  murmur_abs_hash(word)
}

#[cfg(test)]
mod tests {
  use super::*;

  /// `mmh3.hash("fruit", seed=0, signed=True)` is the positive 87364033, so this case alone cannot catch a missing signed reinterpretation.
  #[test]
  fn murmur_abs_hash_of_a_positive_raw_value_equals_the_raw_value() {
    assert_eq!(murmur_abs_hash("fruit"), 87_364_033);
  }

  /// `mmh3.hash("bat", seed=0, signed=True) == -1528583113`, so the answer is 1528583113, not the unsigned 2766384183.
  #[test]
  fn murmur_abs_hash_of_a_negative_raw_value_is_not_the_raw_unsigned_value() {
    let hash = murmur_abs_hash("bat");
    assert_eq!(hash, 1_528_583_113);
    assert_ne!(hash, 2_766_384_183);
  }

  #[test]
  fn bm25_hash_does_not_stem_its_own_input() {
    assert_ne!(bm25_hash("baseball"), 2_068_025_269);
    assert_eq!(bm25_hash("basebal"), 2_068_025_269);
  }
}