1use futures::future::BoxFuture;
9
10use crate::error::RagError;
11use crate::memory::tokenize;
12use crate::types::Embedder;
13
14#[derive(Clone, Debug)]
16pub struct HashEmbedder {
17 dims: usize,
18 model: String,
19}
20
21impl HashEmbedder {
22 pub fn new(dims: usize) -> Self {
24 let dims = dims.max(8);
25 Self { dims, model: format!("hash-{dims}") }
26 }
27
28 pub fn embed_text(&self, text: &str) -> Vec<f32> {
30 let mut v = vec![0.0f32; self.dims];
31 let tokens = tokenize(text);
32 let mut add = |feature: &str, weight: f32| {
33 let h = fnv(feature);
34 let bucket = usize::try_from(h % u64::try_from(self.dims).unwrap_or(1)).unwrap_or(0);
35 let sign = if (h >> 63) == 0 { 1.0 } else { -1.0 };
36 if let Some(slot) = v.get_mut(bucket) {
37 *slot += sign * weight;
38 }
39 };
40 for token in &tokens {
41 add(token, 1.0);
42 }
43 for pair in tokens.windows(2) {
44 if let [a, b] = pair {
45 add(&format!("{a} {b}"), 0.5);
46 }
47 }
48 v
49 }
50}
51
52fn fnv(s: &str) -> u64 {
53 let mut h: u64 = 0xcbf2_9ce4_8422_2325;
54 for b in s.bytes() {
55 h ^= u64::from(b);
56 h = h.wrapping_mul(0x0100_0000_01b3);
57 }
58 h
59}
60
61impl Embedder for HashEmbedder {
62 fn model(&self) -> &str {
63 &self.model
64 }
65
66 fn dimensions(&self) -> Option<usize> {
67 Some(self.dims)
68 }
69
70 fn embed_documents<'a>(&'a self, texts: &'a [String]) -> BoxFuture<'a, Result<Vec<Vec<f32>>, RagError>> {
71 let vectors = texts.iter().map(|t| self.embed_text(t)).collect();
72 Box::pin(futures::future::ready(Ok(vectors)))
73 }
74}
75
76#[cfg(test)]
77mod tests {
78 use super::*;
79 use crate::memory::cosine;
80
81 #[test]
82 fn similar_texts_score_higher() {
83 let e = HashEmbedder::new(256);
84 let q = e.embed_text("how do cats purr");
85 let near = e.embed_text("cats purr when happy");
86 let far = e.embed_text("rocket engines burn fuel");
87 assert!(cosine(&q, &near) > cosine(&q, &far));
88 assert_eq!(e.embed_text("x"), e.embed_text("x"));
89 }
90}