Skip to main content

graphrt_rag/
hash.rs

1//! A deterministic, dependency-free embedder using feature hashing.
2//!
3//! Each token (and each pair of adjacent tokens) is hashed into one of `dims`
4//! buckets with a ± sign. The result captures lexical overlap only — no
5//! semantics — but it is fully offline and reproducible, which makes it useful
6//! for tests, demos and keyword-like retrieval without a model server.
7
8use futures::future::BoxFuture;
9
10use crate::error::RagError;
11use crate::memory::tokenize;
12use crate::types::Embedder;
13
14/// Feature-hashing embedder.
15#[derive(Clone, Debug)]
16pub struct HashEmbedder {
17    dims: usize,
18    model: String,
19}
20
21impl HashEmbedder {
22    /// Create an embedder producing `dims`-dimensional vectors (minimum 8).
23    pub fn new(dims: usize) -> Self {
24        let dims = dims.max(8);
25        Self { dims, model: format!("hash-{dims}") }
26    }
27
28    /// Embed one text synchronously.
29    pub fn embed_text(&self, text: &str) -> Vec<f32> {
30        let mut v = vec![0.0f32; self.dims];
31        let tokens = tokenize(text);
32        let mut add = |feature: &str, weight: f32| {
33            let h = fnv(feature);
34            let bucket = usize::try_from(h % u64::try_from(self.dims).unwrap_or(1)).unwrap_or(0);
35            let sign = if (h >> 63) == 0 { 1.0 } else { -1.0 };
36            if let Some(slot) = v.get_mut(bucket) {
37                *slot += sign * weight;
38            }
39        };
40        for token in &tokens {
41            add(token, 1.0);
42        }
43        for pair in tokens.windows(2) {
44            if let [a, b] = pair {
45                add(&format!("{a} {b}"), 0.5);
46            }
47        }
48        v
49    }
50}
51
52fn fnv(s: &str) -> u64 {
53    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
54    for b in s.bytes() {
55        h ^= u64::from(b);
56        h = h.wrapping_mul(0x0100_0000_01b3);
57    }
58    h
59}
60
61impl Embedder for HashEmbedder {
62    fn model(&self) -> &str {
63        &self.model
64    }
65
66    fn dimensions(&self) -> Option<usize> {
67        Some(self.dims)
68    }
69
70    fn embed_documents<'a>(&'a self, texts: &'a [String]) -> BoxFuture<'a, Result<Vec<Vec<f32>>, RagError>> {
71        let vectors = texts.iter().map(|t| self.embed_text(t)).collect();
72        Box::pin(futures::future::ready(Ok(vectors)))
73    }
74}
75
76#[cfg(test)]
77mod tests {
78    use super::*;
79    use crate::memory::cosine;
80
81    #[test]
82    fn similar_texts_score_higher() {
83        let e = HashEmbedder::new(256);
84        let q = e.embed_text("how do cats purr");
85        let near = e.embed_text("cats purr when happy");
86        let far = e.embed_text("rocket engines burn fuel");
87        assert!(cosine(&q, &near) > cosine(&q, &far));
88        assert_eq!(e.embed_text("x"), e.embed_text("x"));
89    }
90}