docling_rag/embed/
hash.rs

1//! Deterministic, network-free embedder using the hashing trick.
2//!
3//! Not a real semantic model, but it produces stable vectors where texts that
4//! share tokens land near each other in cosine space — enough to wire up and
5//! test the whole pipeline (ingest → store → retrieve → eval) offline.
6
7use super::Embedder;
8use crate::math;
9use crate::Result;
10use async_trait::async_trait;
11
12/// Feature-hashing embedder of a fixed dimensionality.
13#[derive(Debug, Clone)]
14pub struct HashEmbedder {
15    dim: usize,
16    id: String,
17}
18
19impl HashEmbedder {
20    /// Create a hashing embedder producing `dim`-dimensional unit vectors.
21    pub fn new(dim: usize) -> Self {
22        HashEmbedder {
23            dim: dim.max(1),
24            id: format!("hash:{dim}"),
25        }
26    }
27
28    fn embed_text(&self, text: &str) -> Vec<f32> {
29        let mut v = vec![0.0f32; self.dim];
30        let tokens: Vec<&str> = text.split_whitespace().collect();
31        for w in &tokens {
32            self.add_feature(&mut v, w.to_ascii_lowercase().as_bytes());
33        }
34        // Bigrams add a little word-order signal.
35        for pair in tokens.windows(2) {
36            let bg = format!(
37                "{}_{}",
38                pair[0].to_ascii_lowercase(),
39                pair[1].to_ascii_lowercase()
40            );
41            self.add_feature(&mut v, bg.as_bytes());
42        }
43        math::normalize(&mut v);
44        v
45    }
46
47    fn add_feature(&self, v: &mut [f32], key: &[u8]) {
48        let h = fnv1a(key);
49        let idx = (h % self.dim as u64) as usize;
50        let sign = if (h >> 63) & 1 == 0 { 1.0 } else { -1.0 };
51        v[idx] += sign;
52    }
53}
54
55#[async_trait]
56impl Embedder for HashEmbedder {
57    async fn embed(&self, texts: &[String]) -> Result<Vec<Vec<f32>>> {
58        Ok(texts.iter().map(|t| self.embed_text(t)).collect())
59    }
60
61    fn dim(&self) -> usize {
62        self.dim
63    }
64
65    fn id(&self) -> &str {
66        &self.id
67    }
68}
69
70/// 64-bit FNV-1a — small, fast, dependency-free, and deterministic across runs.
71fn fnv1a(bytes: &[u8]) -> u64 {
72    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
73    for &b in bytes {
74        h ^= b as u64;
75        h = h.wrapping_mul(0x0000_0100_0000_01b3);
76    }
77    h
78}
79
80#[cfg(test)]
81mod tests {
82    use super::*;
83    use crate::math::cosine;
84
85    #[tokio::test]
86    async fn deterministic_and_unit_length() {
87        let e = HashEmbedder::new(256);
88        let a = e.embed_one("the quick brown fox").await.unwrap();
89        let b = e.embed_one("the quick brown fox").await.unwrap();
90        assert_eq!(a, b);
91        assert!((math::norm(&a) - 1.0).abs() < 1e-5);
92        assert_eq!(a.len(), 256);
93    }
94
95    #[tokio::test]
96    async fn shared_tokens_are_more_similar() {
97        let e = HashEmbedder::new(1024);
98        let q = e.embed_one("database vector search").await.unwrap();
99        let near = e.embed_one("vector search over a database").await.unwrap();
100        let far = e.embed_one("banana smoothie recipe").await.unwrap();
101        assert!(cosine(&q, &near) > cosine(&q, &far));
102    }
103}
docling_rag/embed/hash.rs

docling_rag/embed/
hash.rs
