use serde::{Deserialize, Serialize};
pub trait EmbeddingEngine: Send + Sync {
fn dimensions(&self) -> usize;
fn embed_text(&self, text: &str) -> Vec<f32>;
fn embed_batch(&self, texts: &[&str]) -> Vec<Vec<f32>> {
texts.iter().map(|t| self.embed_text(t)).collect()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeterministicHashEmbedder {
pub dims: usize,
}
impl Default for DeterministicHashEmbedder {
fn default() -> Self {
Self { dims: 128 }
}
}
impl DeterministicHashEmbedder {
pub fn new(dims: usize) -> Self {
Self { dims: dims.max(16) }
}
fn hash_ngram(bytes: &[u8], seed: u32) -> u32 {
let mut h = seed;
for &b in bytes {
h = h.wrapping_mul(31).wrapping_add(b as u32);
h = (h << 13) | (h >> 19);
h ^= h >> 15;
}
h
}
}
impl EmbeddingEngine for DeterministicHashEmbedder {
fn dimensions(&self) -> usize {
self.dims
}
fn embed_text(&self, text: &str) -> Vec<f32> {
let mut vec = vec![0.0f32; self.dims];
let lower = text.to_lowercase();
let words: Vec<&str> = lower
.split(|c: char| !c.is_alphanumeric())
.filter(|w| !w.is_empty())
.collect();
if words.is_empty() {
return vec;
}
for (pos, &w) in words.iter().enumerate() {
let h1 = Self::hash_ngram(w.as_bytes(), 0x9747b28c);
let idx = (h1 as usize) % self.dims;
let sign = if (h1 & 1) == 0 { 1.0f32 } else { -1.0f32 };
let pos_decay = 1.0 / (1.0 + (pos as f32) * 0.05);
vec[idx] += sign * 1.5 * pos_decay;
}
for window in words.windows(2) {
let mut combined = String::with_capacity(window[0].len() + window[1].len() + 1);
combined.push_str(window[0]);
combined.push('_');
combined.push_str(window[1]);
let h2 = Self::hash_ngram(combined.as_bytes(), 0x5bd1e995);
let idx = (h2 as usize) % self.dims;
let sign = if (h2 & 1) == 0 { 1.0f32 } else { -1.0f32 };
vec[idx] += sign * 2.0;
}
for window in words.windows(3) {
let mut combined = String::with_capacity(window[0].len() + window[1].len() + window[2].len() + 2);
combined.push_str(window[0]);
combined.push('_');
combined.push_str(window[1]);
combined.push('_');
combined.push_str(window[2]);
let h3 = Self::hash_ngram(combined.as_bytes(), 0x27d4eb2f);
let idx = (h3 as usize) % self.dims;
let sign = if (h3 & 1) == 0 { 1.0f32 } else { -1.0f32 };
vec[idx] += sign * 2.5;
}
for w in words.iter().take(30) {
let chars: Vec<char> = w.chars().collect();
for chunk in chars.windows(3) {
let s: String = chunk.iter().collect();
let h = Self::hash_ngram(s.as_bytes(), 0x12b9b0a1);
let idx = (h as usize) % self.dims;
let sign = if (h & 1) == 0 { 1.0f32 } else { -1.0f32 };
vec[idx] += sign * 0.8;
}
for chunk in chars.windows(4) {
let s: String = chunk.iter().collect();
let h = Self::hash_ngram(s.as_bytes(), 0x3c6ef372);
let idx = (h as usize) % self.dims;
let sign = if (h & 1) == 0 { 1.0f32 } else { -1.0f32 };
vec[idx] += sign * 1.0;
}
}
let norm: f32 = vec.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 1e-9 {
for v in vec.iter_mut() {
*v /= norm;
}
}
vec
}
}
use std::io::{Read, Write};
use std::net::TcpStream;
use std::time::Duration;
#[derive(Debug, Clone)]
pub struct HttpEmbeddingEngine {
pub host: String,
pub path: String,
pub model: String,
pub dims: usize,
pub api_key: Option<String>,
pub fallback: DeterministicHashEmbedder,
}
impl HttpEmbeddingEngine {
pub fn new(endpoint: &str, model: impl Into<String>, dims: usize) -> Self {
let trimmed = endpoint.trim_start_matches("http://").trim_start_matches("https://");
let (host, path) = match trimmed.split_once('/') {
Some((h, p)) => (h.to_string(), format!("/{p}")),
None => (trimmed.to_string(), "/".to_string()),
};
Self::new_custom(host, path, model, dims)
}
pub fn new_ollama(model: impl Into<String>, dims: usize) -> Self {
Self {
host: "127.0.0.1:11434".to_string(),
path: "/api/embeddings".to_string(),
model: model.into(),
dims,
api_key: None,
fallback: DeterministicHashEmbedder::new(dims),
}
}
pub fn new_openai(host: impl Into<String>, api_key: impl Into<String>, model: impl Into<String>, dims: usize) -> Self {
Self {
host: host.into(),
path: "/v1/embeddings".to_string(),
model: model.into(),
dims,
api_key: Some(api_key.into()),
fallback: DeterministicHashEmbedder::new(dims),
}
}
pub fn new_custom(host: impl Into<String>, path: impl Into<String>, model: impl Into<String>, dims: usize) -> Self {
Self {
host: host.into(),
path: path.into(),
model: model.into(),
dims,
api_key: None,
fallback: DeterministicHashEmbedder::new(dims),
}
}
fn query_remote(&self, text: &str) -> Option<Vec<f32>> {
let stream = TcpStream::connect_timeout(
&self.host.parse().ok()?,
Duration::from_millis(500),
).ok()?;
let _ = stream.set_read_timeout(Some(Duration::from_secs(3)));
let _ = stream.set_write_timeout(Some(Duration::from_secs(3)));
let mut stream = stream;
let body = serde_json::json!({
"model": self.model,
"prompt": text,
"input": text,
}).to_string();
let auth_hdr = match &self.api_key {
Some(key) => format!("Authorization: Bearer {key}\r\n"),
None => String::new(),
};
let req = format!(
"POST {} HTTP/1.1\r\nHost: {}\r\n{}Content-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
self.path, self.host, auth_hdr, body.len(), body
);
stream.write_all(req.as_bytes()).ok()?;
let mut response = Vec::new();
stream.read_to_end(&mut response).ok()?;
let resp_str = String::from_utf8_lossy(&response);
let json_start = resp_str.find("\r\n\r\n")? + 4;
let json_body = &resp_str[json_start..];
let parsed: serde_json::Value = serde_json::from_str(json_body).ok()?;
if let Some(arr) = parsed.get("embedding").and_then(|v| v.as_array()) {
let vec: Vec<f32> = arr.iter().filter_map(|v| v.as_f64().map(|f| f as f32)).collect();
if !vec.is_empty() {
return Some(vec);
}
}
if let Some(arr) = parsed
.get("data")
.and_then(|v| v.as_array())
.and_then(|data| data.first())
.and_then(|first| first.get("embedding"))
.and_then(|v| v.as_array())
{
let vec: Vec<f32> = arr.iter().filter_map(|v| v.as_f64().map(|f| f as f32)).collect();
if !vec.is_empty() {
return Some(vec);
}
}
None
}
}
impl EmbeddingEngine for HttpEmbeddingEngine {
fn dimensions(&self) -> usize {
self.dims
}
fn embed_text(&self, text: &str) -> Vec<f32> {
if let Some(vec) = self.query_remote(text) {
vec
} else {
self.fallback.embed_text(text)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_deterministic_hash_embedder_consistency() {
let embedder = DeterministicHashEmbedder::new(128);
assert_eq!(embedder.dimensions(), 128);
let v1 = embedder.embed_text("TapirusDB embedded database");
let v2 = embedder.embed_text("TapirusDB embedded database");
assert_eq!(v1, v2);
let norm: f32 = v1.iter().map(|x| x * x).sum::<f32>().sqrt();
assert!((norm - 1.0).abs() < 1e-5);
}
#[test]
fn test_semantic_similarity_direction() {
let embedder = DeterministicHashEmbedder::new(128);
let v_tech1 = embedder.embed_text("Rust embedded database engine");
let v_tech2 = embedder.embed_text("Rust embedded database persistence");
let v_fruit = embedder.embed_text("Fresh organic bananas and apples");
let dot_tech: f32 = v_tech1.iter().zip(v_tech2.iter()).map(|(a, b)| a * b).sum();
let dot_fruit: f32 = v_tech1.iter().zip(v_fruit.iter()).map(|(a, b)| a * b).sum();
assert!(dot_tech > dot_fruit);
}
}