Skip to main content

paper/core/models/
embedder.rs

1use super::specs::ModelSpec;
2use anyhow::{bail, Context, Result};
3use candle_core::{DType, Device, Tensor};
4use candle_nn::VarBuilder;
5use candle_transformers::models::bert::{BertModel, Config as BertConfig};
6use std::path::Path;
7use tokenizers::Tokenizer;
8
9/// A local sentence embedder running on Hugging Face Candle (pure Rust, no
10/// ONNX Runtime, no Python, no network at inference time).
11pub struct Embedder {
12    model: BertModel,
13    tokenizer: Tokenizer,
14    device: Device,
15    spec: ModelSpec,
16}
17
18impl Embedder {
19    /// Load from a verified model directory (`config.json`, `tokenizer.json`,
20    /// `model.safetensors`).
21    pub fn new(model_dir: &Path, spec: &ModelSpec) -> Result<Self> {
22        let config_path = model_dir.join("config.json");
23        let config_text = std::fs::read_to_string(&config_path)
24            .with_context(|| format!("reading {}", config_path.display()))?;
25        let raw: serde_json::Value = serde_json::from_str(&config_text)?;
26        let model_type = raw.get("model_type").and_then(|v| v.as_str()).unwrap_or("");
27        if model_type != "bert" {
28            // Fail loudly rather than returning plausible-looking garbage vectors.
29            bail!(
30                "unsupported encoder architecture {model_type:?} for model {}",
31                spec.id
32            );
33        }
34        let config: BertConfig = serde_json::from_value(raw).context("parsing BERT config")?;
35        if config.hidden_size != spec.dims {
36            bail!(
37                "config hidden_size {} does not match spec dims {} for {}",
38                config.hidden_size,
39                spec.dims,
40                spec.id
41            );
42        }
43
44        let device = Device::Cpu;
45        let tokenizer_path = model_dir.join("tokenizer.json");
46        let mut tokenizer = Tokenizer::from_file(&tokenizer_path)
47            .map_err(|e| anyhow::anyhow!("loading {}: {e}", tokenizer_path.display()))?;
48        tokenizer
49            .with_padding(None)
50            .with_truncation(Some(tokenizers::TruncationParams {
51                max_length: spec.max_tokens,
52                ..Default::default()
53            }))
54            .map_err(|e| anyhow::anyhow!("configuring truncation: {e}"))?;
55
56        let weights = model_dir.join("model.safetensors");
57        let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[&weights], DType::F32, &device)? };
58        let model = BertModel::load(vb, &config).context("loading encoder weights")?;
59
60        Ok(Self {
61            model,
62            tokenizer,
63            device,
64            spec: spec.clone(),
65        })
66    }
67
68    pub fn dims(&self) -> usize {
69        self.spec.dims
70    }
71
72    pub fn model_id(&self) -> &str {
73        self.spec.id
74    }
75
76    /// Embed a search query, applying the model's retrieval instruction.
77    pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
78        self.embed_with_prefix(self.spec.query_prefix, text)
79    }
80
81    /// Embed a document chunk.
82    pub fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
83        self.embed_with_prefix(self.spec.document_prefix, text)
84    }
85
86    /// Backwards-compatible alias for document embeddings.
87    pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
88        self.embed_document(text)
89    }
90
91    fn embed_with_prefix(&self, prefix: &str, text: &str) -> Result<Vec<f32>> {
92        let input = if prefix.is_empty() {
93            text.to_string()
94        } else {
95            format!("{prefix}{text}")
96        };
97        let encoding = self
98            .tokenizer
99            .encode(input, true)
100            .map_err(|e| anyhow::anyhow!("tokenizing input: {e}"))?;
101
102        let input_ids = Tensor::new(encoding.get_ids(), &self.device)?.unsqueeze(0)?;
103        let token_type_ids = Tensor::new(encoding.get_type_ids(), &self.device)?.unsqueeze(0)?;
104        let attention_mask =
105            Tensor::new(encoding.get_attention_mask(), &self.device)?.unsqueeze(0)?;
106
107        let hidden = self
108            .model
109            .forward(&input_ids, &token_type_ids, Some(&attention_mask))?;
110
111        // Mean pooling over real tokens only (never over padding).
112        let mask = attention_mask.to_dtype(DType::F32)?.unsqueeze(2)?;
113        let summed = hidden.broadcast_mul(&mask)?.sum(1)?;
114        let counts = mask.sum(1)?.clamp(1e-9, f64::INFINITY)?;
115        let pooled = summed.broadcast_div(&counts)?;
116
117        // L2-normalised, so sqlite-vec's cosine distance is `1 - dot`.
118        let norm = pooled
119            .sqr()?
120            .sum_keepdim(1)?
121            .sqrt()?
122            .clamp(1e-12, f64::INFINITY)?;
123        let normalized = pooled.broadcast_div(&norm)?;
124        Ok(normalized.flatten_all()?.to_vec1::<f32>()?)
125    }
126}