paper/core/models/
embedder.rs1use super::specs::ModelSpec;
2use anyhow::{bail, Context, Result};
3use candle_core::{DType, Device, Tensor};
4use candle_nn::VarBuilder;
5use candle_transformers::models::bert::{BertModel, Config as BertConfig};
6use std::path::Path;
7use tokenizers::Tokenizer;
8
9pub struct Embedder {
12 model: BertModel,
13 tokenizer: Tokenizer,
14 device: Device,
15 spec: ModelSpec,
16}
17
18impl Embedder {
19 pub fn new(model_dir: &Path, spec: &ModelSpec) -> Result<Self> {
22 let config_path = model_dir.join("config.json");
23 let config_text = std::fs::read_to_string(&config_path)
24 .with_context(|| format!("reading {}", config_path.display()))?;
25 let raw: serde_json::Value = serde_json::from_str(&config_text)?;
26 let model_type = raw.get("model_type").and_then(|v| v.as_str()).unwrap_or("");
27 if model_type != "bert" {
28 bail!(
30 "unsupported encoder architecture {model_type:?} for model {}",
31 spec.id
32 );
33 }
34 let config: BertConfig = serde_json::from_value(raw).context("parsing BERT config")?;
35 if config.hidden_size != spec.dims {
36 bail!(
37 "config hidden_size {} does not match spec dims {} for {}",
38 config.hidden_size,
39 spec.dims,
40 spec.id
41 );
42 }
43
44 let device = Device::Cpu;
45 let tokenizer_path = model_dir.join("tokenizer.json");
46 let mut tokenizer = Tokenizer::from_file(&tokenizer_path)
47 .map_err(|e| anyhow::anyhow!("loading {}: {e}", tokenizer_path.display()))?;
48 tokenizer
49 .with_padding(None)
50 .with_truncation(Some(tokenizers::TruncationParams {
51 max_length: spec.max_tokens,
52 ..Default::default()
53 }))
54 .map_err(|e| anyhow::anyhow!("configuring truncation: {e}"))?;
55
56 let weights = model_dir.join("model.safetensors");
57 let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[&weights], DType::F32, &device)? };
58 let model = BertModel::load(vb, &config).context("loading encoder weights")?;
59
60 Ok(Self {
61 model,
62 tokenizer,
63 device,
64 spec: spec.clone(),
65 })
66 }
67
68 pub fn dims(&self) -> usize {
69 self.spec.dims
70 }
71
72 pub fn model_id(&self) -> &str {
73 self.spec.id
74 }
75
76 pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
78 self.embed_with_prefix(self.spec.query_prefix, text)
79 }
80
81 pub fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
83 self.embed_with_prefix(self.spec.document_prefix, text)
84 }
85
86 pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
88 self.embed_document(text)
89 }
90
91 fn embed_with_prefix(&self, prefix: &str, text: &str) -> Result<Vec<f32>> {
92 let input = if prefix.is_empty() {
93 text.to_string()
94 } else {
95 format!("{prefix}{text}")
96 };
97 let encoding = self
98 .tokenizer
99 .encode(input, true)
100 .map_err(|e| anyhow::anyhow!("tokenizing input: {e}"))?;
101
102 let input_ids = Tensor::new(encoding.get_ids(), &self.device)?.unsqueeze(0)?;
103 let token_type_ids = Tensor::new(encoding.get_type_ids(), &self.device)?.unsqueeze(0)?;
104 let attention_mask =
105 Tensor::new(encoding.get_attention_mask(), &self.device)?.unsqueeze(0)?;
106
107 let hidden = self
108 .model
109 .forward(&input_ids, &token_type_ids, Some(&attention_mask))?;
110
111 let mask = attention_mask.to_dtype(DType::F32)?.unsqueeze(2)?;
113 let summed = hidden.broadcast_mul(&mask)?.sum(1)?;
114 let counts = mask.sum(1)?.clamp(1e-9, f64::INFINITY)?;
115 let pooled = summed.broadcast_div(&counts)?;
116
117 let norm = pooled
119 .sqr()?
120 .sum_keepdim(1)?
121 .sqrt()?
122 .clamp(1e-12, f64::INFINITY)?;
123 let normalized = pooled.broadcast_div(&norm)?;
124 Ok(normalized.flatten_all()?.to_vec1::<f32>()?)
125 }
126}