candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

use serde::Deserialize;

use crate::layers::activation::Activation;

#[derive(Clone, Debug, Deserialize)]
pub struct BertConfig {
  pub vocab_size: usize,
  pub hidden_size: usize,
  pub num_hidden_layers: usize,
  pub num_attention_heads: usize,
  pub intermediate_size: usize,
  #[serde(default = "default_act")]
  pub hidden_act: Activation,
  pub max_position_embeddings: usize,
  #[serde(default = "default_type_vocab_size")]
  pub type_vocab_size: usize,
  #[serde(default = "default_eps")]
  pub layer_norm_eps: f64,
}

fn default_act() -> Activation {
  Activation::Gelu
}

fn default_type_vocab_size() -> usize {
  2
}

fn default_eps() -> f64 {
  1e-12
}

impl BertConfig {
  pub fn head_dim(&self) -> usize {
    self.hidden_size / self.num_attention_heads
  }
}