candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

//! Activation functions named as in Hugging Face configs.

use candle_core::Module;
use candle_core::Result;
use candle_core::Tensor;
use serde::Deserialize;

#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Activation {
  /// Exact erf-based GELU, the BERT default.
  Gelu,
  GeluNew,
  GeluPytorchTanh,
  Relu,
  Silu,
}

impl Module for Activation {
  fn forward(&self, xs: &Tensor) -> Result<Tensor> {
    match self {
      Activation::Gelu => xs.gelu_erf(),
      Activation::GeluNew | Activation::GeluPytorchTanh => xs.gelu(),
      Activation::Relu => xs.relu(),
      Activation::Silu => xs.silu(),
    }
  }
}

#[cfg(test)]
mod tests {
  use candle_core::Device;

  use super::*;

  #[test]
  fn deserializes_hf_names() {
    assert_eq!(
      serde_json::from_str::<Activation>("\"gelu\"").unwrap(),
      Activation::Gelu
    );
    assert_eq!(
      serde_json::from_str::<Activation>("\"gelu_new\"").unwrap(),
      Activation::GeluNew
    );
    assert_eq!(
      serde_json::from_str::<Activation>("\"gelu_pytorch_tanh\"").unwrap(),
      Activation::GeluPytorchTanh
    );
  }

  #[test]
  fn erf_and_tanh_gelu_differ_slightly() {
    let xs = Tensor::new(&[1f32], &Device::Cpu).unwrap();
    let erf = Activation::Gelu
      .forward(&xs)
      .unwrap()
      .to_vec1::<f32>()
      .unwrap()[0];
    let tanh = Activation::GeluNew
      .forward(&xs)
      .unwrap()
      .to_vec1::<f32>()
      .unwrap()[0];
    assert!((erf - 0.841345).abs() < 1e-5, "{erf}");
    assert!((tanh - 0.841192).abs() < 1e-5, "{tanh}");
  }
}