candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

use candle_core::DType;

use super::DtypeSupport;
use super::Family;
use super::Support;
use super::WeightsFormat;
use super::entry::Entry;
use super::t4::T4_BF16_UNTESTED;
use super::t4::T4_F32_AND_BF16_UNTESTED;
use crate::pooling::Pooling;

/// The official prompt: no space after `Query:`, as in the model card and `config_sentence_transformers.json`.
const QUERY: &str = "Instruct: {instruction}\nQuery:{text}";
const INSTRUCTION: &str =
  "Given a web search query, retrieve relevant passages that answer the query";

/// All three dtypes proven by a completed golden run.
const VERIFIED: DtypeSupport = DtypeSupport::verified();

const fn qwen3(
  id: &'static str,
  dim: usize,
  size_gb: f32,
  default_dtype: DType,
  dtypes: DtypeSupport,
  cpu: Support,
  metal: Support,
) -> Entry {
  Entry {
    id,
    family: Family::Qwen3,
    dim,
    default_max_length: 8192,
    pooling: Pooling::LastToken,
    cpu,
    metal,
    cuda: Support::Untested,
    query_template: Some(QUERY),
    passage_template: None,
    default_instruction: Some(INSTRUCTION),
    dense_dirs: &[],
    mrl: true,
    layer_norm_after_pooling: false,
    gated: false,
    default_dtype,
    dtypes,
    license: "Apache-2.0",
    size_gb,
    phase: 1,
    weights: WeightsFormat::Safetensors,
    accel_only_dtypes: &[],
    cell_exceptions: &[],
  }
}

/// Half dtypes verified on Metal. F16 is refused on the CPU: candle's CPU f16 matmul accumulates in f16 and
/// misses the F16 floor (cosine 0.9971 against 0.998) where Metal passes.
const QWEN3_0_6B: Entry = Entry {
  cuda: Support::Verified,
  accel_only_dtypes: &[DType::F16],
  cell_exceptions: T4_BF16_UNTESTED,
  ..qwen3(
    "Qwen/Qwen3-Embedding-0.6B",
    1024,
    1.19,
    DType::F32,
    VERIFIED,
    Support::Verified,
    Support::Verified,
  )
};

/// F32 on the CPU passes at CPU tolerances (about two hours); all three dtypes verified on Metal. F16 is
/// refused on the CPU by analogy with the 0.6B, never run there.
const QWEN3_4B: Entry = Entry {
  cuda: Support::Verified,
  accel_only_dtypes: &[DType::F16],
  cell_exceptions: T4_F32_AND_BF16_UNTESTED,
  ..qwen3(
    "Qwen/Qwen3-Embedding-4B",
    2560,
    8.0,
    DType::BF16,
    VERIFIED,
    Support::Verified,
    Support::Verified,
  )
};

/// F32 weights are 32 GB and stay untested; F16 and BF16 on Metal match the bfloat16 reference.
const QWEN3_8B: Entry = qwen3(
  "Qwen/Qwen3-Embedding-8B",
  4096,
  15.1,
  DType::BF16,
  DtypeSupport {
    f32: Support::Untested,
    f16: Support::Verified,
    bf16: Support::Verified,
  },
  Support::Untested,
  Support::Verified,
);

pub(super) const ENTRIES: &[Entry] = &[QWEN3_0_6B, QWEN3_4B, QWEN3_8B];