candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

//! The two dtype decisions every builder makes before loading a weight: resolve, then refuse.

use candle_core::DType;
use candle_core::Device;

use crate::error::CanddingError;
use crate::error::Result;
use crate::registry::Backend;
use crate::registry::BackendSupport;
use crate::registry::DtypeBackendSupport;
use crate::registry::Support;

/// Refuses a dtype where the device's own cell reads `Unsupported`: everywhere for EmbeddingGemma's f16, which
/// overflows to NaN, on the CPU alone where candle's f16 matmul fails what Metal passes.
pub(crate) fn reject_unsupported_dtype(
  model: &str,
  dtype: DType,
  dtype_backends: DtypeBackendSupport,
  device: &Device,
) -> Result<()> {
  refuse_on(model, dtype, dtype_backends.row(dtype), Backend::of(device))
}

/// One backend's decision: `UnsupportedDtype` when every cell refuses, `UnsupportedOnDevice` when this one does;
/// split from the device so the default suite checks every backend.
fn refuse_on(model: &str, dtype: DType, row: BackendSupport, backend: Backend) -> Result<()> {
  let unsupported = |support: Support| support == Support::Unsupported;
  if unsupported(row.cpu) && unsupported(row.metal) && unsupported(row.cuda) {
    return Err(CanddingError::UnsupportedDtype {
      model: model.to_string(),
      dtype,
    });
  }
  if unsupported(row.get(backend)) {
    return Err(CanddingError::UnsupportedOnDevice {
      model: model.to_string(),
      backend,
      dtype,
    });
  }
  Ok(())
}

/// BF16 has no CPU matmul in candle: an explicit request fails, a descriptor default falls back to F32.
pub(crate) fn resolve_dtype(
  model: &str,
  requested: Option<DType>,
  default: DType,
  device: &Device,
) -> Result<DType> {
  let dtype = match requested {
    Some(DType::BF16) if device.is_cpu() => {
      return Err(CanddingError::UnsupportedOnDevice {
        model: model.to_string(),
        backend: Backend::Cpu,
        dtype: DType::BF16,
      });
    }
    Some(dtype) => dtype,
    None if default == DType::BF16 && device.is_cpu() => {
      tracing::warn!(
        model,
        "default dtype bf16 has no cpu matmul in candle; loading f32 instead"
      );
      DType::F32
    }
    None => default,
  };
  if dtype == DType::F16 && device.is_cpu() {
    tracing::warn!(
      model,
      "f16 on cpu runs through candle's software f16 matmul, which accumulates in f16: slow and less precise than f32"
    );
  }
  Ok(dtype)
}

#[cfg(test)]
mod tests;