candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

//! The catalogs beside the dense one, as the builders' refusals name them.

use super::entry_for;
use super::multi_vector;
use super::rerank;
use super::sparse;

/// A catalog beside the dense one: where an id the dense builder refuses is registered, or where a builder looked
/// an unregistered id up. Each names the builder that loads its entries.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum Catalog {
  /// `registry::sparse`, for a head that runs an encoder: `SparseTextEmbedding::builder`.
  Sparse,
  /// `registry::sparse`'s computed entry, `Qdrant/bm25`, which `Bm25TextEmbedding::builder` loads.
  Bm25,
  /// `registry::multi_vector`: `MultiVectorTextEmbedding::builder`.
  MultiVector,
  /// `registry::rerank`: `TextCrossEncoder::builder`.
  Rerank,
}

impl Catalog {
  pub(crate) fn name(self) -> &'static str {
    match self {
      Catalog::Sparse | Catalog::Bm25 => "sparse",
      Catalog::MultiVector => "multi-vector",
      Catalog::Rerank => "rerank",
    }
  }

  /// The builder that loads an entry of this catalog, and the command that runs it where the CLI has one.
  pub(crate) fn entry_point(self) -> &'static str {
    match self {
      Catalog::Sparse => "`SparseTextEmbedding::builder` (`candding embed --sparse`)",
      Catalog::Bm25 => "`Bm25TextEmbedding::builder`",
      Catalog::MultiVector => {
        "`MultiVectorTextEmbedding::builder` (`candding embed --multi-vector`)"
      }
      Catalog::Rerank => "`TextCrossEncoder::builder` (`candding rerank`)",
    }
  }

  /// What no file of an unregistered checkpoint states, so a caller's descriptor has to.
  pub(crate) fn unstated_facts(self) -> &'static str {
    match self {
      Catalog::Sparse | Catalog::Bm25 => "its head or vocabulary",
      Catalog::MultiVector => "its head or projection shape",
      Catalog::Rerank => "its head or score activation",
    }
  }
}

/// The catalog that registers `id` when the dense one does not, checked in the order sparse, multi-vector, rerank;
/// `None` for a dense id, which the dense builder serves, and for an id no catalog carries.
pub(crate) fn other_catalog(id: &str) -> Option<Catalog> {
  if entry_for(id).is_some() {
    return None;
  }
  if let Some(descriptor) = sparse::lookup(id) {
    return Some(match descriptor.computation {
      sparse::SparseComputation::Computed => Catalog::Bm25,
      sparse::SparseComputation::Encoded => Catalog::Sparse,
    });
  }
  if multi_vector::lookup(id).is_some() {
    return Some(Catalog::MultiVector);
  }
  rerank::lookup(id).map(|_| Catalog::Rerank)
}