candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

//! `Qdrant/bm25`'s entry point beside `SparseTextEmbedding`: no encoder, device, dtype or weights, so `build()` and
//! the embed calls return values rather than a `Result` and take no batch size. English only today.

use crate::models::sparse::bm25_embed_document;
use crate::models::sparse::bm25_embed_query;
use crate::models::sparse::stopwords::Stopwords;
use crate::registry;
use crate::registry::sparse::SparseDescriptor;
use crate::sparse::SparseEmbedding;

/// The published English stopword list `Qdrant/bm25` ships as `english.txt`, compiled in.
const ENGLISH_STOPWORDS: &str = include_str!("../models/sparse/english_stopwords.txt");

/// `Qdrant/bm25`: sparse rows computed from stemmed, hashed text with a stopword list compiled into the crate, and
/// no model, device or dtype.
pub struct Bm25TextEmbedding {
  stopwords: Stopwords,
  descriptor: SparseDescriptor,
}

impl Bm25TextEmbedding {
  pub fn builder() -> Bm25TextEmbeddingBuilder {
    Bm25TextEmbeddingBuilder::new()
  }

  pub fn descriptor(&self) -> &SparseDescriptor {
    &self.descriptor
  }

  /// The passage side, the same rows as `passage_embed`.
  pub fn embed<S: AsRef<str>>(&self, texts: &[S]) -> Vec<SparseEmbedding> {
    self.passage_embed(texts)
  }

  /// The passage side: `Bm25.raw_embed`'s own term-frequency saturation, one row per text.
  pub fn passage_embed<S: AsRef<str>>(&self, texts: &[S]) -> Vec<SparseEmbedding> {
    texts
      .iter()
      .map(|text| bm25_embed_document(text.as_ref(), &self.stopwords))
      .collect()
  }

  /// The query side: `Bm25.query_embed`'s own uniform weighting, one row per text.
  pub fn query_embed<S: AsRef<str>>(&self, texts: &[S]) -> Vec<SparseEmbedding> {
    texts
      .iter()
      .map(|text| bm25_embed_query(text.as_ref(), &self.stopwords))
      .collect()
  }
}

/// No options today; a builder anyway, so a later one, such as a language, leaves
/// `Bm25TextEmbedding::builder().build()` unchanged.
pub struct Bm25TextEmbeddingBuilder {
  _private: (),
}

impl Bm25TextEmbeddingBuilder {
  pub(crate) fn new() -> Self {
    Self { _private: () }
  }

  /// Cannot fail: everything the embedder needs is compiled into the crate.
  pub fn build(self) -> Bm25TextEmbedding {
    Bm25TextEmbedding {
      stopwords: Stopwords::parse(ENGLISH_STOPWORDS),
      descriptor: registry::sparse::lookup("Qdrant/bm25")
        .expect("Qdrant/bm25 is a registered sparse entry"),
    }
  }
}

#[cfg(test)]
mod tests;