candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

//! Which length a `CanddingError::LengthTooSmall` refuses, and its message.

use crate::tokenizer::PromptTemplate;

/// The length `CanddingError::LengthTooSmall` refused, with the fixed tokens it had to leave room beside.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum LengthKind {
  /// A `max_length` for one text, whose post-processor adds `added` special tokens.
  Text { added: usize },
  /// A `max_length` for a (query, document) pair, whose post-processor adds `added` special tokens.
  Pair { added: usize },
  /// A `max_length` for a prompt of `template`, whose fixed prefix and suffix encode to `prefix` and `suffix` ids.
  Prompt {
    template: PromptTemplate,
    prefix: usize,
    suffix: usize,
  },
  /// A `max_length` for a windowed input whose query is cut to `query_cap` ids.
  Windowed { query_cap: usize },
  /// A windowed input's query cap, which has to fit the `added` special tokens a query gets and one real token.
  QueryCap { added: usize },
}

/// The message names the minimum and what the fixed tokens take, so the remedy is in it.
pub(super) fn message(requested: &usize, minimum: &usize, kind: &LengthKind) -> String {
  let below = format!("{requested} is below the minimum {minimum}");
  match kind {
    LengthKind::Text { added } => format!(
      "max_length {below} for this tokenizer: its post-processor adds {added} special tokens to every text, and one real token has to fit beside them"
    ),
    LengthKind::Pair { added } => format!(
      "max_length {below} for a (query, document) pair with this tokenizer: its post-processor adds {added} special tokens to every pair, and one real token has to fit beside them"
    ),
    LengthKind::Prompt {
      template,
      prefix,
      suffix,
    } => format!(
      "max_length {below} for the {} prompt template with this tokenizer: its fixed prefix is {prefix} tokens and its suffix {suffix}, and one content token has to fit between them",
      template.name()
    ),
    LengthKind::Windowed { query_cap } => format!(
      "max_length {below} for a windowed input with query cap {query_cap}: it needs twice the cap, which the reference's own rerank asserts, and room for a document token beside the capped query and two separators"
    ),
    LengthKind::QueryCap { added } => format!(
      "query cap {below} for a windowed input with this tokenizer: its post-processor adds {added} special tokens to a query, and one real token has to fit beside them"
    ),
  }
}