candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

//! `tests/fixtures/prompt_truncation.json` is written by `scripts/prompt_fixture.py` from sentence-transformers' `CrossEncoder`.

use std::fs;
use std::path::PathBuf;

use serde::Deserialize;

use super::prompt::PAD;
use super::prompt::qwen3;
use super::prompt::subset_path;
use crate::error::CanddingError;
use crate::error::LengthKind;
use crate::tokenizer::PadToken;
use crate::tokenizer::PromptTemplate;
use crate::tokenizer::PromptTokenizer;

/// The smallest budget with one content token: 39 prefix ids, 9 suffix ids and one more.
const MINIMUM: usize = 49;

#[derive(Deserialize)]
struct Fixture {
  prefix_ids: Vec<u32>,
  suffix_ids: Vec<u32>,
  rows: Vec<Row>,
  cases: Vec<Case>,
}

#[derive(Deserialize)]
struct Row {
  instruction: String,
  query: String,
  document: String,
  ids: Vec<u32>,
}

#[derive(Deserialize)]
struct Case {
  row: usize,
  max_length: usize,
  reference: String,
  ids: Vec<u32>,
}

fn fixture() -> Fixture {
  let path =
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/prompt_truncation.json");
  serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap()
}

#[test]
fn every_row_encodes_to_the_published_files_ids() {
  let fixture = fixture();
  let tokenizer = qwen3(&subset_path(), 8192);
  assert_eq!(tokenizer.prefix_len(), fixture.prefix_ids.len());
  assert_eq!(tokenizer.suffix_ids(), fixture.suffix_ids.as_slice());
  for (index, row) in fixture.rows.iter().enumerate() {
    let ids = tokenizer
      .prompt_ids(&row.instruction, &row.query, &row.document)
      .unwrap();
    assert_eq!(ids, row.ids, "row {index}");
  }
}

/// Below the minimum the reference still keeps the prefix and the suffix; candding refuses instead.
#[test]
fn every_case_keeps_the_references_ids_token_for_token() {
  let fixture = fixture();
  for case in &fixture.cases {
    let row = &fixture.rows[case.row];
    let label = format!(
      "row {} max_length {} ({})",
      case.row, case.max_length, case.reference
    );
    if case.max_length < MINIMUM {
      let refused = PromptTokenizer::from_file(
        &subset_path(),
        case.max_length,
        PadToken::Id(PAD),
        PromptTemplate::Qwen3Reranker,
      );
      assert!(
        matches!(
          refused,
          Err(CanddingError::LengthTooSmall {
            minimum: MINIMUM,
            kind: LengthKind::Prompt { .. },
            ..
          })
        ),
        "{label}"
      );
      continue;
    }
    let tokenizer = qwen3(&subset_path(), case.max_length);
    let ids = tokenizer
      .prompt_ids(&row.instruction, &row.query, &row.document)
      .unwrap();
    assert_eq!(ids, case.ids, "{label}");
    let pair = [(row.query.as_str(), row.document.as_str())];
    assert_eq!(
      tokenizer.count(&row.instruction, &pair).unwrap(),
      vec![case.ids.len()],
      "{label}"
    );
  }
}

#[test]
fn the_fixture_covers_the_minimum_a_refusal_and_the_edges_of_the_content() {
  let fixture = fixture();
  let raw = tokenizers::Tokenizer::from_file(subset_path()).unwrap();
  let prefix = PromptTemplate::Qwen3Reranker.prefix();
  let head_ids = |row: &Row| -> Vec<u32> {
    let head = format!(
      "{prefix}<Instruct>: {}\n<Query>: {}\n<Document>:",
      row.instruction, row.query
    );
    let encoding = raw.encode(head.as_str(), false).unwrap();
    [encoding.get_ids(), &fixture.suffix_ids].concat()
  };
  let has = |check: &dyn Fn(&Case) -> bool| fixture.cases.iter().any(check);
  assert!(has(&|c| c.max_length == MINIMUM));
  assert!(has(&|c| c.max_length < MINIMUM));
  assert!(has(&|c| {
    let row = &fixture.rows[c.row];
    !row.document.is_empty() && c.ids == head_ids(row)
  }));
  assert!(has(&|c| {
    let row = &fixture.rows[c.row];
    c.ids.len() == c.max_length && head_ids(row).len() > c.max_length && c.max_length > 64
  }));
  assert!(has(
    &|c| fixture.rows[c.row].instruction.is_empty() && c.ids.len() < fixture.rows[c.row].ids.len()
  ));
}