1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
//! Engine configuration.
use crate::types::*;
use serde::{Deserialize, Serialize};
/// Main engine configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EngineConfig {
/// Vector dimension
pub dim: u32,
/// Similarity metric
pub metric: Metric,
/// BPS configuration
pub bps: BpsConfig,
/// RDF configuration
pub rdf: RdfConfig,
/// Rerank configuration
pub rerank: RerankConfig,
/// Router configuration
pub router: RouterConfig,
/// LSM/Segment configuration
pub lsm: LsmConfig,
/// Query defaults
pub query: QueryConfig,
}
impl Default for EngineConfig {
fn default() -> Self {
Self {
dim: DEFAULT_DIM,
metric: Metric::DotProduct,
bps: BpsConfig::default(),
rdf: RdfConfig::default(),
rerank: RerankConfig::default(),
router: RouterConfig::default(),
lsm: LsmConfig::default(),
query: QueryConfig::default(),
}
}
}
impl EngineConfig {
/// Create config for a specific dimension
pub fn with_dim(dim: u32) -> Self {
let num_blocks = (dim + DEFAULT_BPS_BLOCK_SIZE as u32 - 1) / DEFAULT_BPS_BLOCK_SIZE as u32;
Self {
dim,
bps: BpsConfig {
block_size: DEFAULT_BPS_BLOCK_SIZE,
num_blocks: num_blocks as u16,
..Default::default()
},
..Default::default()
}
}
/// Get padded dimension (power of 2 for Hadamard)
pub fn padded_dim(&self) -> u32 {
let mut p = 1u32;
while p < self.dim {
p *= 2;
}
p
}
/// Validate configuration
pub fn validate(&self) -> crate::Result<()> {
if self.dim == 0 {
return Err(crate::Error::Config("Dimension must be > 0".into()));
}
if self.bps.block_size == 0 {
return Err(crate::Error::Config("BPS block size must be > 0".into()));
}
if self.rdf.top_t == 0 {
return Err(crate::Error::Config("RDF top_t must be > 0".into()));
}
Ok(())
}
}
/// BPS (Block Projection Sketch) configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BpsConfig {
/// Dimensions per block
pub block_size: u16,
/// Number of blocks (computed from dim)
pub num_blocks: u16,
/// Number of projections per block (1 or 2)
pub num_projections: u16,
}
impl BpsConfig {
/// Maximum safe value for num_blocks × num_projections to prevent u16 overflow.
/// With max L1 diff of 255 per slot: 65535 / 255 = 257
pub const MAX_SAFE_SLOTS: u32 = 257;
/// Validate configuration to ensure BPS distance won't overflow u16.
/// Returns error if num_blocks × num_projections × 255 > u16::MAX.
pub fn validate(&self) -> Result<(), String> {
let total_slots = self.num_blocks as u32 * self.num_projections as u32;
if total_slots > Self::MAX_SAFE_SLOTS {
return Err(format!(
"BPS configuration would overflow u16: {} blocks × {} projections = {} slots (max {})",
self.num_blocks,
self.num_projections,
total_slots,
Self::MAX_SAFE_SLOTS
));
}
Ok(())
}
/// Theoretical maximum L1 distance for this configuration
pub fn max_distance(&self) -> u32 {
self.num_blocks as u32 * self.num_projections as u32 * 255
}
}
impl Default for BpsConfig {
fn default() -> Self {
let num_blocks =
(DEFAULT_DIM + DEFAULT_BPS_BLOCK_SIZE as u32 - 1) / DEFAULT_BPS_BLOCK_SIZE as u32;
Self {
block_size: DEFAULT_BPS_BLOCK_SIZE,
num_blocks: num_blocks as u16,
num_projections: DEFAULT_BPS_PROJECTIONS,
}
}
}
/// RDF (Rare-Dominant Fingerprint) configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RdfConfig {
/// Number of top dimensions to select per vector
pub top_t: u16,
/// Stripe shift (log2 of stripe size)
pub stripe_shift: u8,
/// Stop-dimension document frequency threshold
pub stop_dim_threshold: u32,
/// IDF weight (alpha)
pub idf_weight: f32,
/// Variance weight (beta)
pub var_weight: f32,
}
impl Default for RdfConfig {
fn default() -> Self {
Self {
top_t: DEFAULT_RDF_TOP_T,
stripe_shift: DEFAULT_STRIPE_SHIFT,
stop_dim_threshold: DEFAULT_STOP_DIM_THRESHOLD,
idf_weight: 0.5,
var_weight: 0.5,
}
}
}
/// Rerank configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RerankConfig {
/// Number of outliers to store per vector
pub num_outliers: u8,
/// Use percentile-based quantization
pub percentile_quantization: bool,
/// Percentile for scale computation (e.g., 0.99)
pub scale_percentile: f32,
}
impl Default for RerankConfig {
fn default() -> Self {
Self {
num_outliers: DEFAULT_NUM_OUTLIERS,
percentile_quantization: true,
scale_percentile: 0.99,
}
}
}
/// Router configuration (for partitioned search)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RouterConfig {
/// Number of partitions/lists
pub n_lists: u32,
/// Number of lists to probe
pub n_probe: u32,
/// Enable router (only for large datasets)
pub enabled: bool,
}
impl Default for RouterConfig {
fn default() -> Self {
Self {
n_lists: 128,
n_probe: 8,
enabled: false,
}
}
}
/// LSM/Segment configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LsmConfig {
/// Maximum vectors per mutable segment before sealing
pub max_mutable_size: usize,
/// Maximum number of segments before triggering compaction
pub max_segments: usize,
/// Compaction ratio (merge N segments into 1)
pub compaction_ratio: usize,
}
impl Default for LsmConfig {
fn default() -> Self {
Self {
max_mutable_size: 100_000,
max_segments: 8,
compaction_ratio: 4,
}
}
}
/// Default query configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QueryConfig {
/// Default k
pub k: usize,
/// Default L_A (RDF candidates)
pub l_a: usize,
/// Default L_B (BPS candidates)
pub l_b: usize,
/// Default R (rerank candidates)
pub r: usize,
/// Enable adaptive widening by default
pub adaptive: bool,
/// Widening factor for low confidence
pub widening_factor: f32,
/// Score gap threshold for confidence
pub score_gap_threshold: f32,
}
impl Default for QueryConfig {
fn default() -> Self {
Self {
k: 10,
l_a: 5000,
l_b: 20000,
r: 500,
adaptive: true,
widening_factor: 2.0,
score_gap_threshold: 0.1,
}
}
}