summa_core/structures/postings/sparse/
config.rs

1//! Configuration types for sparse vector posting lists
2
3use serde::{Deserialize, Serialize};
4
5/// Sparse vector index format
6///
7/// Determines the on-disk layout and query execution strategy:
8/// - **MaxScore**: Per-dimension variable-size blocks (DAAT — document-at-a-time).
9///   Supports general sparse retrieval with block-max pruning.
10/// - **Bmp** (default): Fixed doc_id range blocks (BAAT — block-at-a-time).
11///   Based on Mallia, Suel & Tonellotto (SIGIR 2024). Divides the document
12///   space into fixed-size blocks and processes them in decreasing upper-bound
13///   order, enabling aggressive early termination.
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
15pub enum SparseFormat {
16    /// Per-dimension variable-size blocks (existing format, DAAT MaxScore)
17    MaxScore,
18    /// Fixed doc_id range blocks (BMP, BAAT block-at-a-time)
19    #[default]
20    Bmp,
21    /// Geometric summaries nominate candidates; exact forward values score them.
22    Seismic,
23}
24
25// Metadata written before BMP became the constructor default omitted MaxScore.
26// Keep that serialized meaning stable; new writers always name their backend.
27fn legacy_sparse_format() -> SparseFormat {
28    SparseFormat::MaxScore
29}
30
31/// Size of the index (term/dimension ID) in sparse vectors
32#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
33#[repr(u8)]
34pub enum IndexSize {
35    /// 16-bit index (0-65535), ideal for SPLADE vocabularies
36    U16 = 0,
37    /// 32-bit index (0-4B), for large vocabularies
38    #[default]
39    U32 = 1,
40}
41
42impl IndexSize {
43    /// Bytes per index
44    pub fn bytes(&self) -> usize {
45        match self {
46            IndexSize::U16 => 2,
47            IndexSize::U32 => 4,
48        }
49    }
50
51    /// Maximum value representable
52    pub fn max_value(&self) -> u32 {
53        match self {
54            IndexSize::U16 => u16::MAX as u32,
55            IndexSize::U32 => u32::MAX,
56        }
57    }
58
59    pub(crate) fn from_u8(v: u8) -> Option<Self> {
60        match v {
61            0 => Some(IndexSize::U16),
62            1 => Some(IndexSize::U32),
63            _ => None,
64        }
65    }
66}
67
68/// Quantization format for sparse vector weights
69///
70/// Float32 preserves input precision. Smaller weight representations trade
71/// precision for payload size; retrieval quality depends on the workload and
72/// must be measured. Integer encodings also store per-vector scale and offset,
73/// so total index-size savings differ from the ratio of weight widths.
74#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
75#[repr(u8)]
76pub enum WeightQuantization {
77    /// Full 32-bit float precision
78    #[default]
79    Float32 = 0,
80    /// 16-bit float (half precision), two bytes per weight
81    Float16 = 1,
82    /// 8-bit integer codes with per-vector scale and offset
83    UInt8 = 2,
84    /// 4-bit integer codes (two per byte) with per-vector scale and offset
85    UInt4 = 3,
86}
87
88impl WeightQuantization {
89    /// Bytes per weight (approximate for UInt4)
90    pub fn bytes_per_weight(&self) -> f32 {
91        match self {
92            WeightQuantization::Float32 => 4.0,
93            WeightQuantization::Float16 => 2.0,
94            WeightQuantization::UInt8 => 1.0,
95            WeightQuantization::UInt4 => 0.5,
96        }
97    }
98
99    pub(crate) fn from_u8(v: u8) -> Option<Self> {
100        match v {
101            0 => Some(WeightQuantization::Float32),
102            1 => Some(WeightQuantization::Float16),
103            2 => Some(WeightQuantization::UInt8),
104            3 => Some(WeightQuantization::UInt4),
105            _ => None,
106        }
107    }
108}
109
110/// Query-time weighting strategy for sparse vector queries
111#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
112#[serde(rename_all = "snake_case")]
113pub enum QueryWeighting {
114    /// All terms get weight 1.0
115    #[default]
116    One,
117    /// Terms weighted by IDF (inverse document frequency) from global index statistics
118    /// Uses ln(N/df), where N counts field values and df counts values containing the dimension
119    Idf,
120    /// Terms weighted by pre-computed IDF from model's idf.json file
121    /// Loaded from HuggingFace model repo. No fallback to global stats.
122    IdfFile,
123}
124
125/// Query-time configuration for sparse vectors
126///
127/// Quality-sensitive query optimization knobs. Weight filtering, dimension
128/// caps, fractional pruning, finite LSP gamma, and heap factors below 1.0 can
129/// all change the candidate set. They are disabled by default and should be
130/// tuned against representative Recall@K or relevance judgments.
131#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
132pub struct SparseQueryConfig {
133    /// HuggingFace tokenizer path/name for query-time tokenization
134    /// Example: "Alibaba-NLP/gte-Qwen2-1.5B-instruct"
135    #[serde(default, skip_serializing_if = "Option::is_none")]
136    pub tokenizer: Option<String>,
137    /// Weighting strategy for tokenized query terms
138    #[serde(default)]
139    pub weighting: QueryWeighting,
140    /// Heap factor for approximate search (SEISMIC-style optimization)
141    /// A block is skipped if its max possible score < heap_factor * threshold
142    ///
143    /// - 1.0 = exact search (default)
144    /// - values below 1.0 = increasingly aggressive block pruning
145    #[serde(default = "default_heap_factor")]
146    pub heap_factor: f32,
147    /// Minimum weight for query dimensions (query-time pruning)
148    /// Dimensions with abs(weight) below this threshold are dropped before search.
149    /// Useful for filtering low-IDF tokens that add latency without improving relevance.
150    ///
151    /// - 0.0 = no filtering (default)
152    /// - positive values drop dimensions and require quality validation
153    #[serde(default)]
154    pub weight_threshold: f32,
155    /// Maximum number of query dimensions to process (query pruning)
156    /// Processes only the top-k dimensions by weight
157    ///
158    /// - None = process all dimensions (default, exact)
159    /// - Some(k) = process only the top-k dimensions by absolute weight
160    #[serde(default, skip_serializing_if = "Option::is_none")]
161    pub max_query_dims: Option<usize>,
162    /// Fraction of query dimensions to keep (0.0-1.0), same semantics as
163    /// indexing-time `pruning`: sort by abs(weight) descending and keep the
164    /// top fraction. BMP uses this subset for candidate generation and the
165    /// bounded full query for final scoring; MaxScore uses the subset for both.
166    /// None or 1.0 = no pruning.
167    #[serde(default, skip_serializing_if = "Option::is_none")]
168    pub pruning: Option<f32>,
169    /// Minimum number of query dimensions before pruning and weight_threshold
170    /// filtering are applied. Protects short queries from losing most signal.
171    ///
172    /// Default: 4. Set to 0 to always apply pruning/filtering.
173    #[serde(default = "default_min_terms")]
174    pub min_query_dims: usize,
175    /// LSP/0 top-superblock guarantee γ. `None` selects the paper-derived
176    /// schedule from retrieval depth; `Some(0)` requests exhaustive traversal.
177    #[serde(default, skip_serializing_if = "Option::is_none")]
178    pub lsp_gamma: Option<usize>,
179    /// Number of query dimensions used to nominate Seismic candidates.
180    #[serde(default = "default_seismic_cut")]
181    pub seismic_cut: usize,
182    /// Summary pruning factor for Seismic candidate generation.
183    #[serde(default = "default_seismic_factor")]
184    pub seismic_factor: f32,
185    /// Scan shared forward values exhaustively, bypassing approximate nominations.
186    #[serde(default)]
187    pub exhaustive: bool,
188}
189
190fn default_seismic_cut() -> usize {
191    10
192}
193fn default_seismic_factor() -> f32 {
194    0.85
195}
196
197fn default_heap_factor() -> f32 {
198    1.0
199}
200
201impl Default for SparseQueryConfig {
202    fn default() -> Self {
203        Self {
204            tokenizer: None,
205            weighting: QueryWeighting::One,
206            heap_factor: 1.0,
207            weight_threshold: 0.0,
208            max_query_dims: None,
209            pruning: None,
210            min_query_dims: 4,
211            lsp_gamma: None,
212            seismic_cut: default_seismic_cut(),
213            seismic_factor: default_seismic_factor(),
214            exhaustive: false,
215        }
216    }
217}
218
219/// Bounded Seismic build policy. Merge copies runs without rebuilding them.
220#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
221#[serde(default)]
222pub struct SeismicConfig {
223    /// Maximum retained nominations per term in a newly built run.
224    pub postings: usize,
225    /// Target number of nominations per geometric cluster.
226    pub cluster_size: usize,
227    /// Fraction of summary magnitude retained for candidate ranking.
228    pub summary_energy: f32,
229    /// Lossless U16/U24/DotVByte forward dimension compression (default true).
230    pub forward_compression: bool,
231}
232
233impl Default for SeismicConfig {
234    fn default() -> Self {
235        Self {
236            postings: 4096,
237            cluster_size: 64,
238            summary_energy: 0.4,
239            forward_compression: true,
240        }
241    }
242}
243
244impl SeismicConfig {
245    pub(crate) fn validate(&self) -> Result<(), String> {
246        if self.postings == 0 || self.postings > 65_536 {
247            return Err("seismic postings must be in 1..=65536".into());
248        }
249        if self.cluster_size == 0 || self.cluster_size > self.postings {
250            return Err("seismic cluster_size must be in 1..=postings".into());
251        }
252        if !self.summary_energy.is_finite()
253            || !(0.0..=1.0).contains(&self.summary_energy)
254            || self.summary_energy == 0.0
255        {
256            return Err("seismic summary_energy must be in (0, 1]".into());
257        }
258        Ok(())
259    }
260}
261
262/// Configuration for sparse vector storage
263///
264/// Configuration knobs for learned sparse retrieval (SPLADE, uniCOIL, etc.).
265///
266/// Destructive posting-list and query-dimension pruning are opt-in. Their
267/// quality impact is corpus/model dependent and must be established with
268/// Recall@K or relevance judgments; a fixed retained fraction is not a safe
269/// production default.
270#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
271pub struct SparseVectorConfig {
272    /// Index format: BMP (default), MaxScore, or Seismic
273    #[serde(default = "legacy_sparse_format")]
274    pub format: SparseFormat,
275    #[serde(default)]
276    pub seismic: SeismicConfig,
277    /// Size of dimension/term indices
278    pub index_size: IndexSize,
279    /// Quantization for weights (see WeightQuantization docs for trade-offs)
280    pub weight_quantization: WeightQuantization,
281    /// Minimum weight threshold - weights below this value are not indexed
282    ///
283    /// Positive values reduce posting count but are model/corpus dependent.
284    /// Benchmark retrieval quality before choosing a production threshold.
285    #[serde(default)]
286    pub weight_threshold: f32,
287    /// Document-side mass cropping: keep the top-|weight| entries covering
288    /// this fraction of a sparse vector's total |weight| mass; the excessive
289    /// tail is dropped at indexing time.
290    ///
291    /// SPLADE-style vectors can concentrate importance in a few head terms,
292    /// but the relevance carried by the tail is model/corpus dependent.
293    ///
294    /// - None or >= 1.0 = keep all entries (default)
295    /// - Applied after `weight_threshold`; vectors with <= `min_terms`
296    ///   entries are never cropped.
297    #[serde(default, skip_serializing_if = "Option::is_none")]
298    pub doc_mass: Option<f32>,
299    /// Block size for posting lists (must be power of 2, default 128 for SIMD)
300    /// Larger blocks = better compression, smaller blocks = faster seeks.
301    /// Used by MaxScore format only.
302    #[serde(default = "default_block_size")]
303    pub block_size: usize,
304    /// BMP block size: number of consecutive doc_ids per block (must be power
305    /// of 2, max 256). Only used when format = Bmp. Uniform across every
306    /// segment of the field — set per field in SDL (`bmp_block_size: N`).
307    /// Smaller = better pruning granularity; larger means fewer locally
308    /// bit-packed maximum cells. Default 32 favors pruning granularity;
309    /// increase it only after representative tail-latency testing
310    /// (docs/bmp-grid-compression.md).
311    #[serde(default = "default_bmp_block_size")]
312    pub bmp_block_size: u32,
313    /// Bits per BMP block-grid cell: 4 (default) or 2. Two caps compressed D
314    /// payload groups at two bits; the exact space reduction depends on local
315    /// group widths. Measured pruning cost is small (+0.4-2.2% blocks scored;
316    /// the ceil-u4 superblock grid prunes first).
317    /// Grid bounds are ceil-quantized, so exact top-k results are unchanged
318    /// at any width. Uniform per field across all segments — set in SDL
319    /// (`bmp_grid_bits: 2`) at index creation.
320    #[serde(default = "default_bmp_grid_bits")]
321    pub bmp_grid_bits: u8,
322    /// Store quantized forward values in BMP blobs for L1 and BP (default true).
323    /// False emits a disabled-storage marker; normal BMP search is inverted.
324    #[serde(default = "default_bmp_forward_index", skip_serializing_if = "is_true")]
325    pub bmp_forward_index: bool,
326    /// Static pruning: fraction of postings to keep per inverted list (SEISMIC-style)
327    /// Lists are sorted by weight descending and truncated to top fraction.
328    ///
329    /// - None = keep all postings (default)
330    /// - Some(0.1) = keep only the top 10% of each dimension's postings
331    ///
332    /// A fraction is deliberately not enabled by the SPLADE presets. Per-list
333    /// frequency and score distributions vary widely, and keeping one posting
334    /// from a list of 4-10 entries can destroy candidate recall.
335    ///
336    /// Applied only during initial segment build, not during merge.
337    #[serde(default, skip_serializing_if = "Option::is_none")]
338    pub pruning: Option<f32>,
339    /// Query-time configuration (tokenizer, weighting)
340    #[serde(default, skip_serializing_if = "Option::is_none")]
341    pub query_config: Option<SparseQueryConfig>,
342    /// Fixed vocabulary size (number of dimensions) for BMP format.
343    ///
344    /// When set, all BMP segments use the same grid dimensions (rows = dims),
345    /// enabling zero-copy block-copy merge. The grid is indexed by dim_id directly
346    /// (no dim_ids Section C needed).
347    ///
348    /// Required for BMP format. Typical values:
349    /// - SPLADE/BERT: 30522 or 105879 (WordPiece / Unigram vocabulary)
350    /// - uniCOIL: 30522
351    /// - Custom models: set to vocabulary size
352    ///
353    /// If None, the BMP builder derives dims from observed data.
354    #[serde(default, skip_serializing_if = "Option::is_none")]
355    pub dims: Option<u32>,
356    /// Fixed max weight scale for BMP format.
357    ///
358    /// When set, all BMP segments use the same quantization scale
359    /// (`max_weight_scale = max_weight`), eliminating rescaling during merge.
360    ///
361    /// For SPLADE models: 5.0 (covers typical weight range 0-5).
362    /// If None, the BMP builder derives scale from data.
363    #[serde(default, skip_serializing_if = "Option::is_none")]
364    pub max_weight: Option<f32>,
365    /// Minimum number of postings in a dimension before pruning and
366    /// weight_threshold filtering are applied. Protects dimensions with
367    /// very few postings from losing most of their signal.
368    ///
369    /// Default: 4. Set to 0 to always apply pruning/filtering.
370    #[serde(default = "default_min_terms")]
371    pub min_terms: usize,
372}
373
374fn default_block_size() -> usize {
375    128
376}
377
378fn default_bmp_block_size() -> u32 {
379    SparseVectorConfig::DEFAULT_BMP_BLOCK_SIZE
380}
381
382fn default_bmp_grid_bits() -> u8 {
383    SparseVectorConfig::DEFAULT_BMP_GRID_BITS
384}
385
386fn default_bmp_forward_index() -> bool {
387    true
388}
389
390fn is_true(value: &bool) -> bool {
391    *value
392}
393
394fn default_min_terms() -> usize {
395    4
396}
397
398impl Default for SparseVectorConfig {
399    fn default() -> Self {
400        Self {
401            format: SparseFormat::Bmp,
402            seismic: SeismicConfig::default(),
403            index_size: IndexSize::U32,
404            weight_quantization: WeightQuantization::Float32,
405            weight_threshold: 0.0,
406            doc_mass: None,
407            block_size: 128,
408            bmp_block_size: default_bmp_block_size(),
409            bmp_grid_bits: default_bmp_grid_bits(),
410            bmp_forward_index: default_bmp_forward_index(),
411            pruning: None,
412            query_config: None,
413
414            dims: None,
415            max_weight: None,
416            min_terms: 4,
417        }
418    }
419}
420
421impl SparseVectorConfig {
422    pub const DEFAULT_BMP_BLOCK_SIZE: u32 = 32;
423    pub const DEFAULT_BMP_GRID_BITS: u8 = 4;
424
425    /// Recall-preserving SPLADE storage preset
426    ///
427    /// Optimized for SPLADE, uniCOIL, and similar learned sparse retrieval models.
428    /// UInt8 impacts and a small weight threshold reduce storage. Destructive
429    /// per-list, query-dimension, and heap pruning remain disabled; enable them
430    /// only after a representative quality benchmark.
431    ///
432    /// Vocabulary: ~30K dimensions (fits in u16)
433    pub fn splade() -> Self {
434        Self {
435            format: SparseFormat::MaxScore,
436            seismic: SeismicConfig::default(),
437            index_size: IndexSize::U16,
438            weight_quantization: WeightQuantization::UInt8,
439            weight_threshold: 0.01, // Remove ~30-50% of low-weight postings
440            doc_mass: None,
441            block_size: 128,
442            bmp_block_size: default_bmp_block_size(),
443            bmp_grid_bits: default_bmp_grid_bits(),
444            bmp_forward_index: default_bmp_forward_index(),
445            pruning: None,
446            query_config: Some(SparseQueryConfig {
447                tokenizer: None,
448                weighting: QueryWeighting::One,
449                heap_factor: 1.0,
450                weight_threshold: 0.01,
451                max_query_dims: None,
452                pruning: None,
453                min_query_dims: 4,
454                lsp_gamma: None,
455                seismic_cut: default_seismic_cut(),
456                seismic_factor: default_seismic_factor(),
457                exhaustive: false,
458            }),
459
460            dims: None,
461            max_weight: None,
462            min_terms: 4,
463        }
464    }
465
466    /// SPLADE-optimized config with BMP (Block-Max Pruning) format
467    ///
468    /// Same optimization settings as `splade()` but uses the BMP block-at-a-time
469    /// format (Mallia, Suel & Tonellotto, SIGIR 2024) instead of MaxScore.
470    /// BMP divides the document space into fixed-size blocks and processes them
471    /// in decreasing upper-bound order, enabling aggressive early termination.
472    pub fn splade_bmp() -> Self {
473        Self {
474            format: SparseFormat::Bmp,
475            seismic: SeismicConfig::default(),
476            index_size: IndexSize::U16,
477            weight_quantization: WeightQuantization::UInt8,
478            weight_threshold: 0.01,
479            doc_mass: None,
480            block_size: 128,
481            bmp_block_size: default_bmp_block_size(),
482            bmp_grid_bits: default_bmp_grid_bits(),
483            bmp_forward_index: default_bmp_forward_index(),
484            pruning: None,
485            query_config: Some(SparseQueryConfig {
486                tokenizer: None,
487                weighting: QueryWeighting::One,
488                heap_factor: 1.0,
489                weight_threshold: 0.01,
490                max_query_dims: None,
491                pruning: None,
492                min_query_dims: 4,
493                lsp_gamma: None,
494                seismic_cut: default_seismic_cut(),
495                seismic_factor: default_seismic_factor(),
496                exhaustive: false,
497            }),
498
499            dims: Some(105879),
500            max_weight: Some(5.0),
501            min_terms: 4,
502        }
503    }
504
505    /// Compact config: Maximum compression (experimental)
506    ///
507    /// Uses aggressive UInt4 quantization for smallest possible index size.
508    /// Measure payload size and retrieval quality on the target workload.
509    ///
510    /// Recommended for: Memory-constrained environments, cache-heavy workloads
511    pub fn compact() -> Self {
512        Self {
513            format: SparseFormat::MaxScore,
514            seismic: SeismicConfig::default(),
515            index_size: IndexSize::U16,
516            weight_quantization: WeightQuantization::UInt4,
517            weight_threshold: 0.02, // Slightly higher threshold for UInt4
518            doc_mass: None,
519            block_size: 128,
520            bmp_block_size: default_bmp_block_size(),
521            bmp_grid_bits: default_bmp_grid_bits(),
522            bmp_forward_index: default_bmp_forward_index(),
523            pruning: Some(0.15), // Keep top 15% per dimension
524            query_config: Some(SparseQueryConfig {
525                tokenizer: None,
526                weighting: QueryWeighting::One,
527                heap_factor: 0.7,         // More aggressive approximate search
528                weight_threshold: 0.02,   // Drop low-IDF query tokens
529                max_query_dims: Some(15), // Fewer query dimensions
530                pruning: Some(0.15),      // Keep top 15% of query dims
531                min_query_dims: 4,
532                lsp_gamma: None,
533                seismic_cut: default_seismic_cut(),
534                seismic_factor: default_seismic_factor(),
535                exhaustive: false,
536            }),
537
538            dims: None,
539            max_weight: None,
540            min_terms: 4,
541        }
542    }
543
544    /// Full precision config: No compression, baseline effectiveness
545    ///
546    /// Use for: Research baselines, when effectiveness is critical
547    pub fn full_precision() -> Self {
548        Self {
549            format: SparseFormat::MaxScore,
550            seismic: SeismicConfig::default(),
551            index_size: IndexSize::U32,
552            weight_quantization: WeightQuantization::Float32,
553            weight_threshold: 0.0,
554            doc_mass: None,
555            block_size: 128,
556            bmp_block_size: default_bmp_block_size(),
557            bmp_grid_bits: default_bmp_grid_bits(),
558            bmp_forward_index: default_bmp_forward_index(),
559            pruning: None,
560            query_config: None,
561
562            dims: None,
563            max_weight: None,
564            min_terms: 4,
565        }
566    }
567
568    /// Conservative config: Mild optimizations, minimal effectiveness loss
569    ///
570    /// Balances compression and effectiveness with conservative defaults.
571    /// Measure payload size, latency and retrieval quality on the target workload.
572    ///
573    /// Recommended for: Production deployments prioritizing effectiveness
574    pub fn conservative() -> Self {
575        Self {
576            format: SparseFormat::MaxScore,
577            seismic: SeismicConfig::default(),
578            index_size: IndexSize::U32,
579            weight_quantization: WeightQuantization::Float16,
580            weight_threshold: 0.005, // Minimal pruning
581            doc_mass: None,
582            block_size: 128,
583            bmp_block_size: default_bmp_block_size(),
584            bmp_grid_bits: default_bmp_grid_bits(),
585            bmp_forward_index: default_bmp_forward_index(),
586            pruning: None, // No posting list pruning
587            query_config: Some(SparseQueryConfig {
588                tokenizer: None,
589                weighting: QueryWeighting::One,
590                heap_factor: 0.9,         // Nearly exact search
591                weight_threshold: 0.005,  // Minimal query pruning
592                max_query_dims: Some(50), // Process more dimensions
593                pruning: None,            // No fraction-based pruning
594                min_query_dims: 4,
595                lsp_gamma: None,
596                seismic_cut: default_seismic_cut(),
597                seismic_factor: default_seismic_factor(),
598                exhaustive: false,
599            }),
600
601            dims: None,
602            max_weight: None,
603            min_terms: 4,
604        }
605    }
606
607    /// Set weight threshold (builder pattern)
608    pub fn with_weight_threshold(mut self, threshold: f32) -> Self {
609        self.weight_threshold = threshold;
610        self
611    }
612
613    /// Set document-side mass cropping fraction (builder pattern)
614    /// e.g., 0.9 = keep top-weight entries covering 90% of each vector's mass
615    pub fn with_doc_mass(mut self, fraction: f32) -> Self {
616        self.doc_mass = Some(fraction.clamp(0.0, 1.0));
617        self
618    }
619
620    /// Set posting list pruning fraction (builder pattern)
621    /// e.g., 0.1 = keep top 10% of postings per dimension
622    pub fn with_pruning(mut self, fraction: f32) -> Self {
623        self.pruning = Some(fraction.clamp(0.0, 1.0));
624        self
625    }
626
627    /// Bytes per entry (index + weight)
628    pub fn bytes_per_entry(&self) -> f32 {
629        let dimension_bytes = if self.format == SparseFormat::Seismic {
630            4
631        } else {
632            self.index_size.bytes()
633        };
634        dimension_bytes as f32 + self.weight_quantization.bytes_per_weight()
635    }
636
637    /// Serialize config to a single byte.
638    ///
639    /// Layout: bits 7-4 = IndexSize, bit 3 = format (0=MaxScore, 1=BMP), bits 2-0 = WeightQuantization
640    pub fn to_byte(&self) -> u8 {
641        if self.format == SparseFormat::Seismic {
642            return 0x50 | self.weight_quantization as u8;
643        }
644        let format_bit = if self.format == SparseFormat::Bmp {
645            0x08
646        } else {
647            0
648        };
649        ((self.index_size as u8) << 4) | format_bit | (self.weight_quantization as u8)
650    }
651
652    /// Deserialize config from a single byte.
653    ///
654    /// Note: weight_threshold, block_size, bmp_block_size, and query_config are not
655    /// serialized in the byte — they come from the schema.
656    pub fn from_byte(b: u8) -> Option<Self> {
657        if b & 0xfc == 0x50 {
658            return Some(Self {
659                format: SparseFormat::Seismic,
660                index_size: IndexSize::U32,
661                weight_quantization: WeightQuantization::from_u8(b & 3)?,
662                ..Default::default()
663            });
664        }
665        if b & 0xc0 != 0 {
666            return None;
667        }
668        let index_size = IndexSize::from_u8((b >> 4) & 0x03)?;
669        let format = if b & 0x08 != 0 {
670            SparseFormat::Bmp
671        } else {
672            SparseFormat::MaxScore
673        };
674        let weight_quantization = WeightQuantization::from_u8(b & 0x07)?;
675        Some(Self {
676            format,
677            seismic: SeismicConfig::default(),
678            index_size,
679            weight_quantization,
680            weight_threshold: 0.0,
681            doc_mass: None,
682            block_size: 128,
683            bmp_block_size: default_bmp_block_size(),
684            bmp_grid_bits: default_bmp_grid_bits(),
685            bmp_forward_index: default_bmp_forward_index(),
686            pruning: None,
687            query_config: None,
688
689            dims: None,
690            max_weight: None,
691            min_terms: 4,
692        })
693    }
694
695    /// Set block size (builder pattern)
696    /// Must be power of 2, recommended: 64, 128, 256
697    pub fn with_block_size(mut self, size: usize) -> Self {
698        self.block_size = size.next_power_of_two();
699        self
700    }
701
702    /// Set query configuration (builder pattern)
703    pub fn with_query_config(mut self, config: SparseQueryConfig) -> Self {
704        self.query_config = Some(config);
705        self
706    }
707}
708
709/// A sparse vector entry: (dimension_id, weight)
710#[derive(Debug, Clone, Copy, PartialEq)]
711pub struct SparseEntry {
712    pub dim_id: u32,
713    pub weight: f32,
714}
715
716/// Sparse vector representation
717#[derive(Debug, Clone, Default)]
718pub struct SparseVector {
719    pub(super) entries: Vec<SparseEntry>,
720}
721
722impl SparseVector {
723    /// Create a new sparse vector
724    pub fn new() -> Self {
725        Self {
726            entries: Vec::new(),
727        }
728    }
729
730    /// Create with pre-allocated capacity
731    pub fn with_capacity(capacity: usize) -> Self {
732        Self {
733            entries: Vec::with_capacity(capacity),
734        }
735    }
736
737    /// Create from dimension IDs and weights
738    pub fn from_entries(dim_ids: &[u32], weights: &[f32]) -> Self {
739        assert_eq!(dim_ids.len(), weights.len());
740        let mut entries: Vec<SparseEntry> = dim_ids
741            .iter()
742            .zip(weights.iter())
743            .map(|(&dim_id, &weight)| SparseEntry { dim_id, weight })
744            .collect();
745        // Sort by dimension ID for efficient intersection
746        entries.sort_by_key(|e| e.dim_id);
747        Self { entries }
748    }
749
750    /// Add an entry (must maintain sorted order by dim_id)
751    pub fn push(&mut self, dim_id: u32, weight: f32) {
752        debug_assert!(
753            self.entries.is_empty() || self.entries.last().unwrap().dim_id < dim_id,
754            "Entries must be added in sorted order by dim_id"
755        );
756        self.entries.push(SparseEntry { dim_id, weight });
757    }
758
759    /// Number of non-zero entries
760    pub fn len(&self) -> usize {
761        self.entries.len()
762    }
763
764    /// Check if empty
765    pub fn is_empty(&self) -> bool {
766        self.entries.is_empty()
767    }
768
769    /// Iterate over entries
770    pub fn iter(&self) -> impl Iterator<Item = &SparseEntry> {
771        self.entries.iter()
772    }
773
774    /// Sort by dimension ID (required for posting list encoding)
775    pub fn sort_by_dim(&mut self) {
776        self.entries.sort_by_key(|e| e.dim_id);
777    }
778
779    /// Sort by weight descending
780    pub fn sort_by_weight_desc(&mut self) {
781        self.entries.sort_by(|a, b| {
782            b.weight
783                .partial_cmp(&a.weight)
784                .unwrap_or(std::cmp::Ordering::Equal)
785        });
786    }
787
788    /// Get top-k entries by weight
789    pub fn top_k(&self, k: usize) -> Vec<SparseEntry> {
790        let mut sorted = self.entries.clone();
791        sorted.sort_by(|a, b| {
792            b.weight
793                .partial_cmp(&a.weight)
794                .unwrap_or(std::cmp::Ordering::Equal)
795        });
796        sorted.truncate(k);
797        sorted
798    }
799
800    /// Compute dot product with another sparse vector
801    pub fn dot(&self, other: &SparseVector) -> f32 {
802        let mut result = 0.0f32;
803        let mut i = 0;
804        let mut j = 0;
805
806        while i < self.entries.len() && j < other.entries.len() {
807            let a = &self.entries[i];
808            let b = &other.entries[j];
809
810            match a.dim_id.cmp(&b.dim_id) {
811                std::cmp::Ordering::Less => i += 1,
812                std::cmp::Ordering::Greater => j += 1,
813                std::cmp::Ordering::Equal => {
814                    result += a.weight * b.weight;
815                    i += 1;
816                    j += 1;
817                }
818            }
819        }
820
821        result
822    }
823
824    /// L2 norm squared
825    pub fn norm_squared(&self) -> f32 {
826        self.entries.iter().map(|e| e.weight * e.weight).sum()
827    }
828
829    /// L2 norm
830    pub fn norm(&self) -> f32 {
831        self.norm_squared().sqrt()
832    }
833
834    /// Prune dimensions below a weight threshold
835    pub fn filter_by_weight(&self, min_weight: f32) -> Self {
836        let entries: Vec<SparseEntry> = self
837            .entries
838            .iter()
839            .filter(|e| e.weight.abs() >= min_weight)
840            .cloned()
841            .collect();
842        Self { entries }
843    }
844}
845
846impl From<Vec<(u32, f32)>> for SparseVector {
847    fn from(pairs: Vec<(u32, f32)>) -> Self {
848        Self {
849            entries: pairs
850                .into_iter()
851                .map(|(dim_id, weight)| SparseEntry { dim_id, weight })
852                .collect(),
853        }
854    }
855}
856
857impl From<SparseVector> for Vec<(u32, f32)> {
858    fn from(vec: SparseVector) -> Self {
859        vec.entries
860            .into_iter()
861            .map(|e| (e.dim_id, e.weight))
862            .collect()
863    }
864}
865
866#[cfg(test)]
867mod seismic_config_tests {
868    use super::*;
869
870    #[test]
871    fn sparse_default_uses_bmp_with_bounded_seismic_settings() {
872        let config = SparseVectorConfig::default();
873        assert_eq!(config.format, SparseFormat::Bmp);
874        config.seismic.validate().unwrap();
875        let query = SparseQueryConfig::default();
876        assert!(!query.exhaustive);
877        assert_eq!(query.seismic_cut, 10);
878        let restored: SparseVectorConfig =
879            serde_json::from_value(serde_json::to_value(config).unwrap()).unwrap();
880        assert_eq!(restored.format, SparseFormat::Bmp);
881    }
882
883    #[test]
884    fn seismic_forward_compression_defaults_on_and_preserves_explicit_opt_out() {
885        assert!(SeismicConfig::default().forward_compression);
886        let omitted: SeismicConfig = serde_json::from_str("{}").unwrap();
887        assert!(omitted.forward_compression);
888        let mut field = serde_json::to_value(SparseVectorConfig::default()).unwrap();
889        field.as_object_mut().unwrap().remove("seismic");
890        let omitted_field: SparseVectorConfig = serde_json::from_value(field).unwrap();
891        assert!(omitted_field.seismic.forward_compression);
892        let disabled: SeismicConfig =
893            serde_json::from_str(r#"{"forward_compression":false}"#).unwrap();
894        assert!(!disabled.forward_compression);
895        assert_eq!(
896            serde_json::from_value::<SeismicConfig>(serde_json::to_value(&disabled).unwrap())
897                .unwrap(),
898            disabled
899        );
900    }
901
902    #[test]
903    fn every_sparse_format_roundtrips_independently_of_the_default() {
904        for format in [
905            SparseFormat::Bmp,
906            SparseFormat::MaxScore,
907            SparseFormat::Seismic,
908        ] {
909            for weight_quantization in [
910                WeightQuantization::Float32,
911                WeightQuantization::Float16,
912                WeightQuantization::UInt8,
913                WeightQuantization::UInt4,
914            ] {
915                let config = SparseVectorConfig {
916                    format,
917                    weight_quantization,
918                    ..Default::default()
919                };
920                let decoded = SparseVectorConfig::from_byte(config.to_byte()).unwrap();
921                assert_eq!(decoded.format, format);
922                assert_eq!(decoded.weight_quantization, weight_quantization);
923                let json = serde_json::to_vec(&config).unwrap();
924                assert_eq!(
925                    serde_json::from_slice::<SparseVectorConfig>(&json).unwrap(),
926                    config
927                );
928            }
929        }
930        assert!(SparseVectorConfig::from_byte(0x90).is_none());
931    }
932
933    #[test]
934    fn seismic_build_settings_reject_unbounded_or_nonfinite_work() {
935        for config in [
936            SeismicConfig {
937                postings: 0,
938                ..SeismicConfig::default()
939            },
940            SeismicConfig {
941                postings: 65_537,
942                ..SeismicConfig::default()
943            },
944            SeismicConfig {
945                cluster_size: 0,
946                ..SeismicConfig::default()
947            },
948            SeismicConfig {
949                cluster_size: 4097,
950                ..SeismicConfig::default()
951            },
952            SeismicConfig {
953                summary_energy: f32::NAN,
954                ..SeismicConfig::default()
955            },
956            SeismicConfig {
957                summary_energy: 0.0,
958                ..SeismicConfig::default()
959            },
960        ] {
961            assert!(config.validate().is_err());
962        }
963    }
964}
summa_core/structures/postings/sparse/config.rs

summa_core/structures/postings/sparse/
config.rs
