summa_core/segment/builder/
config.rs

1//! Configuration and statistics types for segment builder
2
3use std::path::PathBuf;
4
5use crate::compression::CompressionLevel;
6
7/// Statistics about segment builder state
8#[derive(Debug, Clone, Default)]
9pub struct SegmentBuilderStats {
10    /// Number of documents indexed
11    pub num_docs: u32,
12    /// Number of unique terms in the inverted index
13    pub unique_terms: usize,
14    /// Total postings in memory (across all terms)
15    pub postings_in_memory: usize,
16    /// Number of interned strings
17    pub interned_strings: usize,
18    /// Size of doc_field_lengths vector
19    pub doc_field_lengths_size: usize,
20    /// Estimated total memory usage in bytes
21    pub estimated_memory_bytes: usize,
22    /// Memory breakdown by component
23    pub memory_breakdown: MemoryBreakdown,
24    /// Documents indexed into a dynamically tokenized field
25    /// (`text<lex(by: ...)>`) without any value in the hint field, so the
26    /// tokenizer's default applied.
27    pub unhinted_dynamic_docs: u64,
28}
29
30/// Detailed memory breakdown by component
31#[derive(Debug, Clone, Default)]
32pub struct MemoryBreakdown {
33    /// Postings memory (CompactPosting structs)
34    pub postings_bytes: usize,
35    /// Inverted index HashMap overhead
36    pub index_overhead_bytes: usize,
37    /// Term interner memory
38    pub interner_bytes: usize,
39    /// Document field lengths
40    pub field_lengths_bytes: usize,
41    /// Dense vector storage
42    pub dense_vectors_bytes: usize,
43    /// Number of dense vectors
44    pub dense_vector_count: usize,
45    /// Sparse vector storage
46    pub sparse_vectors_bytes: usize,
47    /// Position index storage
48    pub position_index_bytes: usize,
49}
50
51/// Configuration for segment builder
52#[derive(Clone)]
53pub struct SegmentBuilderConfig {
54    /// Directory for temporary spill files
55    pub temp_dir: PathBuf,
56    /// Compression level for document store
57    pub compression_level: CompressionLevel,
58    /// Width of the document-store compression pool. Concurrent builders with
59    /// the same width share one process-wide executor.
60    pub num_compression_threads: usize,
61    /// Initial capacity for term interner
62    pub interner_capacity: usize,
63    /// Initial capacity for posting lists hashmap
64    pub posting_map_capacity: usize,
65    /// Term-dictionary compression / bloom configuration (from the index
66    /// optimization mode).
67    pub optimization: crate::structures::IndexOptimization,
68    /// Posting block codec (`docs/posting-codecs.md`).
69    pub posting_codec: crate::structures::PostingCodec,
70    /// New plain-text columns use versioned byte4 norms. Existing segments retain their scores.
71    pub quantized_norms: bool,
72    /// New position streams use a compact directory separate from payload pages.
73    pub compact_text: bool,
74    /// Opt in to compact, score-independent length/TF block bounds.
75    pub posting_ratio_bounds: bool,
76    /// Opt in to bounded competitive frequency/length envelopes. Implies ratio bounds.
77    pub posting_impact_bounds: bool,
78    /// Validated flush target for this segment's term dictionary.
79    pub term_dict_block_size: crate::structures::SSTableBlockSize,
80}
81
82impl SegmentBuilderConfig {
83    /// Block-bound metadata this builder writes; impact bounds imply ratio bounds.
84    pub fn effective_posting_bounds(&self) -> crate::index::PostingBounds {
85        crate::index::PostingBounds::new(self.posting_ratio_bounds, self.posting_impact_bounds)
86    }
87}
88
89impl Default for SegmentBuilderConfig {
90    fn default() -> Self {
91        Self {
92            #[cfg(feature = "native")]
93            temp_dir: std::env::temp_dir(),
94            #[cfg(not(feature = "native"))]
95            temp_dir: PathBuf::from("/tmp"),
96            compression_level: CompressionLevel(3),
97            #[cfg(feature = "native")]
98            num_compression_threads: crate::default_compression_threads(),
99            #[cfg(not(feature = "native"))]
100            num_compression_threads: 1,
101            interner_capacity: 1_000_000,
102            posting_map_capacity: 500_000,
103            optimization: crate::structures::IndexOptimization::default(),
104            posting_codec: crate::structures::PostingCodec::default(),
105            quantized_norms: false,
106            compact_text: false,
107            posting_ratio_bounds: false,
108            posting_impact_bounds: false,
109            term_dict_block_size: crate::structures::SSTableBlockSize::default(),
110        }
111    }
112}
summa_core/segment/builder/config.rs

summa_core/segment/builder/
config.rs
