qdrant_edge/edge/config/
shard.rs

1//! Edge shard configuration: user-facing params and conversion to/from SegmentConfig.
2
3use std::collections::HashMap;
4use std::path::Path;
5
6use crate::common::fs::{atomic_save_json, read_json};
7use crate::segment::common::operation_error::{OperationError, OperationResult};
8use crate::segment::types::{
9    HnswConfig, PayloadStorageType, QuantizationConfig, SegmentConfig, VectorNameBuf,
10};
11use serde::{Deserialize, Serialize};
12use crate::shard::operations::optimization::OptimizerThresholds;
13use crate::wal::WalOptions;
14
15use super::optimizers::EdgeOptimizersConfig;
16use super::vectors::{EdgeSparseVectorParams, EdgeVectorParams};
17
18/// File name for the persisted edge shard config.
19pub(crate) const EDGE_CONFIG_FILE: &str = "edge_config.json";
20
21/// Full configuration for an edge shard.
22#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
23#[serde(rename_all = "snake_case")]
24pub struct EdgeConfig {
25    /// If true, payload is stored on disk (mmap); otherwise in RAM. Same as `CollectionParams::on_disk_payload`.
26    #[serde(default = "default_on_disk_payload")]
27    pub on_disk_payload: bool,
28    /// Dense vector params per vector name.
29    #[serde(default)]
30    pub vectors: HashMap<VectorNameBuf, EdgeVectorParams>,
31    /// Sparse vector params per vector name.
32    #[serde(default)]
33    pub sparse_vectors: HashMap<VectorNameBuf, EdgeSparseVectorParams>,
34    /// Global HNSW config; per-vector override is in `vectors[].hnsw_config`
35    #[serde(default)]
36    pub hnsw_config: HnswConfig,
37    /// Global quantization config for all vectors
38    /// Per-vector override in in `vectors[].quantization_config`
39    #[serde(default, skip_serializing_if = "Option::is_none")]
40    pub quantization_config: Option<QuantizationConfig>,
41    #[serde(default)]
42    pub optimizers: EdgeOptimizersConfig,
43    /// WAL options for the shard. `None` keeps the WAL crate's defaults
44    /// (32 MiB segment capacity). Override for embedded/mobile deployments
45    /// where the default segment size is too large.
46    #[serde(default, skip_serializing_if = "Option::is_none")]
47    pub wal_options: Option<WalOptions>,
48}
49
50fn default_on_disk_payload() -> bool {
51    true
52}
53
54impl Default for EdgeConfig {
55    fn default() -> Self {
56        Self {
57            on_disk_payload: default_on_disk_payload(),
58            vectors: HashMap::new(),
59            sparse_vectors: HashMap::new(),
60            hnsw_config: HnswConfig::default(),
61            quantization_config: None,
62            optimizers: EdgeOptimizersConfig::default(),
63            wal_options: None,
64        }
65    }
66}
67
68impl EdgeConfig {
69    /// Start building an [`EdgeConfig`] with a fluent API.
70    pub fn builder() -> crate::edge::builders::EdgeConfigBuilder {
71        crate::edge::builders::EdgeConfigBuilder::new()
72    }
73
74    /// Build from existing segment config. Fills all parameters that can be inferred.
75    pub fn from_segment_config(segment: &SegmentConfig) -> Self {
76        let SegmentConfig {
77            vector_data,
78            sparse_vector_data,
79            payload_storage_type,
80        } = segment;
81
82        let vectors = vector_data
83            .iter()
84            .map(|(name, v)| (name.clone(), EdgeVectorParams::from_vector_data_config(v)))
85            .collect();
86
87        let sparse_vectors = sparse_vector_data
88            .iter()
89            .map(|(name, s)| {
90                (
91                    name.clone(),
92                    EdgeSparseVectorParams::from_sparse_vector_data_config(s),
93                )
94            })
95            .collect();
96
97        let on_disk_payload = payload_storage_type.is_on_disk();
98
99        // Infer global hnsw_config from per-vector HNSW configs when all agree
100        let hnsw_configs: Vec<HnswConfig> = vector_data
101            .values()
102            .filter_map(|v| match &v.index {
103                crate::segment::types::Indexes::Plain {} => None,
104                crate::segment::types::Indexes::Hnsw(h) => Some(*h),
105            })
106            .collect();
107        let hnsw_config = hnsw_configs
108            .first()
109            .and_then(|first| {
110                if hnsw_configs.iter().all(|h| h == first) {
111                    Some(*first)
112                } else {
113                    None
114                }
115            })
116            .unwrap_or_default();
117
118        Self {
119            on_disk_payload,
120            vectors,
121            sparse_vectors,
122            hnsw_config,
123            quantization_config: None,
124            optimizers: EdgeOptimizersConfig::default(),
125            wal_options: None,
126        }
127    }
128
129    /// Check compatibility with a segment config (e.g. loaded segment).
130    pub fn check_compatible_with_segment_config(
131        &self,
132        other: &SegmentConfig,
133    ) -> Result<(), String> {
134        self.plain_segment_config().check_compatible(other)
135    }
136
137    /// Segment config for creating appendable segments only.
138    /// Does not contain any HNSW configuration (plain index only).
139    pub fn plain_segment_config(&self) -> SegmentConfig {
140        let payload_storage_type = PayloadStorageType::from_on_disk_payload(self.on_disk_payload);
141        let vector_data = self
142            .vectors
143            .iter()
144            .map(|(name, p)| {
145                (
146                    name.clone(),
147                    p.to_plain_vector_data_config(self.quantization_config.as_ref()),
148                )
149            })
150            .collect();
151
152        let sparse_vector_data = self
153            .sparse_vectors
154            .iter()
155            .map(|(name, p)| (name.clone(), p.to_plain_sparse_vector_data_config()))
156            .collect();
157
158        SegmentConfig {
159            vector_data,
160            sparse_vector_data,
161            payload_storage_type,
162        }
163    }
164
165    /// Build segment optimizer config from this config (for blocking optimizers).
166    /// Use this instead of converting to SegmentConfig first.
167    pub fn segment_optimizer_config(&self) -> crate::shard::optimizers::config::SegmentOptimizerConfig {
168        use crate::shard::optimizers::config::SegmentOptimizerConfig;
169
170        let SegmentConfig {
171            vector_data: plain_dense_vector_config,
172            sparse_vector_data: plain_sparse_vector_config,
173            payload_storage_type,
174        } = self.plain_segment_config();
175
176        let dense_vector = self
177            .vectors
178            .iter()
179            .map(|(name, p)| {
180                (
181                    name.clone(),
182                    p.to_dense_vector_optimizer_config(
183                        &self.hnsw_config,
184                        self.quantization_config.as_ref(),
185                    ),
186                )
187            })
188            .collect();
189
190        let sparse_vector = self
191            .sparse_vectors
192            .iter()
193            .map(|(name, p)| (name.clone(), p.to_sparse_vector_optimizer_config()))
194            .collect();
195
196        SegmentOptimizerConfig {
197            payload_storage_type,
198            plain_dense_vector_config,
199            plain_sparse_vector_config,
200            dense_vector,
201            sparse_vector,
202        }
203    }
204
205    /// Return vector data config for a named vector (for read-only use, e.g. query).
206    /// Uses plain index; for optimizer/segment creation use segment_optimizer_config or plain_segment_config.
207    pub fn vector_data_config(
208        &self,
209        name: &VectorNameBuf,
210    ) -> Option<crate::segment::types::VectorDataConfig> {
211        self.vectors
212            .get(name)
213            .map(|p| p.to_plain_vector_data_config(self.quantization_config.as_ref()))
214    }
215
216    pub fn optimizer_thresholds(&self, num_indexing_threads: usize) -> OptimizerThresholds {
217        let indexing_threshold_kb = self.optimizers.get_indexing_threshold_kb();
218        OptimizerThresholds {
219            memmap_threshold_kb: usize::MAX,
220            indexing_threshold_kb,
221            max_segment_size_kb: self
222                .optimizers
223                .get_max_segment_size_kb(num_indexing_threads),
224            deferred_internal_id: None,
225        }
226    }
227
228    pub fn save(&self, path: &Path) -> OperationResult<()> {
229        let config_path = path.join(EDGE_CONFIG_FILE);
230        atomic_save_json(&config_path, self).map_err(|e| {
231            OperationError::service_error(format!(
232                "failed to write {}: {}",
233                config_path.display(),
234                e
235            ))
236        })
237    }
238
239    pub fn load(path: &Path) -> Option<OperationResult<Self>> {
240        let config_path = path.join(EDGE_CONFIG_FILE);
241        match fs_err::exists(&config_path) {
242            Ok(false) => return None,
243            Err(e) => return Some(Err(OperationError::from(e))),
244            Ok(true) => {}
245        }
246        Some(read_json(&config_path).map_err(OperationError::from))
247    }
248
249    pub fn set_hnsw_config(&mut self, hnsw_config: HnswConfig) {
250        self.hnsw_config = hnsw_config;
251    }
252
253    pub fn set_vector_hnsw_config(
254        &mut self,
255        vector_name: &str,
256        hnsw_config: HnswConfig,
257    ) -> OperationResult<()> {
258        let name = VectorNameBuf::from(vector_name);
259        let params = self
260            .vectors
261            .get_mut(&name)
262            .ok_or_else(|| OperationError::vector_name_not_exists(vector_name))?;
263        params.hnsw_config = Some(hnsw_config);
264        Ok(())
265    }
266
267    pub fn set_optimizers_config(&mut self, optimizers: EdgeOptimizersConfig) {
268        self.optimizers = optimizers;
269    }
270}
qdrant_edge/edge/config/shard.rs

qdrant_edge/edge/config/
shard.rs
