Skip to main content

qql_protocol/
schema.rs

1//! Transport-neutral QQL backend model.
2//!
3//! Types in this module are the boundary between QQL compilation and a Qdrant
4//! transport adapter. They deliberately do not depend on generated OpenAPI or
5//! protobuf types. Query hits live on [`crate::response::SearchHit`]; this
6//! module holds collection metadata used by `USING` resolution and dump.
7//!
8//! # Deliberate JSON
9//!
10//! Collection-level [`CollectionSchema::hnsw`] / [`CollectionSchema::optimizers`]
11//! / [`CollectionSchema::quantization`] are typed plan configs. What stays
12//! `serde_json` is intentional:
13//!
14//! - [`VectorSpec::hnsw`] / [`VectorSpec::quantization`] /
15//!   [`VectorSpec::multivector`] and [`SparseVectorSpec::index`]: per-vector
16//!   config fragments Qdrant owns and dump re-emits verbatim.
17//! - [`PayloadIndexSpec::params`]: Qdrant's payload-index parameter union.
18//! - Formula `DEFAULTS`: the expression engine takes JSON values directly
19//!   (`qql-edge`'s formula lowering converts the typed plan tree at the
20//!   boundary).
21//! - REST request/response bodies (`qql_plan::Route`, [`crate::rest`]):
22//!   the wire format itself.
23
24use qql_plan::{HnswConfig, OptimizersConfig, QuantizationConfig};
25use serde::de::DeserializeOwned;
26use serde::{Deserialize, Serialize};
27
28/// One dense vector definition from collection config.
29///
30/// `name == None` means Qdrant's default unnamed vector.
31#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
32pub struct VectorSpec {
33    /// Vector name; `None` for the unnamed default vector.
34    pub name: Option<String>,
35    /// Vector dimension.
36    pub size: u64,
37    /// Distance metric as reported by Qdrant (e.g. `Cosine`, `Dot`).
38    pub distance: String,
39    /// Per-vector HNSW overrides (`hnsw_config`).
40    #[serde(default, skip_serializing_if = "Option::is_none")]
41    pub hnsw: Option<serde_json::Map<String, serde_json::Value>>,
42    /// Per-vector quantization config (`quantization_config`).
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub quantization: Option<serde_json::Value>,
45    /// Multivector (ColBERT) config (`multivector_config`).
46    #[serde(default, skip_serializing_if = "Option::is_none")]
47    pub multivector: Option<serde_json::Map<String, serde_json::Value>>,
48    /// Whether vector storage stays on disk (`on_disk`).
49    #[serde(default, skip_serializing_if = "Option::is_none")]
50    pub on_disk: Option<bool>,
51    /// Storage element type (`datatype`, e.g. `float32`).
52    #[serde(default, skip_serializing_if = "Option::is_none")]
53    pub datatype: Option<String>,
54    /// Memory placement hint for vector storage (`memory`).
55    #[serde(default, skip_serializing_if = "Option::is_none")]
56    pub memory: Option<String>,
57}
58
59/// A payload field index declared on the collection.
60#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
61pub struct PayloadIndexSpec {
62    /// Payload field path the index is built on.
63    pub field: String,
64    /// Indexed data type (e.g. `keyword`, `integer`, `datetime`).
65    pub data_type: String,
66    /// Extra index parameters (e.g. full-text analyzer settings).
67    #[serde(default)]
68    pub params: serde_json::Map<String, serde_json::Value>,
69    /// Whether this is a tenant index (`is_tenant`).
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub is_tenant: Option<bool>,
72}
73
74/// Collection-level params relevant to dump / DDL reconstruction.
75#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
76pub struct CollectionParamsSpec {
77    /// Number of shards (`shard_number`).
78    #[serde(default, skip_serializing_if = "Option::is_none")]
79    pub shard_number: Option<u64>,
80    /// Sharding method name (`sharding_method`, e.g. `auto`).
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub sharding_method: Option<String>,
83    /// Whether payloads are stored on disk (`on_disk_payload`).
84    #[serde(default, skip_serializing_if = "Option::is_none")]
85    pub on_disk_payload: Option<bool>,
86    /// Payload memory placement (`params.payload.memory`).
87    #[serde(default, skip_serializing_if = "Option::is_none")]
88    pub payload_memory: Option<String>,
89    /// Shard replication factor (`replication_factor`).
90    #[serde(default, skip_serializing_if = "Option::is_none")]
91    pub replication_factor: Option<u64>,
92}
93
94/// A named sparse vector definition from collection config.
95#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
96pub struct SparseVectorSpec {
97    /// Sparse vector name as declared on the collection.
98    pub name: String,
99    /// Sparse index settings (`index`), e.g. on-disk placement.
100    #[serde(default, skip_serializing_if = "Option::is_none")]
101    pub index: Option<serde_json::Map<String, serde_json::Value>>,
102    /// Optional modifier (`modifier`, e.g. `idf` for server-side BM25 IDF).
103    #[serde(default, skip_serializing_if = "Option::is_none")]
104    pub modifier: Option<String>,
105}
106
107/// The vector / index schema needed by topology resolution and dump.
108#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
109pub struct CollectionSchema {
110    /// Named dense vector names. Empty for a single unnamed default vector.
111    #[serde(default)]
112    pub dense_vectors: Vec<String>,
113    /// Named sparse vector definitions.
114    #[serde(default)]
115    pub sparse_vectors: Vec<SparseVectorSpec>,
116    /// Full dense vector definitions (size + distance) when the backend provides them.
117    #[serde(default)]
118    pub vectors: Vec<VectorSpec>,
119    /// Payload field indexes declared on the collection.
120    #[serde(default)]
121    pub payload_indexes: Vec<PayloadIndexSpec>,
122    /// Collection-level parameters relevant to dump / DDL reconstruction.
123    #[serde(default)]
124    pub params: CollectionParamsSpec,
125    /// Collection-level HNSW config (`hnsw_config`).
126    #[serde(default, skip_serializing_if = "Option::is_none")]
127    pub hnsw: Option<HnswConfig>,
128    /// Collection-level optimizer config (`optimizer_config`).
129    #[serde(default, skip_serializing_if = "Option::is_none")]
130    pub optimizers: Option<OptimizersConfig>,
131    /// Collection-level quantization config (`quantization_config`).
132    #[serde(default, skip_serializing_if = "Option::is_none")]
133    pub quantization: Option<QuantizationConfig>,
134}
135
136/// Transport-neutral collection metadata consumed by the executor.
137#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
138pub struct CollectionInfo {
139    /// Collection health status reported by the backend (e.g. `green`).
140    #[serde(default)]
141    pub status: String,
142    /// Number of points currently stored in the collection.
143    #[serde(default)]
144    pub points_count: u64,
145    /// Approximate number of vectors added to a vector index (HNSW/sparse).
146    ///
147    /// Not exact — Qdrant documents these counts as approximate and
148    /// `indexed_vectors_count < points_count` on edge means optimization (and
149    /// therefore indexing) is pending. `None` when the backend omits it.
150    #[serde(default, skip_serializing_if = "Option::is_none")]
151    pub indexed_vectors_count: Option<u64>,
152    /// Number of storage segments backing the collection.
153    #[serde(default)]
154    pub segments_count: u64,
155    /// Vector and index schema used for `USING` resolution and dumps.
156    #[serde(default)]
157    pub schema: CollectionSchema,
158}
159
160/// Parse Qdrant REST `/collections/{name}` result JSON into a typed schema.
161///
162/// Shared by the REST adapter so dump and topology checks never dig through
163/// untyped JSON on their own.
164pub fn schema_from_rest_result(result: &serde_json::Value) -> CollectionSchema {
165    let mut schema = CollectionSchema::default();
166    let params = result.get("config").and_then(|c| c.get("params"));
167
168    if let Some(vectors) = params
169        .and_then(|p| p.get("vectors"))
170        .and_then(|v| v.as_object())
171    {
172        // Unnamed default vector: { "size": N, "distance": "Cosine" }
173        if vectors.contains_key("size") && vectors.contains_key("distance") {
174            let vec_obj = serde_json::Value::Object(vectors.clone());
175            if let Some(spec) = extract_vector_spec(None, &vec_obj) {
176                schema.vectors.push(spec);
177            }
178            schema.dense_vectors.clear();
179        } else {
180            let mut named = Vec::new();
181            for (name, cfg) in vectors {
182                if is_pseudo_vector_key(name) {
183                    continue;
184                }
185                named.push(name.clone());
186                if let Some(spec) = extract_vector_spec(Some(name.clone()), cfg) {
187                    schema.vectors.push(spec);
188                }
189            }
190            named.sort();
191            schema.vectors.sort_by(|a, b| a.name.cmp(&b.name));
192            schema.dense_vectors = named;
193        }
194    }
195
196    if let Some(sparse) = params
197        .and_then(|p| p.get("sparse_vectors"))
198        .and_then(|v| v.as_object())
199    {
200        for (name, cfg) in sparse {
201            let modifier = cfg
202                .get("modifier")
203                .and_then(|m| m.as_str())
204                .map(String::from);
205            let index = cfg.get("index").and_then(|i| i.as_object()).cloned();
206            schema.sparse_vectors.push(SparseVectorSpec {
207                name: name.clone(),
208                index,
209                modifier,
210            });
211        }
212        schema.sparse_vectors.sort_by(|a, b| a.name.cmp(&b.name));
213    }
214
215    if let Some(p) = params {
216        schema.params.shard_number = p.get("shard_number").and_then(|v| v.as_u64());
217        schema.params.sharding_method = p
218            .get("sharding_method")
219            .and_then(|v| v.as_str())
220            .map(String::from);
221        schema.params.on_disk_payload = p.get("on_disk_payload").and_then(|v| v.as_bool());
222        schema.params.replication_factor = p.get("replication_factor").and_then(|v| v.as_u64());
223        schema.params.payload_memory = result
224            .get("config")
225            .and_then(|c| c.get("params"))
226            .and_then(|p| p.get("payload"))
227            .and_then(|payload| payload.get("memory"))
228            .and_then(|v| v.as_str())
229            .map(str::to_string);
230    }
231
232    if let Some(payload_schema) = result.get("payload_schema").and_then(|s| s.as_object()) {
233        for (field, meta) in payload_schema {
234            let data_type = meta
235                .get("data_type")
236                .and_then(|t| t.as_str())
237                .or_else(|| meta.get("type").and_then(|t| t.as_str()))
238                .unwrap_or("keyword")
239                .to_ascii_lowercase();
240
241            let mut params_map = serde_json::Map::new();
242            if let Some(obj) = meta.get("params").and_then(|p| p.as_object()) {
243                for (k, v) in obj {
244                    if k != "type" {
245                        params_map.insert(k.clone(), v.clone());
246                    }
247                }
248            }
249            let is_tenant = meta
250                .get("is_tenant")
251                .and_then(|v| v.as_bool())
252                .or_else(|| params_map.get("is_tenant").and_then(|v| v.as_bool()));
253
254            schema.payload_indexes.push(PayloadIndexSpec {
255                field: field.clone(),
256                data_type,
257                params: params_map,
258                is_tenant,
259            });
260        }
261        schema.payload_indexes.sort_by(|a, b| a.field.cmp(&b.field));
262    }
263
264    if let Some(config_obj) = result.get("config").and_then(|c| c.as_object()) {
265        schema.hnsw = config_obj.get("hnsw_config").and_then(parse_config);
266        // Qdrant REST historically uses optimizer_config (singular); accept both.
267        schema.optimizers = config_obj
268            .get("optimizer_config")
269            .or_else(|| config_obj.get("optimizers_config"))
270            .and_then(parse_config);
271        schema.quantization = config_obj.get("quantization_config").and_then(parse_config);
272    }
273
274    schema
275}
276
277/// Parse one optional REST config object into its typed plan config.
278///
279/// Unknown keys are ignored, `null` reads as absent, and a shape the plan type
280/// cannot represent drops the section instead of failing the whole response —
281/// the same leniency the previous map filters had.
282fn parse_config<T: DeserializeOwned>(value: &serde_json::Value) -> Option<T> {
283    serde_json::from_value(value.clone()).ok()
284}
285
286fn is_pseudo_vector_key(name: &str) -> bool {
287    matches!(
288        name,
289        "size"
290            | "distance"
291            | "hnsw_config"
292            | "quantization_config"
293            | "multivector_config"
294            | "on_disk"
295            | "datatype"
296    )
297}
298
299fn extract_vector_spec(name: Option<String>, cfg: &serde_json::Value) -> Option<VectorSpec> {
300    let size = cfg.get("size").and_then(|s| s.as_u64())?;
301    let distance = cfg
302        .get("distance")
303        .and_then(|d| d.as_str())
304        .unwrap_or("Cosine")
305        .to_string();
306    let hnsw = cfg.get("hnsw_config").and_then(|h| h.as_object()).cloned();
307    let quantization = cfg.get("quantization_config").cloned();
308    let multivector = cfg
309        .get("multivector_config")
310        .and_then(|m| m.as_object())
311        .cloned();
312    let on_disk = cfg.get("on_disk").and_then(|b| b.as_bool());
313    let datatype = cfg
314        .get("datatype")
315        .and_then(|d| d.as_str())
316        .map(String::from);
317    let memory = cfg.get("memory").and_then(|m| m.as_str()).map(String::from);
318
319    Some(VectorSpec {
320        name,
321        size,
322        distance,
323        hnsw,
324        quantization,
325        multivector,
326        on_disk,
327        datatype,
328        memory,
329    })
330}
331
332#[cfg(test)]
333mod tests {
334    use super::*;
335    use serde_json::json;
336
337    /// Kept in lockstep with `qql-wasm/src/schema.rs`'s
338    /// `collection_info_matches_runtime_shape`: the same REST fixture must
339    /// serialize to the same canonical `CollectionSchema` JSON on both sides.
340    #[test]
341    fn schema_from_rest_result_matches_wasm_fixture() {
342        let result = json!({
343            "config": {
344                "params": {
345                    "vectors": {
346                        "title": {"size": 4, "distance": "Dot"},
347                        "body": {"size": 8, "distance": "Cosine", "multivector_config": {"comparator": "max_sim"}},
348                    },
349                    "sparse_vectors": {"text": {"modifier": "idf"}},
350                    "shard_number": 3,
351                    "sharding_method": "custom",
352                    "on_disk_payload": true,
353                    "replication_factor": 2,
354                    "payload": {"memory": "cold"},
355                },
356                "hnsw_config": {"m": 16, "ef_construct": 100, "unknown": 1},
357                "optimizer_config": {"deleted_threshold": 0.2, "unknown": 2},
358                "quantization_config": {"scalar": {"type": "int8"}},
359            },
360            "payload_schema": {
361                "tenant_id": {"data_type": "keyword", "params": {"is_tenant": true}},
362                "title": {"data_type": "text"},
363            },
364        });
365
366        assert_eq!(
367            serde_json::to_value(schema_from_rest_result(&result)).unwrap(),
368            json!({
369                "dense_vectors": ["body", "title"],
370                "sparse_vectors": [{"name": "text", "modifier": "idf"}],
371                "vectors": [
372                    {"name": "body", "size": 8, "distance": "Cosine", "multivector": {"comparator": "max_sim"}},
373                    {"name": "title", "size": 4, "distance": "Dot"},
374                ],
375                "payload_indexes": [
376                    {"field": "tenant_id", "data_type": "keyword", "params": {"is_tenant": true}, "is_tenant": true},
377                    {"field": "title", "data_type": "text", "params": {}},
378                ],
379                "params": {
380                    "shard_number": 3,
381                    "sharding_method": "custom",
382                    "on_disk_payload": true,
383                    "payload_memory": "cold",
384                    "replication_factor": 2,
385                },
386                "hnsw": {"m": 16, "ef_construct": 100},
387                "optimizers": {"deleted_threshold": 0.2},
388                "quantization": {"scalar": {"type": "int8"}},
389            })
390        );
391    }
392
393    /// A `null` REST config section reads as absent (matches gRPC/edge, where
394    /// the key is simply omitted).
395    #[test]
396    fn null_config_sections_are_absent() {
397        let result = json!({
398            "config": {
399                "hnsw_config": null,
400                "optimizer_config": null,
401                "quantization_config": null,
402            },
403        });
404        let value = serde_json::to_value(schema_from_rest_result(&result)).unwrap();
405        assert!(value.get("hnsw").is_none());
406        assert!(value.get("optimizers").is_none());
407        assert!(value.get("quantization").is_none());
408    }
409}