qql 0.4.0

QQL runtime and executor for Qdrant REST and gRPC
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
use std::collections::{BTreeSet, HashMap};

use qql_core::ast::{self, Stmt, Value};
use qql_core::error::QqlError;
use qql_core::parser;
use qql_plan::plan_template;

use crate::backend::CollectionInfo;
use crate::executor::{ExecResponse, ExecutionReport, Executor, OnError};

/// A parsed and pre-compiled query template for repeated execution with different parameters.
#[derive(Debug, Clone)]
pub struct PreparedStatement {
    pub(crate) sql: String,
    pub(crate) stmt: Stmt,
    pub(crate) planned: Option<qql_plan::PlannedOperation>,
    pub(crate) named_params: BTreeSet<String>,
    pub(crate) positional_count: usize,
    /// Whether the template carries whole-point placeholders (`VALUES :p`).
    pub(crate) has_point_params: bool,
    /// Collection schema fetched once at [`Executor::prepare`] for templates
    /// with point placeholders, reused by every point-splice execution
    /// instead of re-fetching per call.
    pub(crate) upsert_schema: Option<CollectionInfo>,
}

impl PreparedStatement {
    /// The original SQL query template string.
    pub fn sql(&self) -> &str {
        &self.sql
    }

    /// The parsed AST statement.
    pub fn stmt(&self) -> &Stmt {
        &self.stmt
    }

    /// Whether this prepared statement has a fast execution path: either a
    /// pre-compiled planned operation template or whole-point placeholders
    /// bound via the point-splice path.
    pub fn is_planned(&self) -> bool {
        self.planned.is_some() || self.has_point_params
    }

    /// Parameter names referenced in this statement template.
    pub fn named_params(&self) -> &BTreeSet<String> {
        &self.named_params
    }

    /// Total count of positional parameter placeholders (`?`) referenced in this statement template.
    ///
    /// Positional parameters use 0-based indexing: the first `?` corresponds to index 0
    /// (`params[0]`), the second to index 1 (`params[1]`), etc.
    pub fn positional_count(&self) -> usize {
        self.positional_count
    }
}

impl Executor {
    /// Prepare a query template for repeated execution with different parameters.
    pub async fn prepare(&self, sql: &str) -> Result<PreparedStatement, QqlError> {
        self.ensure_open()?;
        let mut stmts = parser::Parser::parse_all(sql)?;
        let stmt = if stmts.len() == 1 {
            stmts.pop().expect("single stmt")
        } else if stmts.is_empty() {
            return Err(QqlError::validation(
                "QQL-VALIDATION-EMPTY-SCRIPT",
                "cannot prepare an empty query template",
                None,
            ));
        } else {
            return Err(QqlError::validation(
                "QQL-VALIDATION-MULTI-STMT",
                "cannot prepare a multi-statement script; prepare one statement at a time",
                None,
            ));
        };
        self.prepare_from_stmt(sql.to_string(), stmt).await
    }

    /// Bulk ingest helper: the canonical fast path for application ingest.
    ///
    /// Prepares `UPSERT INTO <collection> VALUES :rows` once (schema fetched
    /// once), then splices each `batch_size` chunk of point dicts through the
    /// point-splice path. Each `rows` entry is one `{id, vector, …payload}`
    /// point dict — the same shape as inline `VALUES {…}` rows.
    ///
    /// The template is built as AST (never interpolated SQL), so collection
    /// names are data, not syntax. `batch_size == 0` fails closed;
    /// empty `rows` returns an empty `ok` report without I/O.
    ///
    /// Peak live set is the caller's `rows` vec plus one in-flight batch:
    /// each chunk moves (never clones) through bind, and the plan/proto
    /// layers copy one batch at a time. For larger-than-memory datasets,
    /// partition client-side into multiple `upsert_many` calls — each extra
    /// call costs one metadata round-trip plus a template plan (no re-parse,
    /// schema served from cache), negligible next to a round-trip per batch.
    /// No streaming iterator API is provided: the harness and all current
    /// callers already hold `rows` by construction, so one would add surface
    /// without moving any measured number.
    pub async fn upsert_many(
        &self,
        collection: &str,
        rows: Vec<Value>,
        batch_size: usize,
        on_error: OnError,
    ) -> Result<ExecutionReport, QqlError> {
        self.ensure_open()?;
        if batch_size == 0 {
            return Err(QqlError::validation(
                "QQL-VALIDATION-UPSERT-BATCH",
                "upsert_many batch_size must be >= 1",
                None,
            ));
        }
        if rows.is_empty() {
            return Ok(ExecutionReport::from_results(Vec::new()));
        }
        let stmt = Stmt::Upsert(Box::new(qql_core::ast::UpsertStmt {
            collection: collection.to_string(),
            points: vec![ast::PointEntry::Param("rows".to_string(), None)],
            embedding: None,
            embed: Vec::new(),
            update_filter: None,
            update_mode: None,
            shard_key: None,
            wait: None,
        }));
        // `prepare` re-checks openness; the template carries no user SQL.
        let prepared = self
            .prepare_from_stmt(format!("UPSERT INTO {collection} VALUES :rows"), stmt)
            .await?;
        let stop_on_error = matches!(on_error, OnError::Stop);
        let mut results = Vec::new();
        // Move (never clone) each chunk out of `rows` and straight through
        // bind: the chunk sits in a one-shot slot behind the `Fn` lookup
        // `bind_stmt` demands, so the point splice takes ownership with zero
        // deep clones. Routing through `execute_prepared`'s generic
        // `params.get(k).cloned()` map lookup instead would duplicate every
        // vector element once more (~8ms per 10k 128-dim points, measured).
        // `std::sync::Mutex` — not `RefCell` — keeps the future `Send`.
        // `bind_stmt`'s public `Fn` bound is untouched.
        let mut rows = rows.into_iter();
        loop {
            let chunk: Vec<Value> = rows.by_ref().take(batch_size).collect();
            if chunk.is_empty() {
                break;
            }
            let slot = std::sync::Mutex::new(Some(Value::List(chunk)));
            let lookup = |name: &str| {
                if name == "rows" {
                    slot.lock().expect("upsert_many param slot").take()
                } else {
                    None
                }
            };
            // Bypass `execute_prepared`: template and param are both built
            // here, so its no-placeholder/unused-param checks hold by
            // construction (`:rows` is the template's only param).
            debug_assert!(prepared.named_params.contains("rows"));
            match self
                .execute_prepared_upsert_points(&prepared, &lookup, &[])
                .await
            {
                Ok(resp) => results.push(resp),
                Err(e) if !stop_on_error => results.push(ExecResponse {
                    ok: false,
                    operation: "UPSERT".to_string(),
                    message: e.to_string(),
                    data: None,
                    telemetry: None,
                }),
                Err(e) => return Err(e),
            }
        }
        Ok(ExecutionReport::from_results(results))
    }

    /// Shared preparation body: schema resolution + template planning for an
    /// already-parsed statement. [`Executor::prepare`] parses first;
    /// [`Executor::upsert_many`] builds its template as AST (no SQL
    /// interpolation, so collection names stay data).
    pub(crate) async fn prepare_from_stmt(
        &self,
        sql: String,
        stmt: Stmt,
    ) -> Result<PreparedStatement, QqlError> {
        self.ensure_open()?;
        let (named_params, positional_count) = qql_core::params::collect_statement_params(&stmt);
        let has_point_params = qql_core::params::stmt_has_point_params(&stmt);
        let planned = match self.prepare_statement(stmt.clone()).await {
            Ok(prep) => plan_template(&prep).ok(),
            Err(_) => None,
        };
        // Point-placeholder templates resolve bound points against this
        // schema on every execution instead of re-fetching per call. Fetched
        // once here; `None` (missing collection) disables the fast path and
        // every execution takes the checking slow path instead.
        let upsert_schema = if has_point_params {
            if let Stmt::Upsert(upsert) = &stmt
                && self
                    .client
                    .collection_exists(&upsert.collection)
                    .await
                    .unwrap_or(false)
            {
                self.get_cached_collection_info(&upsert.collection)
                    .await
                    .ok()
            } else {
                None
            }
        } else {
            None
        };
        Ok(PreparedStatement {
            sql,
            stmt,
            planned,
            named_params,
            positional_count,
            has_point_params,
            upsert_schema,
        })
    }

    /// Execute a prepared statement with named parameters.
    pub async fn execute_prepared(
        &self,
        prepared: &PreparedStatement,
        params: &HashMap<String, Value>,
    ) -> Result<ExecResponse, QqlError> {
        self.ensure_open()?;
        if prepared.named_params.is_empty() && prepared.positional_count == 0 && !params.is_empty()
        {
            return Err(QqlError::validation(
                "QQL-BIND-UNUSED-PARAMS",
                "query has no parameter placeholders, but parameters were provided",
                None,
            ));
        }
        for k in params.keys() {
            if !prepared.named_params.contains(k) {
                return Err(QqlError::validation(
                    "QQL-BIND-UNUSED-PARAMS",
                    format!("parameter ':{k}' was provided but is not used in the query template"),
                    None,
                ));
            }
        }
        if prepared.has_point_params {
            for required in &prepared.named_params {
                if !params.contains_key(required) {
                    return Err(QqlError::validation(
                        "QQL-BIND-MISSING-PARAM",
                        format!("missing value for named parameter ':{required}'"),
                        None,
                    ));
                }
            }
            return self
                .execute_prepared_upsert_points(prepared, &|k| params.get(k).cloned(), &[])
                .await;
        }
        if let Some(ref template_op) = prepared.planned {
            for required in &prepared.named_params {
                let Some(val) = params.get(required) else {
                    return Err(QqlError::validation(
                        "QQL-BIND-MISSING-PARAM",
                        format!("missing value for named parameter ':{required}'"),
                        None,
                    ));
                };
                if qql_plan::PlanVectorValue::from_value(val).is_none() {
                    return Err(QqlError::validation(
                        "QQL-BIND-INVALID-PARAMS",
                        format!("parameter ':{required}' must be a vector (list of floats)"),
                        None,
                    ));
                }
            }
            let mut op = template_op.clone();
            op.bind_vector_params(
                &|name| {
                    params
                        .get(name)
                        .and_then(qql_plan::PlanVectorValue::from_value)
                },
                &|_| None,
            );
            return self.dispatch_planned(&op).await;
        }
        let mut stmt = prepared.stmt.clone();
        qql_core::params::bind_stmt(&mut stmt, |k| params.get(k).cloned(), &[])?;
        self.execute_node(stmt).await
    }

    /// Execute a prepared statement with positional parameters (`?`).
    ///
    /// Positional parameters use 0-based indexing: `params[0]` binds to the first `?`,
    /// `params[1]` binds to the second `?`, up to `prepared.positional_count() - 1`.
    ///
    /// Returns `QQL-BIND-UNUSED-PARAMS` if more parameters are supplied than `?` placeholders,
    /// or `QQL-BIND-MISSING-PARAM` if fewer parameters are supplied.
    pub async fn execute_prepared_positional(
        &self,
        prepared: &PreparedStatement,
        params: &[Value],
    ) -> Result<ExecResponse, QqlError> {
        self.ensure_open()?;
        if prepared.positional_count == 0 && prepared.named_params.is_empty() && !params.is_empty()
        {
            return Err(QqlError::validation(
                "QQL-BIND-UNUSED-PARAMS",
                "query has no parameter placeholders, but positional parameters were provided",
                None,
            ));
        }
        if params.len() > prepared.positional_count {
            return Err(QqlError::validation(
                "QQL-BIND-UNUSED-PARAMS",
                format!(
                    "too many positional parameters provided: expected {}, but {} were provided",
                    prepared.positional_count,
                    params.len()
                ),
                None,
            ));
        }
        if prepared.has_point_params {
            if params.len() < prepared.positional_count {
                return Err(QqlError::validation(
                    "QQL-BIND-MISSING-PARAM",
                    format!(
                        "missing value for positional parameter '?{}' (only {} parameter(s) provided)",
                        params.len() + 1,
                        params.len()
                    ),
                    None,
                ));
            }
            return self
                .execute_prepared_upsert_points(prepared, &|_| None, params)
                .await;
        }
        if let Some(ref template_op) = prepared.planned {
            if params.len() < prepared.positional_count {
                return Err(QqlError::validation(
                    "QQL-BIND-MISSING-PARAM",
                    format!(
                        "missing value for positional parameter '?{}' (only {} parameter(s) provided)",
                        params.len() + 1,
                        params.len()
                    ),
                    None,
                ));
            }
            for (idx, val) in params.iter().take(prepared.positional_count).enumerate() {
                if qql_plan::PlanVectorValue::from_value(val).is_none() {
                    return Err(QqlError::validation(
                        "QQL-BIND-INVALID-PARAMS",
                        format!(
                            "positional parameter '?{}' must be a vector (list of floats)",
                            idx + 1
                        ),
                        None,
                    ));
                }
            }
            let mut op = template_op.clone();
            op.bind_vector_params(&|_| None, &|idx| {
                params
                    .get(idx)
                    .and_then(qql_plan::PlanVectorValue::from_value)
            });
            return self.dispatch_planned(&op).await;
        }
        let mut stmt = prepared.stmt.clone();
        qql_core::params::bind_stmt(&mut stmt, |_| None, params)?;
        self.execute_node(stmt).await
    }

    /// Point-splice fast path for upsert templates with whole-point
    /// placeholders (`VALUES :p` / `VALUES ?`).
    async fn execute_prepared_upsert_points(
        &self,
        prepared: &PreparedStatement,
        lookup: &impl Fn(&str) -> Option<Value>,
        positional: &[Value],
    ) -> Result<ExecResponse, QqlError> {
        let mut stmt = prepared.stmt.clone();
        qql_core::params::bind_stmt(&mut stmt, lookup, positional)?;
        let Stmt::Upsert(mut upsert) = stmt else {
            return Err(QqlError::validation(
                "QQL-BIND-INVALID-PARAMS",
                "point parameters are only supported by UPSERT templates",
                None,
            ));
        };
        if self.embedder.is_some() || upsert.embedding.is_some() || !upsert.embed.is_empty() {
            return self.execute_node(Stmt::Upsert(upsert)).await;
        }
        let needs_slow_path = upsert.points.iter().any(|point| {
            !matches!(
                point,
                ast::PointEntry::Inline(inline) if inline.vectors.is_some()
            )
        });
        if needs_slow_path || prepared.upsert_schema.is_none() {
            return self.execute_node(Stmt::Upsert(upsert)).await;
        }
        if let Some(info) = prepared.upsert_schema.as_ref() {
            crate::executor::dml::upsert::map_unnamed_to_single_dense(&mut upsert, info);
            self.validate_embedded_upsert(&upsert, info)?;
        }
        let request = qql_plan::mutation::lower_upsert_request(&upsert);
        let wait = upsert
            .wait
            .unwrap_or(upsert.embedding.is_some() || !upsert.embed.is_empty());
        let op = qql_plan::PlannedOperation::Upsert {
            collection: upsert.collection.clone(),
            request,
            wait,
        };
        self.dispatch_planned(&op).await
    }
}