1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
use std::collections::{BTreeSet, HashMap};
use std::sync::Arc;
use qql_core::ast::{self, Stmt, Value};
use qql_core::error::QqlError;
use qql_core::parser;
use qql_plan::plan_template;
use crate::backend::CollectionInfo;
use crate::executor::{ExecResponse, ExecutionReport, Executor, OnError};
/// A parsed and pre-compiled query template for repeated execution with different parameters.
#[derive(Debug, Clone)]
pub struct PreparedStatement {
pub(crate) sql: String,
pub(crate) stmt: Stmt,
pub(crate) planned: Option<qql_plan::PlannedOperation>,
pub(crate) named_params: BTreeSet<String>,
pub(crate) positional_count: usize,
/// Whether the template carries whole-point placeholders (`VALUES :p`).
pub(crate) has_point_params: bool,
/// Collection schema fetched once at [`Executor::prepare`] for templates
/// with point placeholders, reused by every point-splice execution
/// instead of re-fetching per call.
pub(crate) upsert_schema: Option<Arc<CollectionInfo>>,
}
impl PreparedStatement {
/// The original SQL query template string.
pub fn sql(&self) -> &str {
&self.sql
}
/// The parsed AST statement.
pub fn stmt(&self) -> &Stmt {
&self.stmt
}
/// Whether this prepared statement has a fast execution path: either a
/// pre-compiled planned operation template or whole-point placeholders
/// bound via the point-splice path.
pub fn is_planned(&self) -> bool {
self.planned.is_some() || self.has_point_params
}
/// Parameter names referenced in this statement template.
pub fn named_params(&self) -> &BTreeSet<String> {
&self.named_params
}
/// Total count of positional parameter placeholders (`?`) referenced in this statement template.
///
/// Positional parameters use 0-based indexing: the first `?` corresponds to index 0
/// (`params[0]`), the second to index 1 (`params[1]`), etc.
pub fn positional_count(&self) -> usize {
self.positional_count
}
}
impl Executor {
/// Prepare a query template for repeated execution with different parameters.
pub async fn prepare(&self, sql: &str) -> Result<PreparedStatement, QqlError> {
self.ensure_open()?;
let mut stmts = parser::Parser::parse_all(sql)?;
let stmt = if stmts.len() == 1 {
stmts.pop().expect("single stmt")
} else if stmts.is_empty() {
return Err(QqlError::validation(
"QQL-VALIDATION-EMPTY-SCRIPT",
"cannot prepare an empty query template",
None,
));
} else {
return Err(QqlError::validation(
"QQL-VALIDATION-MULTI-STMT",
"cannot prepare a multi-statement script; prepare one statement at a time",
None,
));
};
self.prepare_from_stmt(sql.to_string(), stmt).await
}
/// Bulk ingest helper: the canonical fast path for application ingest.
///
/// Prepares `UPSERT INTO <collection> VALUES :rows` once (schema fetched
/// once), then splices each `batch_size` chunk of point dicts through the
/// point-splice path. Each `rows` entry is one `{id, vector, …payload}`
/// point dict — the same shape as inline `VALUES {…}` rows.
///
/// The template is built as AST (never interpolated SQL), so collection
/// names are data, not syntax. `batch_size == 0` fails closed;
/// empty `rows` returns an empty `ok` report without I/O.
///
/// Peak live set is the caller's `rows` vec plus one in-flight batch:
/// each chunk moves (never clones) through bind, and the plan/proto
/// layers copy one batch at a time. For larger-than-memory datasets,
/// partition client-side into multiple `upsert_many` calls — each extra
/// call costs one metadata round-trip plus a template plan (no re-parse,
/// schema served from cache), negligible next to a round-trip per batch.
/// No streaming iterator API is provided: the harness and all current
/// callers already hold `rows` by construction, so one would add surface
/// without moving any measured number.
pub async fn upsert_many(
&self,
collection: &str,
rows: Vec<Value>,
batch_size: usize,
on_error: OnError,
) -> Result<ExecutionReport, QqlError> {
self.ensure_open()?;
if batch_size == 0 {
return Err(QqlError::validation(
"QQL-VALIDATION-UPSERT-BATCH",
"upsert_many batch_size must be >= 1",
None,
));
}
if rows.is_empty() {
return Ok(ExecutionReport::from_results(Vec::new()));
}
let stmt = Stmt::Upsert(Box::new(qql_core::ast::UpsertStmt {
collection: collection.to_string(),
points: vec![ast::PointEntry::Param("rows".to_string(), None)],
embedding: None,
embed: Vec::new(),
update_filter: None,
update_mode: None,
shard_key: None,
wait: None,
}));
// `prepare` re-checks openness; the template carries no user SQL.
let prepared = self
.prepare_from_stmt(format!("UPSERT INTO {collection} VALUES :rows"), stmt)
.await?;
let stop_on_error = matches!(on_error, OnError::Stop);
let mut results = Vec::new();
// Move (never clone) each chunk out of `rows` and straight through
// bind: the chunk sits in a one-shot slot behind the `Fn` lookup
// `bind_stmt` demands, so the point splice takes ownership with zero
// deep clones. Routing through `execute_prepared`'s generic
// `params.get(k).cloned()` map lookup instead would duplicate every
// vector element once more (~8ms per 10k 128-dim points, measured).
// `std::sync::Mutex` — not `RefCell` — keeps the future `Send`.
// `bind_stmt`'s public `Fn` bound is untouched.
let mut rows = rows.into_iter();
loop {
let chunk: Vec<Value> = rows.by_ref().take(batch_size).collect();
if chunk.is_empty() {
break;
}
let slot = std::sync::Mutex::new(Some(Value::List(chunk)));
let lookup = |name: &str| {
if name == "rows" {
slot.lock().expect("upsert_many param slot").take()
} else {
None
}
};
// Bypass `execute_prepared`: template and param are both built
// here, so its no-placeholder/unused-param checks hold by
// construction (`:rows` is the template's only param).
debug_assert!(prepared.named_params.contains("rows"));
match self
.execute_prepared_upsert_points(&prepared, &lookup, &[])
.await
{
Ok(resp) => results.push(resp),
Err(e) if !stop_on_error => results.push(ExecResponse {
ok: false,
operation: "UPSERT".to_string(),
message: e.to_string(),
data: None,
telemetry: None,
}),
Err(e) => return Err(e),
}
}
Ok(ExecutionReport::from_results(results))
}
/// Shared preparation body: schema resolution + template planning for an
/// already-parsed statement. [`Executor::prepare`] parses first;
/// [`Executor::upsert_many`] builds its template as AST (no SQL
/// interpolation, so collection names stay data).
pub(crate) async fn prepare_from_stmt(
&self,
sql: String,
stmt: Stmt,
) -> Result<PreparedStatement, QqlError> {
self.ensure_open()?;
let (named_params, positional_count) = qql_core::params::collect_statement_params(&stmt);
let has_point_params = qql_core::params::stmt_has_point_params(&stmt);
// Borrow probe first: a template that cannot plan needs no expensive
// preparation (schema I/O, embeddings, backend-mutating auto-create)
// and no whole-`Stmt` clone. Only clone + prepare when the probe says
// the template is plannable; outcomes match the old always-prepare
// probe in every case (`None` whenever either stage fails).
let planned = if plan_template(&stmt).is_ok() {
match self.prepare_statement(stmt.clone()).await {
Ok(prep) => plan_template(&prep).ok(),
Err(_) => None,
}
} else {
None
};
// Point-placeholder templates resolve bound points against this
// schema on every execution instead of re-fetching per call. Fetched
// once here; `None` (missing collection) disables the fast path and
// every execution takes the checking slow path instead. Cache-first:
// a hit costs no round trip; a miss probes existence (one GET) and
// fetches schema only when the collection exists, so a missing
// collection never pays for a fetch.
let upsert_schema = if has_point_params {
if let Stmt::Upsert(upsert) = &stmt {
if let Some(info) = self.peek_cached_collection_info(&upsert.collection).await {
Some(info)
} else if self
.client
.collection_exists(&upsert.collection)
.await
.unwrap_or(false)
{
self.get_cached_collection_info(&upsert.collection)
.await
.ok()
} else {
None
}
} else {
None
}
} else {
None
};
Ok(PreparedStatement {
sql,
stmt,
planned,
named_params,
positional_count,
has_point_params,
upsert_schema,
})
}
/// Execute a prepared statement with named parameters.
pub async fn execute_prepared(
&self,
prepared: &PreparedStatement,
params: &HashMap<String, Value>,
) -> Result<ExecResponse, QqlError> {
self.ensure_open()?;
if prepared.named_params.is_empty() && prepared.positional_count == 0 && !params.is_empty()
{
return Err(QqlError::validation(
"QQL-BIND-UNUSED-PARAMS",
"query has no parameter placeholders, but parameters were provided",
None,
));
}
for k in params.keys() {
if !prepared.named_params.contains(k) {
return Err(QqlError::validation(
"QQL-BIND-UNUSED-PARAMS",
format!("parameter ':{k}' was provided but is not used in the query template"),
None,
));
}
}
if prepared.has_point_params {
for required in &prepared.named_params {
if !params.contains_key(required) {
return Err(QqlError::validation(
"QQL-BIND-MISSING-PARAM",
format!("missing value for named parameter ':{required}'"),
None,
));
}
}
return self
.execute_prepared_upsert_points(prepared, &|k| params.get(k).cloned(), &[])
.await;
}
if let Some(ref template_op) = prepared.planned {
for required in &prepared.named_params {
let Some(val) = params.get(required) else {
return Err(QqlError::validation(
"QQL-BIND-MISSING-PARAM",
format!("missing value for named parameter ':{required}'"),
None,
));
};
if qql_plan::PlanVectorValue::from_value(val).is_none() {
return Err(QqlError::validation(
"QQL-BIND-INVALID-PARAMS",
format!("parameter ':{required}' must be a vector (list of floats)"),
None,
));
}
}
let mut op = template_op.clone();
op.bind_vector_params(
&|name| {
params
.get(name)
.and_then(qql_plan::PlanVectorValue::from_value)
},
&|_| None,
);
return self.with_timeout(self.dispatch_planned(&op)).await;
}
// Load-bearing clone: `prepared` is a reusable template behind a `&`
// receiver (callers execute it repeatedly with different params, and
// `upsert_many` reuses one template across chunks), while `bind_stmt`
// binds in place (`&mut Stmt`). The working copy must not alias the
// template — binding the template itself would corrupt every later
// execution. There is no owned single-use overload: the `&` contract
// is what the CLI and all multi-exec callers share.
let mut stmt = prepared.stmt.clone();
qql_core::params::bind_stmt(&mut stmt, |k| params.get(k).cloned(), &[])?;
self.execute_node(stmt).await
}
/// Execute a prepared statement with positional parameters (`?`).
///
/// Positional parameters use 0-based indexing: `params[0]` binds to the first `?`,
/// `params[1]` binds to the second `?`, up to `prepared.positional_count() - 1`.
///
/// Returns `QQL-BIND-UNUSED-PARAMS` if more parameters are supplied than `?` placeholders,
/// or `QQL-BIND-MISSING-PARAM` if fewer parameters are supplied.
pub async fn execute_prepared_positional(
&self,
prepared: &PreparedStatement,
params: &[Value],
) -> Result<ExecResponse, QqlError> {
self.ensure_open()?;
if prepared.positional_count == 0 && prepared.named_params.is_empty() && !params.is_empty()
{
return Err(QqlError::validation(
"QQL-BIND-UNUSED-PARAMS",
"query has no parameter placeholders, but positional parameters were provided",
None,
));
}
if params.len() > prepared.positional_count {
return Err(QqlError::validation(
"QQL-BIND-UNUSED-PARAMS",
format!(
"too many positional parameters provided: expected {}, but {} were provided",
prepared.positional_count,
params.len()
),
None,
));
}
if prepared.has_point_params {
if params.len() < prepared.positional_count {
return Err(QqlError::validation(
"QQL-BIND-MISSING-PARAM",
format!(
"missing value for positional parameter '?{}' (only {} parameter(s) provided)",
params.len() + 1,
params.len()
),
None,
));
}
return self
.execute_prepared_upsert_points(prepared, &|_| None, params)
.await;
}
if let Some(ref template_op) = prepared.planned {
if params.len() < prepared.positional_count {
return Err(QqlError::validation(
"QQL-BIND-MISSING-PARAM",
format!(
"missing value for positional parameter '?{}' (only {} parameter(s) provided)",
params.len() + 1,
params.len()
),
None,
));
}
for (idx, val) in params.iter().take(prepared.positional_count).enumerate() {
if qql_plan::PlanVectorValue::from_value(val).is_none() {
return Err(QqlError::validation(
"QQL-BIND-INVALID-PARAMS",
format!(
"positional parameter '?{}' must be a vector (list of floats)",
idx + 1
),
None,
));
}
}
let mut op = template_op.clone();
op.bind_vector_params(&|_| None, &|idx| {
params
.get(idx)
.and_then(qql_plan::PlanVectorValue::from_value)
});
return self.with_timeout(self.dispatch_planned(&op)).await;
}
// Load-bearing clone: same reusable-template contract as
// `execute_prepared` — `bind_stmt` mutates in place, so the per-exec
// working copy must not alias `prepared.stmt`.
let mut stmt = prepared.stmt.clone();
qql_core::params::bind_stmt(&mut stmt, |_| None, params)?;
self.execute_node(stmt).await
}
/// Point-splice fast path for upsert templates with whole-point
/// placeholders (`VALUES :p` / `VALUES ?`).
async fn execute_prepared_upsert_points(
&self,
prepared: &PreparedStatement,
lookup: &impl Fn(&str) -> Option<Value>,
positional: &[Value],
) -> Result<ExecResponse, QqlError> {
// Load-bearing clone: the point-splice template is shared across all
// `upsert_many` chunks (and across repeated `execute_prepared` calls),
// while the splice rebuilds the point vec in place. Each execution
// binds a fresh copy; the template keeps its placeholders.
let mut stmt = prepared.stmt.clone();
qql_core::params::bind_stmt(&mut stmt, lookup, positional)?;
let Stmt::Upsert(mut upsert) = stmt else {
return Err(QqlError::validation(
"QQL-BIND-INVALID-PARAMS",
"point parameters are only supported by UPSERT templates",
None,
));
};
if self.embedder.is_some() || upsert.embedding.is_some() || !upsert.embed.is_empty() {
return self.execute_node(Stmt::Upsert(upsert)).await;
}
let needs_slow_path = upsert.points.iter().any(|point| {
!matches!(
point,
ast::PointEntry::Inline(inline) if inline.vectors.is_some()
)
});
if needs_slow_path || prepared.upsert_schema.is_none() {
return self.execute_node(Stmt::Upsert(upsert)).await;
}
if let Some(info) = prepared.upsert_schema.as_ref() {
crate::executor::dml::upsert::map_unnamed_to_single_dense(&mut upsert, info);
self.validate_embedded_upsert(&upsert, info)?;
}
let request = qql_plan::mutation::lower_upsert_request(&upsert)?;
let wait = upsert
.wait
.unwrap_or(upsert.embedding.is_some() || !upsert.embed.is_empty());
let op = qql_plan::PlannedOperation::Upsert {
collection: upsert.collection.clone(),
request,
wait,
};
self.with_timeout(self.dispatch_planned(&op)).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use qql_core::parser::Parser;
/// The per-exec `prepared.stmt.clone()` is load-bearing, not waste:
/// `bind_stmt` binds in place, and the template behind `&PreparedStatement`
/// must survive repeated executions with different params. Binding two
/// clones must yield two correct statements while the template keeps its
/// placeholders — binding the template itself would corrupt the second
/// execution. This pins the `&`-receiver + clone contract against a
/// future "bind in place on the template" restructure.
#[test]
fn template_survives_repeated_cloned_binds() {
let stmt = Parser::parse("QUERY [0.1] FROM docs WHERE x = :x").unwrap();
let (named_params, positional_count) = qql_core::params::collect_statement_params(&stmt);
let template = PreparedStatement {
sql: "QUERY [0.1] FROM docs WHERE x = :x".to_string(),
stmt,
planned: None,
named_params,
positional_count,
has_point_params: false,
upsert_schema: None,
};
let bind = |v: i64| {
let mut work = template.stmt.clone();
qql_core::params::bind_stmt(&mut work, |k| (k == "x").then_some(Value::Int(v)), &[])
.unwrap();
work
};
assert_eq!(
qql_core::fmt::format_stmt(&bind(1)),
"QUERY [0.1] FROM docs WHERE x = 1"
);
assert_eq!(
qql_core::fmt::format_stmt(&bind(2)),
"QUERY [0.1] FROM docs WHERE x = 2"
);
// Template untouched: still carries the unbound placeholder.
assert!(qql_plan::ensure_no_unbound_params(&template.stmt).is_err());
}
}