Skip to main content

qql_edge/
lib.rs

1//! # QQL Edge — zero-network vector search
2//!
3//! Combines [fastembed-rs] for local ONNX embedding inference with
4//! [qdrant-edge] for in-process HNSW vector search. No network hops,
5//! no external services, no API keys — unless you choose an external provider.
6//!
7//! ## Embedder options
8//!
9//! | Function | Embedder | Network? |
10//! |---|---|---|
11//! | [`local_executor`] | fastembed (ONNX, local CPU) | ❌ none |
12//! | [`local_executor_with_options`] | fastembed + model/cache selection | ❌ none |
13//! | [`http_executor`] | OpenAI-compatible HTTP endpoint | ✅ provider only |
14//! | [`custom_executor`] | Any `Arc<dyn Embedder>` | up to you |
15//!
16//! ## Quick start
17//!
18//! ```rust,no_run
19//! use qql_edge::local_executor;
20//! use qql::executor::OnError;
21//!
22//! # async fn example() -> Result<(), qql_core::error::QqlError> {
23//! let mut executor = local_executor("/tmp/qql-edge-data", false)?;
24//! let resp = executor.execute("CREATE COLLECTION docs HYBRID", OnError::Stop).await?;
25//! # Ok(())
26//! # }
27//! ```
28//!
29//! ## JSON scope ("envelope removal")
30//!
31//! Responses are fully typed (`ExecData`, no JSON envelope). `serde_json::Value`
32//! remains only where the engine API requires it: point payloads
33//! (`PointStruct::new`), typed-spec introspection maps, the `turbo`
34//! quantization builder map, and formula `DEFAULTS` (edge formulas are JSON).
35//! `Value`-based parity helpers are test-only (`#[cfg(test)]`).
36//!
37//! [fastembed-rs]: https://crates.io/crates/fastembed
38//! [qdrant-edge]: https://crates.io/crates/qdrant-edge
39
40mod backend;
41mod bootstrap;
42#[cfg(feature = "fastembed-local")]
43mod embedder;
44
45pub use backend::EdgeQdrant;
46pub use bootstrap::{ShardSummary, inspect_shard, unpack_snapshot};
47#[cfg(feature = "fastembed-local")]
48pub use embedder::{
49    EmbeddingModelInfo, FastEmbedder, FastEmbedderOptions, list_embedding_models,
50    resolve_embedding_model, resolve_image_model, resolve_multi_model,
51};
52
53use qql::config::QqlConfig;
54use qql::embedder::Embedder;
55use qql::executor::Executor;
56use std::path::PathBuf;
57use std::sync::Arc;
58
59/// Options for [`local_executor_with_options`].
60#[derive(Debug, Clone, Default)]
61pub struct LocalExecutorOptions {
62    /// Store payloads on disk (default: `true` when constructed via bindings;
63    /// this struct defaults to `false` for Rust ergonomics matching the
64    /// historical `local_executor(path, false)` tests).
65    pub on_disk_payload: bool,
66    /// Write-ahead-log segment capacity in bytes. qdrant-edge pre-allocates
67    /// each WAL segment to this size (default 32 MiB), which dominates the
68    /// on-disk footprint of small embedded shards.
69    ///
70    /// Seeds the value when a shard is created or when it exists without a
71    /// persisted WAL capacity; a capacity already persisted in
72    /// `edge_config.json` wins, so later opens cannot ratchet the shard's
73    /// config. Hosts configure it in MiB — [`wal_segment_capacity_bytes`]
74    /// performs the conversion: the CLI via `--wal-segment-mb`, Python via
75    /// `wal_segment_mb`, and Node via `walSegmentMb`.
76    pub wal_segment_capacity: Option<usize>,
77    /// Local ONNX dense model name. See [`resolve_embedding_model`] for accepted forms.
78    /// `None` → default `BGESmallENV15` (384-d).
79    #[cfg(feature = "fastembed-local")]
80    pub model: Option<String>,
81    /// Offline sparse model (SPLADE or BGE-M3 via `SparseTextEmbedding`).
82    /// e.g. `"splade"`, `"bge-m3"`. When set, sparse embedding uses real ONNX
83    /// inference. `None` → local wire-compatible BM25 (Qdrant
84    /// `qdrant/bm25`-identical token IDs) for sparse requests. `"bm25"` names
85    /// the built-in encoder, not an ONNX model: leave this unset to use it.
86    #[cfg(feature = "fastembed-local")]
87    pub sparse_model: Option<String>,
88    /// Offline multivector model (BGE-M3 ColBERT). e.g. `"bge-m3"`.
89    /// When set, `embed_multi` / multivector RERANK work with no network.
90    #[cfg(feature = "fastembed-local")]
91    pub multi_model: Option<String>,
92    /// Offline image / CLIP vision model. e.g. `"clip-vision"` / `ClipVitB32`.
93    /// Pair with dense CLIP text (`model: Some("ClipVitB32".into())`) for multimodal.
94    #[cfg(feature = "fastembed-local")]
95    pub image_model: Option<String>,
96    /// Offline cross-encoder (`bge-reranker-base`, `BGERerankerBase`, …).
97    #[cfg(feature = "fastembed-local")]
98    pub reranker_model: Option<String>,
99    /// Override fastembed model cache directory.
100    #[cfg(feature = "fastembed-local")]
101    pub cache_dir: Option<PathBuf>,
102    /// Show HuggingFace download progress bars (default: `false`).
103    #[cfg(feature = "fastembed-local")]
104    pub show_download_progress: bool,
105    /// Client-side BM25 `k1` for the local wire-compatible document encoder
106    /// (used when no offline sparse model is configured). `None` keeps the
107    /// Qdrant `qdrant/bm25` default (`1.2`). Write-path only; invalid values
108    /// fail closed with `QQL-VALIDATION-CONFIG` at executor construction.
109    #[cfg(feature = "fastembed-local")]
110    pub bm25_k1: Option<f64>,
111    /// Client-side BM25 `b` (`[0, 1]`); `None` keeps the Qdrant default
112    /// (`0.75`). See [`Self::bm25_k1`].
113    #[cfg(feature = "fastembed-local")]
114    pub bm25_b: Option<f64>,
115    /// Client-side BM25 expected average document length in tokens; `None`
116    /// keeps the Qdrant default (`256`). See [`Self::bm25_k1`].
117    #[cfg(feature = "fastembed-local")]
118    pub bm25_avg_len: Option<f64>,
119    /// BM25 text-processing language (Qdrant name/alias); `None` keeps
120    /// English. Forwards to the engine (drives its stopwords/stemmer).
121    #[cfg(feature = "fastembed-local")]
122    pub bm25_language: Option<String>,
123    /// BM25 tokenizer (`"word"`, `"whitespace"`, `"prefix"`,
124    /// `"multilingual"`); `None` keeps `"word"`.
125    #[cfg(feature = "fastembed-local")]
126    pub bm25_tokenizer: Option<String>,
127    /// Lowercase before matching; `None` keeps `true`.
128    #[cfg(feature = "fastembed-local")]
129    pub bm25_lowercase: Option<bool>,
130    /// Lucene ASCII folding before lowercasing; `None` keeps `false`.
131    #[cfg(feature = "fastembed-local")]
132    pub bm25_ascii_folding: Option<bool>,
133    /// Drop tokens shorter than this (chars); `None` keeps no minimum.
134    #[cfg(feature = "fastembed-local")]
135    pub bm25_min_token_len: Option<usize>,
136    /// Drop over-long tokens on the document path (chars); `None` keeps no
137    /// maximum.
138    #[cfg(feature = "fastembed-local")]
139    pub bm25_max_token_len: Option<usize>,
140    /// Custom stopwords **replacing** the language default (`None` keeps it;
141    /// `Some(vec![])` disables filtering).
142    #[cfg(feature = "fastembed-local")]
143    pub bm25_stopwords: Option<Vec<String>>,
144    /// Stemmer override (`None` = language default; `Some("none")` disables).
145    #[cfg(feature = "fastembed-local")]
146    pub bm25_stemmer: Option<String>,
147    /// Additional language stopword lists merged with `bm25_stopwords`.
148    #[cfg(feature = "fastembed-local")]
149    pub bm25_stopwords_languages: Option<Vec<String>>,
150}
151
152/// Convert a MiB WAL segment capacity into the byte count
153/// [`LocalExecutorOptions::wal_segment_capacity`] expects.
154///
155/// Host surfaces configure the knob in whole MiB (CLI `--wal-segment-mb`,
156/// Python `wal_segment_mb`, Node `walSegmentMb`). `None` keeps the engine
157/// default (32 MiB); zero or a value that overflows `usize` bytes fails closed
158/// with `QQL-VALIDATION-CONFIG` instead of silently producing a nonsensical
159/// WAL capacity.
160pub fn wal_segment_capacity_bytes(
161    mb: Option<u64>,
162) -> Result<Option<usize>, qql_core::error::QqlError> {
163    match mb {
164        None => Ok(None),
165        Some(0) => Err(qql_core::error::QqlError::validation(
166            "QQL-VALIDATION-CONFIG",
167            "wal_segment_mb must be greater than zero; omit it for the qdrant-edge 32 MiB default",
168            None,
169        )),
170        Some(mb) => usize::try_from(mb)
171            .ok()
172            .and_then(|mb| mb.checked_mul(1024 * 1024))
173            .map(Some)
174            .ok_or_else(|| {
175                qql_core::error::QqlError::validation(
176                    "QQL-VALIDATION-CONFIG",
177                    "wal_segment_mb is too large for this platform",
178                    None,
179                )
180            }),
181    }
182}
183
184/// Build a fully-local [`Executor`] backed by fastembed-rs and qdrant-edge.
185///
186/// Uses the default embedding model (`BGESmallENV15`, 384-d). Prefer
187/// [`local_executor_with_options`] when you need a different model or cache dir.
188///
189/// No network calls are made at inference time — embedding runs on-device via ONNX.
190/// The model is downloaded from HuggingFace (when not already cached on disk)
191/// and loaded before this function returns, so the first construction may
192/// block for minutes; construct once and share the executor.
193#[cfg(feature = "fastembed-local")]
194pub fn local_executor(
195    data_dir: impl Into<PathBuf>,
196    on_disk_payload: bool,
197) -> Result<Executor, qql_core::error::QqlError> {
198    local_executor_with_options(
199        data_dir,
200        LocalExecutorOptions {
201            on_disk_payload,
202            ..Default::default()
203        },
204    )
205}
206
207/// Build a fully-local [`Executor`] with explicit model / cache options.
208///
209/// Like [`local_executor`], every configured model slot is downloaded (when
210/// needed) and loaded synchronously before this returns. Loaded sessions are
211/// shared process-wide and released when the last embedder using them drops.
212#[cfg(feature = "fastembed-local")]
213pub fn local_executor_with_options(
214    data_dir: impl Into<PathBuf>,
215    opts: LocalExecutorOptions,
216) -> Result<Executor, qql_core::error::QqlError> {
217    let client = Box::new(
218        EdgeQdrant::new(data_dir, opts.on_disk_payload)
219            .with_wal_segment_capacity(opts.wal_segment_capacity),
220    );
221    let embedder = FastEmbedder::try_with_options(FastEmbedderOptions {
222        model: opts.model,
223        sparse_model: opts.sparse_model,
224        multi_model: opts.multi_model,
225        image_model: opts.image_model,
226        reranker_model: opts.reranker_model,
227        cache_dir: opts.cache_dir,
228        show_download_progress: opts.show_download_progress,
229        bm25_k1: opts.bm25_k1,
230        bm25_b: opts.bm25_b,
231        bm25_avg_len: opts.bm25_avg_len,
232        bm25_language: opts.bm25_language,
233        bm25_tokenizer: opts.bm25_tokenizer,
234        bm25_lowercase: opts.bm25_lowercase,
235        bm25_ascii_folding: opts.bm25_ascii_folding,
236        bm25_min_token_len: opts.bm25_min_token_len,
237        bm25_max_token_len: opts.bm25_max_token_len,
238        bm25_stopwords: opts.bm25_stopwords,
239        bm25_stemmer: opts.bm25_stemmer,
240        bm25_stopwords_languages: opts.bm25_stopwords_languages,
241    })?;
242
243    // Pin collection vector size to the actual model dimension. Without this,
244    // CREATE COLLECTION HYBRID always falls back to the hard-coded 384 and any
245    // non-default model (768 / 1024-d) silently dimension-mismatches on upsert.
246    let multi_dim = embedder.multi_dimension().unwrap_or(0);
247    let image_dim = embedder.image_dimension().unwrap_or(0);
248    let config = QqlConfig {
249        inference_mode: "local".to_string(),
250        embedding_dimension: embedder.dimension(),
251        embedding_model: Some(embedder.model_name().to_string()),
252        sparse_inference_model: embedder.sparse_model_code().map(str::to_string),
253        multi_embedding_model: embedder.multi_model_code().map(str::to_string),
254        multi_embedding_dimension: multi_dim,
255        image_embedding_model: embedder.image_model_code().map(str::to_string),
256        image_embedding_dimension: image_dim,
257        rerank_model: embedder.reranker_model_code().map(str::to_string),
258        ..Default::default()
259    };
260
261    let embedder = Some(Arc::new(embedder) as Arc<dyn Embedder>);
262    Ok(Executor::with_embedder(client, Some(config), embedder))
263}
264
265/// Build an edge [`Executor`] that calls an external OpenAI-compatible embedding
266/// endpoint instead of running fastembed locally.
267///
268/// Works with: OpenAI, Ollama (`/v1/embeddings`), Cohere, Together AI,
269/// Mistral, and any other provider that follows the OpenAI embeddings spec.
270///
271/// - `endpoint` — full URL, e.g. `"https://api.openai.com/v1/embeddings"` or
272///   `"http://localhost:11434/v1/embeddings"` for local Ollama.
273/// - `api_key` — Bearer token. Pass `""` for unauthenticated local providers.
274/// - `model` — model name sent in the request body, e.g. `"text-embedding-3-small"`.
275/// - `dimension` — expected output dimension. Must match what the model returns.
276#[cfg(feature = "http-embedding")]
277pub fn http_executor(
278    data_dir: impl Into<PathBuf>,
279    on_disk_payload: bool,
280    endpoint: impl Into<String>,
281    api_key: impl Into<String>,
282    model: impl Into<String>,
283    dimension: usize,
284) -> Result<Executor, qql_core::error::QqlError> {
285    http_executor_with_options(
286        data_dir,
287        on_disk_payload,
288        qql::embedder::HttpEmbedderOptions {
289            endpoint: endpoint.into(),
290            api_key: api_key.into(),
291            model: model.into(),
292            dimension,
293            ..Default::default()
294        },
295    )
296}
297
298/// Edge executor with OpenAI-compatible dense + optional multi/ColBERT endpoints.
299#[cfg(feature = "http-embedding")]
300#[allow(clippy::too_many_arguments)]
301pub fn http_executor_with_multi(
302    data_dir: impl Into<PathBuf>,
303    on_disk_payload: bool,
304    endpoint: impl Into<String>,
305    api_key: impl Into<String>,
306    model: impl Into<String>,
307    dimension: usize,
308    multi_endpoint: Option<String>,
309    multi_api_key: Option<String>,
310    multi_model: Option<String>,
311    multi_dimension: usize,
312) -> Result<Executor, qql_core::error::QqlError> {
313    http_executor_with_options(
314        data_dir,
315        on_disk_payload,
316        qql::embedder::HttpEmbedderOptions {
317            endpoint: endpoint.into(),
318            api_key: api_key.into(),
319            model: model.into(),
320            dimension,
321            multi_endpoint,
322            multi_api_key,
323            multi_model,
324            multi_dimension,
325            ..Default::default()
326        },
327    )
328}
329
330/// Edge executor from full [`qql::embedder::HttpEmbedderOptions`] (dense + multi + image/CLIP).
331#[cfg(feature = "http-embedding")]
332pub fn http_executor_with_options(
333    data_dir: impl Into<PathBuf>,
334    on_disk_payload: bool,
335    opts: qql::embedder::HttpEmbedderOptions,
336) -> Result<Executor, qql_core::error::QqlError> {
337    http_executor_with_options_and_wal(data_dir, on_disk_payload, None, opts)
338}
339
340/// Like [`http_executor_with_options`], with an explicit WAL segment capacity
341/// (bytes) applied to every shard the executor opens or creates; `None` keeps
342/// the engine default (32 MiB) for new shards and the persisted value for
343/// existing ones.
344#[cfg(feature = "http-embedding")]
345pub fn http_executor_with_options_and_wal(
346    data_dir: impl Into<PathBuf>,
347    on_disk_payload: bool,
348    wal_segment_capacity: Option<usize>,
349    opts: qql::embedder::HttpEmbedderOptions,
350) -> Result<Executor, qql_core::error::QqlError> {
351    let client = Box::new(
352        EdgeQdrant::new(data_dir, on_disk_payload).with_wal_segment_capacity(wal_segment_capacity),
353    );
354    let config = QqlConfig {
355        inference_mode: "local".to_string(),
356        embedding_dimension: opts.dimension,
357        embedding_model: Some(opts.model.clone()),
358        embedding_endpoint: None, // edge uses the Arc embedder, not config probing
359        multi_embedding_endpoint: opts.multi_endpoint.clone(),
360        multi_embedding_api_key: opts.multi_api_key.clone(),
361        multi_embedding_model: opts.multi_model.clone(),
362        multi_embedding_dimension: opts.multi_dimension,
363        image_embedding_endpoint: opts.image_endpoint.clone(),
364        image_embedding_api_key: opts.image_api_key.clone(),
365        image_embedding_model: opts.image_model.clone(),
366        image_embedding_dimension: opts.image_dimension,
367        rerank_endpoint: opts.rerank_endpoint.clone(),
368        rerank_api_key: opts.rerank_api_key.clone(),
369        rerank_model: opts.rerank_model.clone(),
370        ..Default::default()
371    };
372    let embedder = qql::embedder::HttpEmbedder::try_with_options(opts)?;
373    Ok(Executor::with_embedder(
374        client,
375        Some(config),
376        Some(Arc::new(embedder) as Arc<dyn Embedder>),
377    ))
378}
379
380/// Build an edge [`Executor`] with a fully custom [`Embedder`].
381///
382/// Use this to plug in GPU-backed embedders, caching layers, ensemble
383/// embedders, or any other custom implementation.
384///
385/// **Callers must set `embedding_dimension` on a [`QqlConfig`] themselves** if
386/// they rely on `CREATE COLLECTION HYBRID` auto-sizing — pass the config via
387/// [`Executor::with_embedder`] if the defaults (384) are wrong for your model.
388pub fn custom_executor(
389    data_dir: impl Into<PathBuf>,
390    on_disk_payload: bool,
391    embedder: Arc<dyn Embedder>,
392) -> Result<Executor, qql_core::error::QqlError> {
393    custom_executor_with_dimension(data_dir, on_disk_payload, embedder, None)
394}
395
396/// Build an edge [`Executor`] with a custom embedder and an optional explicit
397/// dense dimension. The embedder-reported dimension takes precedence.
398pub fn custom_executor_with_dimension(
399    data_dir: impl Into<PathBuf>,
400    on_disk_payload: bool,
401    embedder: Arc<dyn Embedder>,
402    dimension: Option<usize>,
403) -> Result<Executor, qql_core::error::QqlError> {
404    custom_executor_with_storage(data_dir, on_disk_payload, None, embedder, dimension)
405}
406
407/// Like [`custom_executor_with_dimension`], with an explicit WAL segment
408/// capacity (bytes) applied to every shard the executor opens or creates;
409/// `None` keeps the engine default.
410pub fn custom_executor_with_storage(
411    data_dir: impl Into<PathBuf>,
412    on_disk_payload: bool,
413    wal_segment_capacity: Option<usize>,
414    embedder: Arc<dyn Embedder>,
415    dimension: Option<usize>,
416) -> Result<Executor, qql_core::error::QqlError> {
417    let client = Box::new(
418        EdgeQdrant::new(data_dir, on_disk_payload).with_wal_segment_capacity(wal_segment_capacity),
419    );
420    let config = QqlConfig {
421        inference_mode: "local".to_string(),
422        embedding_dimension: embedder.dimension().or(dimension).unwrap_or(0),
423        ..Default::default()
424    };
425    Ok(Executor::with_embedder(
426        client,
427        Some(config),
428        Some(embedder),
429    ))
430}
431
432#[cfg(test)]
433mod tests {
434    use super::*;
435    use async_trait::async_trait;
436    use qql::embedder::Embedder;
437    use qql::executor::OnError;
438    use qql_core::error::QqlError;
439    use qql_embed::sparse::SparseVector;
440
441    struct TestEmbedder;
442
443    #[async_trait]
444    impl Embedder for TestEmbedder {
445        fn dimension(&self) -> Option<usize> {
446            Some(3)
447        }
448
449        async fn embed_dense(&self, _text: &str, _model: &str) -> Result<Vec<f32>, QqlError> {
450            Ok(vec![1.0, 0.0, 0.0])
451        }
452
453        async fn embed_sparse_query(
454            &self,
455            _text: &str,
456            _model: &str,
457        ) -> Result<SparseVector, QqlError> {
458            Ok(SparseVector {
459                indices: vec![1],
460                values: vec![1.0],
461            })
462        }
463
464        async fn embed_sparse_document(
465            &self,
466            _text: &str,
467            _model: &str,
468        ) -> Result<SparseVector, QqlError> {
469            Ok(SparseVector {
470                indices: vec![1],
471                values: vec![1.0],
472            })
473        }
474    }
475
476    #[test]
477    #[cfg(feature = "fastembed-local")]
478    fn local_executor_options_default_has_no_sparse_model() {
479        let opts = LocalExecutorOptions::default();
480        assert!(opts.sparse_model.is_none());
481    }
482
483    #[test]
484    #[cfg(feature = "fastembed-local")]
485    fn local_executor_options_with_sparse_model() {
486        let opts = LocalExecutorOptions {
487            sparse_model: Some("splade".into()),
488            ..Default::default()
489        };
490        assert_eq!(opts.sparse_model.as_deref(), Some("splade"));
491    }
492
493    #[test]
494    #[cfg(feature = "fastembed-local")]
495    fn local_executor_options_bm25_defaults_are_unset() {
496        let opts = LocalExecutorOptions::default();
497        assert_eq!(opts.bm25_k1, None);
498        assert_eq!(opts.bm25_b, None);
499        assert_eq!(opts.bm25_avg_len, None);
500    }
501
502    /// Executor-level end-to-end: BM25 parameters configured on the embedder
503    /// must land in the sparse vector qdrant-edge actually stores for a TEXT
504    /// upsert, matching `qql_embed::sparse::embed_document_with_params`.
505    ///
506    /// Uses the edge HTTP executor so no ONNX model is needed: dense/multi
507    /// inference goes to the (unused) endpoint, while sparse document encoding
508    /// is the local wire-compatible BM25 path.
509    #[test]
510    #[cfg(feature = "http-embedding")]
511    fn configured_bm25_params_land_in_stored_sparse_vector() {
512        use qql::embedder::HttpEmbedderOptions;
513        use qql::executor::OnError;
514        use qql_plan::{PlanVectorStruct, PlanVectorValue};
515
516        let runtime = tokio::runtime::Builder::new_current_thread()
517            .enable_all()
518            .build()
519            .expect("test runtime");
520        runtime.block_on(async {
521            let data_dir =
522                std::env::temp_dir().join(format!("qql-edge-bm25-{}", std::process::id()));
523            let _ = std::fs::remove_dir_all(&data_dir);
524
525            let params = qql_embed::Bm25Params::new(2.0, 0.5, 4.0).expect("valid params");
526            let executor = http_executor_with_options_and_wal(
527                &data_dir,
528                false,
529                None,
530                HttpEmbedderOptions {
531                    endpoint: "http://127.0.0.1:9/v1/embeddings".to_string(),
532                    api_key: String::new(),
533                    model: "unused-dense".to_string(),
534                    dimension: 3,
535                    bm25_k1: Some(params.k1()),
536                    bm25_b: Some(params.b()),
537                    bm25_avg_len: Some(params.avg_len()),
538                    ..Default::default()
539                },
540            )
541            .expect("http edge executor");
542
543            let report = executor
544                .execute("CREATE COLLECTION bm25_docs (sparse SPARSE)", OnError::Stop)
545                .await
546                .expect("create collection");
547            assert!(report.ok, "create failed: {report:?}");
548
549            let text = "cat sat mat cat";
550            let report = executor
551                .execute(
552                    &format!("UPSERT INTO bm25_docs VALUES {{id: 1, text: '{text}'}}"),
553                    OnError::Stop,
554                )
555                .await
556                .expect("upsert");
557            assert!(report.ok, "upsert failed: {report:?}");
558
559            let report = executor
560                .execute(
561                    "QUERY POINTS (1) FROM bm25_docs WITH VECTOR true",
562                    OnError::Stop,
563                )
564                .await
565                .expect("point lookup");
566            assert!(report.ok, "lookup failed: {report:?}");
567            let hits = report.results[0].hits_ref().expect("hits");
568            let vector = hits[0].vector.as_ref().expect("stored vector");
569            let stored = match vector {
570                PlanVectorStruct::Single(v) => v,
571                PlanVectorStruct::Named(map) => map.get("sparse").expect("sparse named vector"),
572            };
573            let (indices, values) = match stored {
574                PlanVectorValue::Sparse { indices, values } => (indices, values),
575                other => panic!("expected sparse vector, got {other:?}"),
576            };
577
578            let expected = qql_embed::sparse::embed_document_with_params(text, &params);
579            assert_eq!(*indices, expected.indices, "stored indices must match");
580            assert_eq!(values.len(), expected.values.len());
581            for (got, want) in values.iter().zip(&expected.values) {
582                assert!(
583                    (got - want).abs() < 1e-6,
584                    "stored BM25 weight {got} != configured {want}"
585                );
586            }
587
588            // The configured k1/b/avg_len genuinely differ from the defaults.
589            let default = qql_embed::sparse::embed_document(text);
590            assert_ne!(*values, default.values);
591
592            executor.close().await.expect("close edge executor");
593            let _ = std::fs::remove_dir_all(data_dir);
594        });
595    }
596
597    #[test]
598    fn wal_segment_capacity_bytes_scales_mib() {
599        assert_eq!(wal_segment_capacity_bytes(None).unwrap(), None);
600        assert_eq!(wal_segment_capacity_bytes(Some(4)).unwrap(), Some(4 << 20));
601    }
602
603    #[test]
604    fn wal_segment_capacity_bytes_rejects_zero_and_overflow() {
605        let zero = wal_segment_capacity_bytes(Some(0)).expect_err("zero must fail closed");
606        assert_eq!(zero.code, "QQL-VALIDATION-CONFIG");
607        let overflow =
608            wal_segment_capacity_bytes(Some(u64::MAX)).expect_err("overflow must fail closed");
609        assert_eq!(overflow.code, "QQL-VALIDATION-CONFIG");
610    }
611
612    #[test]
613    fn custom_edge_executor_is_schema_aware_and_rejects_shards() {
614        let runtime = tokio::runtime::Builder::new_current_thread()
615            .enable_all()
616            .build()
617            .expect("test runtime");
618        runtime.block_on(async {
619            let data_dir =
620                std::env::temp_dir().join(format!("qql-edge-test-{}", std::process::id()));
621            let _ = std::fs::remove_dir_all(&data_dir);
622            let executor =
623                custom_executor(&data_dir, false, Arc::new(TestEmbedder)).expect("custom executor");
624
625            let report = executor
626                .execute("CREATE COLLECTION docs HYBRID", OnError::Stop)
627                .await
628                .expect("create collection");
629            assert!(report.ok);
630            let report = executor
631                .execute(
632                    "UPSERT INTO docs VALUES {id: 1, text: 'hello'}",
633                    OnError::Stop,
634                )
635                .await
636                .expect("schema-aware upsert");
637            assert!(report.ok);
638            let report = executor
639                .execute(
640                    "QUERY 'hello' FROM docs USING dense AS DENSE LIMIT 1",
641                    OnError::Stop,
642                )
643                .await
644                .expect("schema-aware query");
645            assert!(report.ok);
646            let report = executor
647                .execute(
648                    "CREATE COLLECTION dense_only (dense VECTOR(3, COSINE))",
649                    OnError::Stop,
650                )
651                .await
652                .expect("create dense-only collection");
653            assert!(report.ok);
654            let report = executor
655                .execute(
656                    "UPSERT INTO dense_only VALUES {id: 2, body: 'dense only'}",
657                    OnError::Stop,
658                )
659                .await
660                .expect("dense-only auto-embed");
661            assert!(report.ok);
662            let report = executor
663                .execute(
664                    "CREATE COLLECTION sparse_only (sparse SPARSE)",
665                    OnError::Stop,
666                )
667                .await
668                .expect("create sparse-only collection");
669            assert!(report.ok);
670            let report = executor
671                .execute(
672                    "UPSERT INTO sparse_only VALUES {id: 3, content: 'sparse only'}",
673                    OnError::Stop,
674                )
675                .await
676                .expect("sparse-only auto-embed");
677            assert!(report.ok);
678            let report = executor
679                .execute(
680                    "QUERY 'hello' FROM docs USING dense AS DENSE SHARD 'tenant-a' LIMIT 1",
681                    OnError::Continue,
682                )
683                .await
684                .expect("edge should report unsupported shard");
685            assert!(!report.ok);
686            assert!(report.results[0].message.contains("UNSUPPORTED-SHARD"));
687            executor.close().await.expect("close edge executor");
688            let _ = std::fs::remove_dir_all(data_dir);
689        });
690    }
691
692    #[test]
693    fn edge_reads_do_not_create_missing_collections() {
694        // Regression (B-2): read operations against a missing collection must
695        // error with QQL-EDGE-COLLECTION-NOT-FOUND and must not materialise a
696        // ghost collection on disk — matching remote 404 semantics.
697        let runtime = tokio::runtime::Builder::new_current_thread()
698            .enable_all()
699            .build()
700            .expect("test runtime");
701        runtime.block_on(async {
702            let data_dir =
703                std::env::temp_dir().join(format!("qql-edge-readtest-{}", std::process::id()));
704            let _ = std::fs::remove_dir_all(&data_dir);
705            let executor =
706                custom_executor(&data_dir, false, Arc::new(TestEmbedder)).expect("custom executor");
707
708            let cases = [
709                "QUERY 'hello' FROM never_queried USING dense AS DENSE LIMIT 5",
710                "SCROLL FROM never_scrolled LIMIT 5",
711                "QUERY POINTS (1, 2) FROM never_points WITH PAYLOAD true",
712                "COUNT FROM never_counted",
713                "DELETE FROM never_deleted WHERE id = 1",
714            ];
715            for query in cases {
716                let report = executor
717                    .execute(query, OnError::Continue)
718                    .await
719                    .expect("read against missing collection must return a report");
720                assert!(
721                    !report.ok,
722                    "expected failure for '{query}': {:?}",
723                    report.results
724                );
725                assert!(
726                    report.results[0]
727                        .message
728                        .contains("QQL-EDGE-COLLECTION-NOT-FOUND"),
729                    "expected QQL-EDGE-COLLECTION-NOT-FOUND for '{query}', got {:?}",
730                    report.results[0].message
731                );
732            }
733
734            // No ghost collections were created on disk.
735            let report = executor
736                .execute("SHOW COLLECTIONS", OnError::Stop)
737                .await
738                .expect("list collections");
739            assert!(report.ok);
740            let cols = report.results[0]
741                .data
742                .as_ref()
743                .and_then(|d| d.collections())
744                .expect("collections list");
745            assert!(
746                cols.is_empty(),
747                "read operations must not create collections: {cols:?}"
748            );
749
750            executor.close().await.expect("close edge executor");
751            let _ = std::fs::remove_dir_all(data_dir);
752        });
753    }
754
755    #[test]
756    fn edge_upsert_autocreate_and_get_points_shape() {
757        // Regression (B-2 preserve + B-3): an embedding UPSERT to a missing
758        // collection still auto-creates it with the inferred schema, and
759        // QUERY POINTS on a populated collection returns non-empty hits.
760        let runtime = tokio::runtime::Builder::new_current_thread()
761            .enable_all()
762            .build()
763            .expect("test runtime");
764        runtime.block_on(async {
765            let data_dir =
766                std::env::temp_dir().join(format!("qql-edge-pointstest-{}", std::process::id()));
767            let _ = std::fs::remove_dir_all(&data_dir);
768            let executor =
769                custom_executor(&data_dir, false, Arc::new(TestEmbedder)).expect("custom executor");
770
771            // Auto-create via embedding upsert (collection does not exist yet).
772            let report = executor
773                .execute(
774                    "UPSERT INTO auto_docs VALUES {id: 1, text: 'hello'}, {id: 2, text: 'world'}",
775                    OnError::Stop,
776                )
777                .await
778                .expect("auto-creating upsert");
779            assert!(report.ok, "auto-creating UPSERT failed: {:?}", report);
780
781            let report = executor
782                .execute("SHOW COLLECTIONS", OnError::Stop)
783                .await
784                .expect("list collections");
785            let cols = report.results[0]
786                .data
787                .as_ref()
788                .and_then(|d| d.collections())
789                .map(<[String]>::len)
790                .unwrap_or(0);
791            assert_eq!(
792                cols, 1,
793                "UPSERT auto-create should materialise one collection"
794            );
795
796            // QUERY POINTS must surface the retrieved points (response shape
797            // regression) and report the hit count.
798            let report = executor
799                .execute(
800                    "QUERY POINTS (1, 2) FROM auto_docs WITH PAYLOAD true",
801                    OnError::Stop,
802                )
803                .await
804                .expect("points lookup");
805            assert!(report.ok, "POINTS lookup failed: {:?}", report);
806            assert_eq!(
807                report.results[0].message, "Found 2 hits",
808                "POINTS lookup should report 2 hits, got {:?}",
809                report.results[0].message
810            );
811            let hits = report.results[0].hits_ref().expect("data");
812            assert_eq!(hits.len(), 2, "POINTS lookup should return 2 hits");
813
814            executor.close().await.expect("close edge executor");
815            let _ = std::fs::remove_dir_all(data_dir);
816        });
817    }
818}