Skip to main content

graphrt_rag/
kb.rs

1//! [`KnowledgeBase`]: the service the `knowledge-retrieval` node uses.
2//!
3//! A knowledge base maps dataset ids to retrievers. A dataset is either backed
4//! by the built-in pipeline (a [`VectorStore`] + [`Embedder`], optionally a
5//! [`Reranker`]) or by any external [`Retriever`]. Register it in
6//! `graphrt_core::Services` as a plain `KnowledgeBase` value:
7//!
8//! ```
9//! # use std::sync::Arc;
10//! # use graphrt_core::Services;
11//! # use graphrt_rag::{KnowledgeBase, MemoryStore, HashEmbedder};
12//! let kb = KnowledgeBase::new()
13//!     .with_store_dataset("docs", Arc::new(MemoryStore::new()), Arc::new(HashEmbedder::new(64)), None);
14//! let services = Services::new().with(kb);
15//! ```
16
17use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
18
19use indexmap::IndexMap;
20use serde::{Deserialize, Serialize};
21
22use crate::chunker::ChunkerConfig;
23use crate::error::RagError;
24use crate::ingest::Ingestor;
25use crate::pipeline::{StoreRetriever, finalize, rerank};
26use crate::types::{
27    DocumentSummary, Embedder, Hit, MAX_DOCUMENT_PAGE, Reranker, RetrievalRequest, Retriever, StoredChunk, VectorStore,
28};
29
30/// Public description of a dataset.
31#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
32pub struct DatasetInfo {
33    pub id: String,
34    #[serde(default)]
35    pub name: String,
36    #[serde(default)]
37    pub description: String,
38}
39
40#[derive(Clone, Debug)]
41struct DatasetEntry {
42    info: DatasetInfo,
43    retriever: Arc<dyn Retriever>,
44    store: Option<(Arc<dyn VectorStore>, Arc<dyn Embedder>)>,
45    /// How documents ingested into this dataset are chunked.
46    chunking: ChunkerConfig,
47    /// Names of the config's store and embedder backing this dataset, when known.
48    names: Option<(String, String)>,
49    /// Set for datasets created at runtime with [`KnowledgeBase::create_dataset`].
50    runtime: Option<DatasetSpec>,
51}
52
53/// A dataset created at runtime from the stores, embedders and rerankers a
54/// knowledge config declares (see [`KnowledgeBase::create_dataset`]).
55#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
56#[serde(deny_unknown_fields)]
57pub struct DatasetSpec {
58    pub id: String,
59    #[serde(default)]
60    pub name: String,
61    #[serde(default)]
62    pub description: String,
63    /// Name of a store in the knowledge config.
64    pub store: String,
65    /// Name of an embedder in the knowledge config.
66    pub embedder: String,
67    /// Name of a reranker in the knowledge config.
68    #[serde(default)]
69    pub reranker: Option<String>,
70    #[serde(default)]
71    pub chunking: Option<ChunkerConfig>,
72}
73
74/// A dataset as listed for management UIs.
75#[derive(Clone, Debug, PartialEq, Serialize)]
76pub struct DatasetDetails {
77    #[serde(flatten)]
78    pub info: DatasetInfo,
79    /// Documents can be added, listed and removed (store-backed datasets).
80    pub managed: bool,
81    /// Created at runtime (and removable); `false` for datasets from the config.
82    pub runtime: bool,
83    pub store: Option<String>,
84    pub embedder: Option<String>,
85    pub chunking: Option<ChunkerConfig>,
86}
87
88/// Names of the stores, embedders and rerankers runtime datasets can use.
89#[derive(Clone, Debug, Default, PartialEq, Serialize)]
90pub struct ResourceNames {
91    pub stores: Vec<String>,
92    pub embedders: Vec<String>,
93    pub rerankers: Vec<String>,
94}
95
96/// Named building blocks declared by a knowledge config.
97#[derive(Clone, Debug, Default)]
98struct Resources {
99    stores: IndexMap<String, Arc<dyn VectorStore>>,
100    embedders: IndexMap<String, Arc<dyn Embedder>>,
101    rerankers: IndexMap<String, Arc<dyn Reranker>>,
102}
103
104type Registry = IndexMap<String, DatasetEntry>;
105
106/// Dataset id -> retriever registry.
107///
108/// Clones share their datasets: a dataset added to or removed from one clone
109/// (for example by the management API) is seen by every other clone (for
110/// example the one the workflows' `knowledge-retrieval` nodes hold).
111#[derive(Clone, Debug, Default)]
112pub struct KnowledgeBase {
113    datasets: Arc<RwLock<Registry>>,
114    resources: Arc<Resources>,
115    /// Reranker applied across datasets in multi-dataset retrieval.
116    reranker: Option<Arc<dyn Reranker>>,
117}
118
119impl KnowledgeBase {
120    pub fn new() -> Self {
121        Self::default()
122    }
123
124    // Every mutation replaces whole entries, so data behind a poisoned lock is still consistent.
125    fn read(&self) -> RwLockReadGuard<'_, Registry> {
126        self.datasets.read().unwrap_or_else(PoisonError::into_inner)
127    }
128
129    fn write(&self) -> RwLockWriteGuard<'_, Registry> {
130        self.datasets.write().unwrap_or_else(PoisonError::into_inner)
131    }
132
133    /// Add a dataset backed by the built-in store pipeline.
134    pub fn with_store_dataset(
135        mut self,
136        id: impl Into<String>,
137        store: Arc<dyn VectorStore>,
138        embedder: Arc<dyn Embedder>,
139        reranker: Option<Arc<dyn Reranker>>,
140    ) -> Self {
141        let id = id.into();
142        self.add_store_dataset(DatasetInfo { id, ..DatasetInfo::default() }, store, embedder, reranker);
143        self
144    }
145
146    /// Add (or replace) a store-backed dataset with descriptive info.
147    pub fn add_store_dataset(
148        &mut self,
149        info: DatasetInfo,
150        store: Arc<dyn VectorStore>,
151        embedder: Arc<dyn Embedder>,
152        reranker: Option<Arc<dyn Reranker>>,
153    ) -> &mut Self {
154        self.insert_store_dataset(info, store, embedder, reranker, ChunkerConfig::default(), None, None);
155        self
156    }
157
158    #[allow(clippy::too_many_arguments)]
159    fn insert_store_dataset(
160        &self,
161        info: DatasetInfo,
162        store: Arc<dyn VectorStore>,
163        embedder: Arc<dyn Embedder>,
164        reranker: Option<Arc<dyn Reranker>>,
165        chunking: ChunkerConfig,
166        names: Option<(String, String)>,
167        runtime: Option<DatasetSpec>,
168    ) {
169        let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
170        let entry = DatasetEntry {
171            info,
172            retriever: Arc::new(retriever),
173            store: Some((store, embedder)),
174            chunking,
175            names,
176            runtime,
177        };
178        self.write().insert(entry.info.id.clone(), entry);
179    }
180
181    /// Add a dataset served by an external retriever.
182    pub fn with_retriever(mut self, id: impl Into<String>, retriever: Arc<dyn Retriever>) -> Self {
183        let id = id.into();
184        self.add_retriever(DatasetInfo { id, ..DatasetInfo::default() }, retriever);
185        self
186    }
187
188    /// Add (or replace) a dataset served by an external retriever.
189    pub fn add_retriever(&mut self, info: DatasetInfo, retriever: Arc<dyn Retriever>) -> &mut Self {
190        let entry = DatasetEntry {
191            info,
192            retriever,
193            store: None,
194            chunking: ChunkerConfig::default(),
195            names: None,
196            runtime: None,
197        };
198        self.write().insert(entry.info.id.clone(), entry);
199        self
200    }
201
202    /// Reranker applied to the merged results of multi-dataset retrieval.
203    pub fn with_reranker(mut self, reranker: Option<Arc<dyn Reranker>>) -> Self {
204        self.reranker = reranker;
205        self
206    }
207
208    /// Every dataset's id, name and description, in registration order.
209    pub fn datasets(&self) -> Vec<DatasetInfo> {
210        self.read().values().map(|d| d.info.clone()).collect()
211    }
212
213    /// Every dataset with what a management UI shows about it.
214    pub fn dataset_details(&self) -> Vec<DatasetDetails> {
215        self.read()
216            .values()
217            .map(|d| DatasetDetails {
218                info: d.info.clone(),
219                managed: d.store.is_some(),
220                runtime: d.runtime.is_some(),
221                store: d.names.as_ref().map(|(store, _)| store.clone()),
222                embedder: d.names.as_ref().map(|(_, embedder)| embedder.clone()),
223                chunking: d.store.as_ref().map(|_| d.chunking.clone()),
224            })
225            .collect()
226    }
227
228    /// The stores, embedders and rerankers [`KnowledgeBase::create_dataset`] can use.
229    pub fn resource_names(&self) -> ResourceNames {
230        ResourceNames {
231            stores: self.resources.stores.keys().cloned().collect(),
232            embedders: self.resources.embedders.keys().cloned().collect(),
233            rerankers: self.resources.rerankers.keys().cloned().collect(),
234        }
235    }
236
237    /// Create a store-backed dataset at runtime from the named resources of the
238    /// knowledge config. Every clone of this knowledge base sees it at once.
239    ///
240    /// @throws `RagError::Input` for an invalid or taken id, an unknown store,
241    /// embedder or reranker name, or invalid chunking.
242    pub fn create_dataset(&self, spec: DatasetSpec) -> Result<DatasetDetails, RagError> {
243        validate_dataset_id(&spec.id)?;
244        let lookup = |kind: &str, name: &str, found: bool| {
245            if found {
246                Ok(())
247            } else {
248                Err(RagError::Input(format!("unknown {kind} `{name}` in the knowledge config")))
249            }
250        };
251        let store = self.resources.stores.get(&spec.store).cloned();
252        lookup("store", &spec.store, store.is_some())?;
253        let embedder = self.resources.embedders.get(&spec.embedder).cloned();
254        lookup("embedder", &spec.embedder, embedder.is_some())?;
255        let reranker = match &spec.reranker {
256            Some(name) => {
257                let reranker = self.resources.rerankers.get(name).cloned();
258                lookup("reranker", name, reranker.is_some())?;
259                reranker
260            }
261            None => None,
262        };
263        let chunking = spec.chunking.clone().unwrap_or_default();
264        validate_chunking(&spec.id, &chunking).map_err(|e| RagError::Input(e.to_string()))?;
265        let (Some(store), Some(embedder)) = (store, embedder) else {
266            return Err(RagError::Input("store and embedder are required".into()));
267        };
268        let info = DatasetInfo { id: spec.id.clone(), name: spec.name.clone(), description: spec.description.clone() };
269        {
270            // Check and insert under one lock so two requests cannot create the same id.
271            let mut registry = self.write();
272            if registry.contains_key(&spec.id) {
273                return Err(RagError::Input(format!("dataset `{}` already exists", spec.id)));
274            }
275            let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
276            registry.insert(
277                spec.id.clone(),
278                DatasetEntry {
279                    info,
280                    retriever: Arc::new(retriever),
281                    store: Some((store, embedder)),
282                    chunking,
283                    names: Some((spec.store.clone(), spec.embedder.clone())),
284                    runtime: Some(spec.clone()),
285                },
286            );
287        }
288        self.dataset_details()
289            .into_iter()
290            .find(|d| d.info.id == spec.id)
291            .ok_or_else(|| RagError::Store(format!("dataset `{}` vanished after creation", spec.id)))
292    }
293
294    /// Delete a dataset created at runtime together with its documents.
295    /// Returns the number of chunks removed.
296    ///
297    /// @throws `RagError::NotFound` for an unknown dataset, `RagError::Unsupported`
298    /// for one declared in the knowledge config (edit the config instead).
299    pub async fn remove_dataset(&self, dataset_id: &str) -> Result<usize, RagError> {
300        let entry = self.entry(dataset_id)?;
301        if entry.runtime.is_none() {
302            return Err(RagError::Unsupported(format!(
303                "dataset `{dataset_id}` is declared in the knowledge config; remove it there"
304            )));
305        }
306        // Delete the data first: if that fails the dataset stays, instead of leaving orphaned chunks.
307        let removed = match &entry.store {
308            Some((store, _)) => store.delete_dataset(dataset_id).await?,
309            None => 0,
310        };
311        self.write().shift_remove(dataset_id);
312        Ok(removed)
313    }
314
315    /// Specs of the datasets created at runtime, for persisting them.
316    pub fn runtime_datasets(&self) -> Vec<DatasetSpec> {
317        self.read().values().filter_map(|d| d.runtime.clone()).collect()
318    }
319
320    pub fn contains(&self, dataset_id: &str) -> bool {
321        self.read().contains_key(dataset_id)
322    }
323
324    /// An ingestor for a store-backed dataset (`None` for external retrievers or unknown ids),
325    /// using the dataset's chunking settings.
326    pub fn ingestor(&self, dataset_id: &str) -> Option<Ingestor> {
327        let registry = self.read();
328        let entry = registry.get(dataset_id)?;
329        let (store, embedder) = entry.store.as_ref()?;
330        Some(Ingestor::new(store.clone(), embedder.clone()).chunker(entry.chunking.clone()))
331    }
332
333    /// Set how documents ingested into `dataset_id` are chunked (e.g. parent-child).
334    ///
335    /// @throws `RagError::NotFound` for an unknown dataset, `RagError::Config` when
336    /// `parent_max_chars` is not larger than `max_chars`.
337    pub fn set_chunking(&mut self, dataset_id: &str, chunking: ChunkerConfig) -> Result<&mut Self, RagError> {
338        validate_chunking(dataset_id, &chunking)?;
339        {
340            let mut registry = self.write();
341            let entry = registry
342                .get_mut(dataset_id)
343                .ok_or_else(|| RagError::NotFound(format!("unknown dataset `{dataset_id}`")))?;
344            entry.chunking = chunking;
345        }
346        Ok(self)
347    }
348
349    /// The chunking settings of a dataset.
350    pub fn chunking(&self, dataset_id: &str) -> Option<ChunkerConfig> {
351        self.read().get(dataset_id).map(|d| d.chunking.clone())
352    }
353
354    /// A copy of a dataset's entry (cheap: shared handles), so no lock is held across awaits.
355    fn entry(&self, id: &str) -> Result<DatasetEntry, RagError> {
356        self.read().get(id).cloned().ok_or_else(|| RagError::NotFound(format!("unknown dataset `{id}`")))
357    }
358
359    /// Whether documents can be added to, listed in and removed from this dataset
360    /// (store-backed datasets; `false` for external retrievers and unknown ids).
361    pub fn is_managed(&self, dataset_id: &str) -> bool {
362        self.read().get(dataset_id).is_some_and(|d| d.store.is_some())
363    }
364
365    /// The store of a store-backed dataset.
366    ///
367    /// @throws `RagError::NotFound` for an unknown dataset, `RagError::Unsupported`
368    /// for one served by an external retriever.
369    fn managed_store(&self, dataset_id: &str) -> Result<Arc<dyn VectorStore>, RagError> {
370        let entry = self.entry(dataset_id)?;
371        entry
372            .store
373            .map(|(store, _)| store)
374            .ok_or_else(|| RagError::Unsupported(format!("dataset `{dataset_id}` is served by an external retriever")))
375    }
376
377    /// Up to `limit` documents of a dataset (capped at [`MAX_DOCUMENT_PAGE`]),
378    /// ordered by id and starting after the `after` id.
379    pub async fn list_documents(
380        &self,
381        dataset_id: &str,
382        limit: usize,
383        after: Option<&str>,
384    ) -> Result<Vec<DocumentSummary>, RagError> {
385        let limit = limit.clamp(1, MAX_DOCUMENT_PAGE);
386        self.managed_store(dataset_id)?.list_documents(dataset_id, limit, after).await
387    }
388
389    /// Chunks of one document in chunk order (vectors included).
390    pub async fn document_chunks(&self, dataset_id: &str, document_id: &str) -> Result<Vec<StoredChunk>, RagError> {
391        let mut chunks = self.managed_store(dataset_id)?.document_chunks(dataset_id, document_id).await?;
392        chunks.sort_by_key(|c| c.metadata.get("chunk_index").and_then(serde_json::Value::as_u64).unwrap_or(u64::MAX));
393        Ok(chunks)
394    }
395
396    /// Enable or disable a document: a disabled document keeps its chunks (and
397    /// stays disabled across re-ingestion) but no search returns it. Returns the
398    /// number of chunks of the document.
399    ///
400    /// @throws `RagError::NotFound` when the document does not exist.
401    pub async fn set_document_enabled(
402        &self,
403        dataset_id: &str,
404        document_id: &str,
405        enabled: bool,
406    ) -> Result<usize, RagError> {
407        let store = self.managed_store(dataset_id)?;
408        let mut chunks = store.document_chunks(dataset_id, document_id).await?;
409        if chunks.is_empty() {
410            return Err(RagError::NotFound(format!("unknown document `{document_id}` in dataset `{dataset_id}`")));
411        }
412        let count = chunks.len();
413        chunks.retain(|c| crate::types::is_disabled(&c.metadata) == enabled);
414        for chunk in &mut chunks {
415            if enabled {
416                chunk.metadata.remove(crate::types::DISABLED_KEY);
417            } else {
418                chunk.metadata.insert(crate::types::DISABLED_KEY.into(), serde_json::Value::Bool(true));
419            }
420        }
421        if !chunks.is_empty() {
422            store.upsert(chunks).await?;
423        }
424        Ok(count)
425    }
426
427    /// Remove a document. Returns the number of chunks removed (0 when it did not exist).
428    pub async fn delete_document(&self, dataset_id: &str, document_id: &str) -> Result<usize, RagError> {
429        self.managed_store(dataset_id)?.delete_document(dataset_id, document_id).await
430    }
431
432    /// Query every dataset concurrently. Returns `(dataset id, hits)` in request order.
433    async fn query_each(
434        &self,
435        request: &RetrievalRequest,
436        rerank_each: bool,
437    ) -> Result<Vec<(String, Vec<Hit>)>, RagError> {
438        let mut jobs = Vec::with_capacity(request.datasets.len());
439        for id in &request.datasets {
440            let retriever = self.entry(id)?.retriever;
441            let sub = RetrievalRequest {
442                datasets: vec![id.clone()],
443                // Threshold and top-k are applied after merging.
444                score_threshold: None,
445                rerank: rerank_each,
446                ..request.clone()
447            };
448            jobs.push(async move { retriever.retrieve(&sub).await.map(|hits| (id.clone(), hits)) });
449        }
450        futures::future::try_join_all(jobs).await
451    }
452
453    /// Retrieve across all requested datasets and merge by score.
454    ///
455    /// When `request.rerank` is set and the knowledge base has a cross-dataset
456    /// reranker, the merged list is reranked as a whole (making scores from
457    /// different datasets comparable). Otherwise each dataset's own reranker, if
458    /// any, is applied before merging.
459    pub async fn retrieve(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
460        validate_request(request)?;
461        let global_rerank = request.rerank && self.reranker.is_some() && request.datasets.len() > 1;
462        let per_dataset = self.query_each(request, request.rerank && !global_rerank).await?;
463        let mut merged: Vec<Hit> = per_dataset.into_iter().flat_map(|(_, hits)| hits).collect();
464        if global_rerank && let Some(reranker) = &self.reranker {
465            merged = rerank(reranker.as_ref(), &request.query, merged).await?;
466        }
467        Ok(finalize(merged, request.score_threshold, request.top_k))
468    }
469
470    /// Retrieve from the single best-matching dataset: every dataset is queried,
471    /// and only the hits of the dataset whose top hit scores highest are kept.
472    pub async fn retrieve_single(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
473        validate_request(request)?;
474        let per_dataset = self.query_each(request, request.rerank).await?;
475        let best = per_dataset
476            .into_iter()
477            .filter(|(_, hits)| !hits.is_empty())
478            .max_by(|a, b| top_score(&a.1).total_cmp(&top_score(&b.1)));
479        Ok(best.map(|(_, hits)| finalize(hits, request.score_threshold, request.top_k)).unwrap_or_default())
480    }
481}
482
483/// Runtime dataset ids appear in URLs and file names: 1-64 ASCII letters,
484/// digits, `-`, `_` or `.`, not starting with `.`.
485fn validate_dataset_id(id: &str) -> Result<(), RagError> {
486    let ok = !id.is_empty()
487        && id.len() <= 64
488        && !id.starts_with('.')
489        && id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'));
490    if ok {
491        Ok(())
492    } else {
493        Err(RagError::Input(format!(
494            "invalid dataset id `{id}`: use 1-64 ASCII letters, digits, `-`, `_` or `.`, not starting with `.`"
495        )))
496    }
497}
498
499fn validate_chunking(dataset_id: &str, chunking: &ChunkerConfig) -> Result<(), RagError> {
500    if chunking.max_chars == 0 {
501        return Err(RagError::Config(format!("dataset `{dataset_id}`: `chunking.max_chars` must be positive")));
502    }
503    if chunking.parent_max_chars.is_some_and(|p| p <= chunking.max_chars) {
504        return Err(RagError::Config(format!(
505            "dataset `{dataset_id}`: `chunking.parent_max_chars` must be larger than `max_chars`"
506        )));
507    }
508    Ok(())
509}
510
511/// Reject requests that would otherwise silently match nothing.
512fn validate_request(request: &RetrievalRequest) -> Result<(), RagError> {
513    if request.datasets.is_empty() {
514        return Err(RagError::Input("no dataset ids given".into()));
515    }
516    if let Some(filter) = &request.filter {
517        filter.validate().map_err(|e| RagError::Input(e.to_string()))?;
518    }
519    Ok(())
520}
521
522fn top_score(hits: &[Hit]) -> f32 {
523    hits.iter().map(|h| h.score).fold(f32::MIN, f32::max)
524}
525
526// ---------------------------------------------------------------------------
527// Configuration
528// ---------------------------------------------------------------------------
529
530/// Declarative knowledge-base configuration (`knowledge.yaml` / `.json`).
531///
532/// ```yaml
533/// embedders:
534///   default: { type: openai, base_url: "http://localhost:11434/v1", model: nomic-embed-text }
535///   inproc:  { type: local, model: sentence-transformers/all-MiniLM-L6-v2 }   # feature `local-embed`
536///   cloud:   { type: openai, model: text-embedding-3-small, api_key_env: OPENAI_API_KEY }
537/// rerankers:
538///   cohere: { type: http, base_url: "https://api.cohere.com/v2", model: rerank-v3.5, api_key_env: COHERE_API_KEY }
539/// stores:
540///   local:  { type: sqlite, path: ./kb.db }
541///   json:   { type: memory, path: ./kb.json }
542///   vector: { type: qdrant, url: "http://localhost:6333", collection: kb, api_key_env: QDRANT_API_KEY }
543/// datasets:
544///   - { id: handbook, name: Handbook, store: local, embedder: default, reranker: cohere }
545///   - { id: crm, retriever: { type: http, url: "https://rag.internal/search", api_key_env: RAG_TOKEN } }
546/// reranker: cohere   # optional, used across datasets
547/// ```
548///
549/// Secrets are never written inline: every provider takes `api_key_env`, the
550/// name of an environment variable resolved when the configuration is built.
551#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
552#[serde(deny_unknown_fields)]
553pub struct KnowledgeConfig {
554    #[serde(default)]
555    pub embedders: IndexMap<String, serde_json::Value>,
556    #[serde(default)]
557    pub rerankers: IndexMap<String, serde_json::Value>,
558    #[serde(default)]
559    pub stores: IndexMap<String, serde_json::Value>,
560    #[serde(default)]
561    pub datasets: Vec<DatasetConfig>,
562    /// Cross-dataset reranker (a key of `rerankers`).
563    #[serde(default)]
564    pub reranker: Option<String>,
565}
566
567/// One dataset entry.
568#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
569#[serde(deny_unknown_fields)]
570pub struct DatasetConfig {
571    pub id: String,
572    #[serde(default)]
573    pub name: String,
574    #[serde(default)]
575    pub description: String,
576    /// Key of `stores` (store-backed datasets).
577    #[serde(default)]
578    pub store: Option<String>,
579    /// Key of `embedders` (store-backed datasets).
580    #[serde(default)]
581    pub embedder: Option<String>,
582    /// Key of `rerankers`.
583    #[serde(default)]
584    pub reranker: Option<String>,
585    /// External retriever definition (instead of store + embedder).
586    #[serde(default)]
587    pub retriever: Option<serde_json::Value>,
588    /// Chunking of ingested documents, e.g. `{ max_chars: 400, parent_max_chars: 1500 }`
589    /// for parent-child retrieval (store-backed datasets).
590    #[serde(default)]
591    pub chunking: Option<ChunkerConfig>,
592}
593
594/// Resolves environment variable names to values (`std::env::var` in production).
595pub type EnvLookup<'a> = &'a dyn Fn(&str) -> Option<String>;
596
597impl KnowledgeConfig {
598    pub fn from_json(text: &str) -> Result<Self, RagError> {
599        serde_json::from_str(text).map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
600    }
601
602    #[cfg(feature = "yaml")]
603    pub fn from_yaml(text: &str) -> Result<Self, RagError> {
604        let options = serde_saphyr::options! { strict_booleans: true };
605        serde_saphyr::from_str_with_options(text, options)
606            .map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
607    }
608
609    /// Build the knowledge base, resolving secrets with `std::env::var`.
610    pub fn build(&self) -> Result<KnowledgeBase, RagError> {
611        self.build_with_env(&|name| std::env::var(name).ok())
612    }
613
614    /// Build the knowledge base with a custom environment lookup.
615    pub fn build_with_env(&self, env: EnvLookup<'_>) -> Result<KnowledgeBase, RagError> {
616        let mut embedders: IndexMap<&str, Arc<dyn Embedder>> = IndexMap::new();
617        for (name, raw) in &self.embedders {
618            embedders.insert(name, build_embedder(name, raw, env)?);
619        }
620        let mut rerankers: IndexMap<&str, Arc<dyn Reranker>> = IndexMap::new();
621        for (name, raw) in &self.rerankers {
622            rerankers.insert(name, build_reranker(name, raw, env)?);
623        }
624        let mut stores: IndexMap<&str, Arc<dyn VectorStore>> = IndexMap::new();
625        for (name, raw) in &self.stores {
626            stores.insert(name, build_store(name, raw, env)?);
627        }
628
629        let lookup_reranker = |key: &Option<String>| -> Result<Option<Arc<dyn Reranker>>, RagError> {
630            match key {
631                None => Ok(None),
632                Some(k) => rerankers
633                    .get(k.as_str())
634                    .cloned()
635                    .map(Some)
636                    .ok_or_else(|| RagError::Config(format!("unknown reranker `{k}`"))),
637            }
638        };
639
640        let mut kb = KnowledgeBase::new().with_reranker(lookup_reranker(&self.reranker)?);
641        for ds in &self.datasets {
642            if ds.id.trim().is_empty() {
643                return Err(RagError::Config("dataset id must not be empty".into()));
644            }
645            if kb.contains(&ds.id) {
646                return Err(RagError::Config(format!("duplicate dataset `{}`", ds.id)));
647            }
648            let info = DatasetInfo { id: ds.id.clone(), name: ds.name.clone(), description: ds.description.clone() };
649            if let Some(raw) = &ds.retriever {
650                kb.add_retriever(info, build_retriever(&ds.id, raw, env)?);
651                continue;
652            }
653            let store_key = ds.store.as_deref().ok_or_else(|| {
654                RagError::Config(format!("dataset `{}` needs either `store` + `embedder` or `retriever`", ds.id))
655            })?;
656            let store = stores
657                .get(store_key)
658                .cloned()
659                .ok_or_else(|| RagError::Config(format!("unknown store `{store_key}`")))?;
660            let embedder_key = match ds.embedder.as_deref() {
661                Some(k) => k,
662                None if embedders.len() == 1 => embedders.keys().next().copied().unwrap_or_default(),
663                None => return Err(RagError::Config(format!("dataset `{}` needs `embedder`", ds.id))),
664            };
665            let embedder = embedders
666                .get(embedder_key)
667                .cloned()
668                .ok_or_else(|| RagError::Config(format!("unknown embedder `{embedder_key}`")))?;
669            let chunking = ds.chunking.clone().unwrap_or_default();
670            validate_chunking(&ds.id, &chunking)?;
671            let names = Some((store_key.to_owned(), embedder_key.to_owned()));
672            kb.insert_store_dataset(info, store, embedder, lookup_reranker(&ds.reranker)?, chunking, names, None);
673        }
674        kb.resources = Arc::new(Resources {
675            stores: stores.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
676            embedders: embedders.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
677            rerankers: rerankers.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
678        });
679        Ok(kb)
680    }
681}
682
683/// Keys accepted by every HTTP-backed provider entry.
684#[cfg_attr(not(feature = "http"), allow(dead_code))]
685const HTTP_KEYS: &[&str] = &["type", "api_key", "api_key_env", "api_key_header", "headers", "timeout_ms"];
686
687/// Reject keys a provider entry does not understand (typos would otherwise be
688/// ignored silently, e.g. `file:` instead of `path:` turning a persistent store
689/// into an in-memory one).
690fn check_keys(name: &str, raw: &serde_json::Value, allowed: &[&str], with_http: bool) -> Result<(), RagError> {
691    let obj = raw.as_object().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
692    for key in obj.keys() {
693        let known = allowed.contains(&key.as_str()) || (with_http && HTTP_KEYS.contains(&key.as_str()));
694        if !known && key != "type" {
695            return Err(RagError::Config(format!("`{name}`: unknown key `{key}`")));
696        }
697    }
698    Ok(())
699}
700
701fn provider_type(name: &str, raw: &serde_json::Value) -> Result<String, RagError> {
702    raw.get("type")
703        .and_then(|t| t.as_str())
704        .map(str::to_owned)
705        .ok_or_else(|| RagError::Config(format!("`{name}` needs a `type`")))
706}
707
708/// Strip `type` / `api_key_env`, reject inline secrets, and inject the resolved key.
709#[cfg_attr(not(feature = "http"), allow(dead_code))]
710fn with_secret(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<serde_json::Value, RagError> {
711    let mut obj = raw.as_object().cloned().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
712    obj.remove("type");
713    if obj.contains_key("api_key") {
714        return Err(RagError::Config(format!(
715            "`{name}`: inline `api_key` is not allowed; use `api_key_env` with an environment variable name"
716        )));
717    }
718    if let Some(var) = obj.remove("api_key_env") {
719        let var = var.as_str().ok_or_else(|| RagError::Config(format!("`{name}`: `api_key_env` must be a string")))?;
720        let value =
721            env(var).ok_or_else(|| RagError::Config(format!("`{name}`: environment variable `{var}` is not set")))?;
722        obj.insert("api_key".into(), serde_json::Value::String(value));
723    }
724    Ok(serde_json::Value::Object(obj))
725}
726
727#[cfg_attr(not(feature = "http"), allow(dead_code))]
728fn parse<T: serde::de::DeserializeOwned>(name: &str, value: serde_json::Value) -> Result<T, RagError> {
729    serde_json::from_value(value).map_err(|e| RagError::Config(format!("`{name}`: {e}")))
730}
731
732fn build_embedder(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Embedder>, RagError> {
733    let kind = provider_type(name, raw)?;
734    match kind.as_str() {
735        #[cfg(feature = "http")]
736        "openai" | "openai-compatible" | "ollama" => {
737            check_keys(
738                name,
739                raw,
740                &["base_url", "model", "dimensions", "batch_size", "query_prefix", "document_prefix"],
741                true,
742            )?;
743            let mut value = with_secret(name, raw, env)?;
744            if kind == "ollama"
745                && let Some(obj) = value.as_object_mut()
746            {
747                obj.entry("base_url").or_insert_with(|| "http://localhost:11434/v1".into());
748            }
749            let cfg: crate::http::OpenAiEmbedderConfig = parse(name, value)?;
750            Ok(Arc::new(crate::http::OpenAiEmbedder::new(cfg)?))
751        }
752        #[cfg(feature = "local-embed")]
753        "local" => {
754            let mut value = raw.as_object().cloned().unwrap_or_default();
755            value.remove("type");
756            let config: crate::local_embed::LocalEmbedderConfig = parse(name, serde_json::Value::Object(value))?;
757            Ok(Arc::new(crate::local_embed::LocalEmbedder::new(config)?))
758        }
759        "hash" => {
760            check_keys(name, raw, &["dimensions"], false)?;
761            let dims = match raw.get("dimensions") {
762                None => 256,
763                Some(v) => v
764                    .as_u64()
765                    .and_then(|d| usize::try_from(d).ok())
766                    .filter(|d| *d > 0)
767                    .ok_or_else(|| RagError::Config(format!("`{name}`: `dimensions` must be a positive integer")))?,
768            };
769            let _ = env;
770            Ok(Arc::new(crate::hash::HashEmbedder::new(dims)))
771        }
772        other => Err(RagError::Config(format!("embedder `{name}`: unsupported type `{other}` in this build"))),
773    }
774}
775
776fn build_reranker(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Reranker>, RagError> {
777    let kind = provider_type(name, raw)?;
778    match kind.as_str() {
779        #[cfg(feature = "http")]
780        "http" | "cohere" | "jina" | "voyage" => {
781            check_keys(name, raw, &["base_url", "model"], true)?;
782            let cfg: crate::http::HttpRerankerConfig = parse(name, with_secret(name, raw, env)?)?;
783            Ok(Arc::new(crate::http::HttpReranker::new(cfg)?))
784        }
785        other => {
786            let _ = env;
787            Err(RagError::Config(format!("reranker `{name}`: unsupported type `{other}` in this build")))
788        }
789    }
790}
791
792fn build_store(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn VectorStore>, RagError> {
793    let kind = provider_type(name, raw)?;
794    match kind.as_str() {
795        "memory" => {
796            check_keys(name, raw, &["path"], false)?;
797            let path = match raw.get("path") {
798                None => None,
799                Some(serde_json::Value::String(p)) => Some(p.as_str()),
800                Some(_) => return Err(RagError::Config(format!("store `{name}`: `path` must be a string"))),
801            };
802            match path {
803                #[cfg(feature = "fs")]
804                Some(path) => Ok(Arc::new(crate::memory::MemoryStore::open(path)?)),
805                #[cfg(not(feature = "fs"))]
806                Some(_) => Err(RagError::Config(format!("store `{name}`: `path` requires the `fs` feature"))),
807                None => Ok(Arc::new(crate::memory::MemoryStore::new())),
808            }
809        }
810        #[cfg(feature = "sqlite")]
811        "sqlite" => {
812            check_keys(name, raw, &["path"], false)?;
813            let path =
814                raw.get("path").and_then(serde_json::Value::as_str).filter(|p| !p.trim().is_empty()).ok_or_else(
815                    || RagError::Config(format!("store `{name}`: `path` (the database file) is required")),
816                )?;
817            Ok(Arc::new(crate::sqlite::SqliteStore::open(path)?))
818        }
819        #[cfg(feature = "qdrant")]
820        "qdrant" => {
821            check_keys(name, raw, &["url", "collection", "create_collection"], true)?;
822            let mut value = with_secret(name, raw, env)?;
823            if let Some(obj) = value.as_object_mut() {
824                obj.entry("api_key_header").or_insert_with(|| "api-key".into());
825            }
826            let cfg: crate::qdrant::QdrantConfig = parse(name, value)?;
827            Ok(Arc::new(crate::qdrant::QdrantStore::new(cfg)?))
828        }
829        other => {
830            let _ = env;
831            Err(RagError::Config(format!("store `{name}`: unsupported type `{other}` in this build")))
832        }
833    }
834}
835
836fn build_retriever(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Retriever>, RagError> {
837    let kind = provider_type(name, raw)?;
838    match kind.as_str() {
839        #[cfg(feature = "http")]
840        "http" => {
841            check_keys(name, raw, &["url"], true)?;
842            let cfg: crate::http::HttpRetrieverConfig = parse(name, with_secret(name, raw, env)?)?;
843            Ok(Arc::new(crate::http::HttpRetriever::new(cfg)?))
844        }
845        other => {
846            let _ = env;
847            Err(RagError::Config(format!("retriever for `{name}`: unsupported type `{other}` in this build")))
848        }
849    }
850}
851
852#[cfg(test)]
853mod tests {
854    use super::*;
855    use crate::hash::HashEmbedder;
856    use crate::ingest::Document;
857    use crate::memory::MemoryStore;
858
859    fn kb_with_two_datasets() -> KnowledgeBase {
860        let embedder: Arc<dyn Embedder> = Arc::new(HashEmbedder::new(128));
861        KnowledgeBase::new()
862            .with_store_dataset("pets", Arc::new(MemoryStore::new()), embedder.clone(), None)
863            .with_store_dataset("space", Arc::new(MemoryStore::new()), embedder, None)
864    }
865
866    async fn ingest(kb: &KnowledgeBase, ds: &str, id: &str, text: &str) {
867        kb.ingestor(ds).unwrap().ingest(&Document::new(ds, id, text)).await.unwrap();
868    }
869
870    #[tokio::test]
871    async fn documents_can_be_listed_paged_inspected_and_deleted() {
872        let kb = kb_with_two_datasets();
873        let ingestor = kb.ingestor("pets").unwrap();
874        let mut meta = crate::types::Metadata::new();
875        meta.insert("lang".into(), serde_json::json!("en"));
876        let long = "Cats purr. ".repeat(200);
877        ingestor.ingest(&Document::new("pets", "cats", long).title("Cats").metadata(meta)).await.unwrap();
878        ingest(&kb, "pets", "dogs", "Dogs bark.").await;
879        ingest(&kb, "pets", "birds", "Birds sing.").await;
880
881        let first = kb.list_documents("pets", 2, None).await.unwrap();
882        let ids: Vec<&str> = first.iter().map(|d| d.document_id.as_str()).collect();
883        assert_eq!(ids, ["birds", "cats"]);
884        let cats = &first[1];
885        assert_eq!(cats.title, "Cats");
886        assert!(cats.chunks > 1, "long document is split");
887        assert_eq!(cats.metadata.get("lang"), Some(&serde_json::json!("en")));
888        assert!(!cats.metadata.contains_key("chunk_index"), "per-chunk keys are not document metadata");
889        let next = kb.list_documents("pets", 2, Some("cats")).await.unwrap();
890        assert_eq!(next.iter().map(|d| d.document_id.as_str()).collect::<Vec<_>>(), ["dogs"]);
891
892        let chunks = kb.document_chunks("pets", "cats").await.unwrap();
893        assert_eq!(chunks.len(), cats.chunks);
894        let order: Vec<u64> = chunks.iter().map(|c| c.metadata["chunk_index"].as_u64().unwrap()).collect();
895        assert!(order.windows(2).all(|w| w[0] < w[1]), "chunks come in document order: {order:?}");
896
897        assert_eq!(kb.delete_document("pets", "cats").await.unwrap(), cats.chunks);
898        assert_eq!(kb.delete_document("pets", "cats").await.unwrap(), 0);
899        assert_eq!(kb.list_documents("pets", 50, None).await.unwrap().len(), 2);
900    }
901
902    #[tokio::test]
903    async fn disabled_documents_are_kept_but_never_retrieved() {
904        let kb = kb_with_two_datasets();
905        ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
906        ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
907        let query = RetrievalRequest::new("pets that purr", vec!["pets".into()]).top_k(5);
908        let found = |hits: &[Hit]| hits.iter().any(|h| h.document_id == "cats");
909        assert!(found(&kb.retrieve(&query).await.unwrap()));
910
911        assert!(kb.set_document_enabled("pets", "cats", false).await.unwrap() >= 1);
912        assert!(!found(&kb.retrieve(&query).await.unwrap()), "disabled document retrieved");
913        let docs = kb.list_documents("pets", 10, None).await.unwrap();
914        let cats = docs.iter().find(|d| d.document_id == "cats").unwrap();
915        assert!(!cats.enabled);
916        assert!(!cats.metadata.contains_key(crate::types::DISABLED_KEY), "flag is not document metadata");
917
918        // A new version of a disabled document stays disabled.
919        ingest(&kb, "pets", "cats", "Cats are small furry pets that purr and nap.").await;
920        assert!(!found(&kb.retrieve(&query).await.unwrap()), "re-ingestion re-enabled the document");
921
922        kb.set_document_enabled("pets", "cats", true).await.unwrap();
923        assert!(found(&kb.retrieve(&query).await.unwrap()));
924        assert_eq!(kb.set_document_enabled("pets", "nope", false).await.unwrap_err().kind(), "not_found");
925    }
926
927    #[tokio::test]
928    async fn parent_child_datasets_return_whole_parents_once() {
929        let mut kb = KnowledgeBase::new().with_store_dataset(
930            "docs",
931            Arc::new(MemoryStore::new()),
932            Arc::new(HashEmbedder::new(256)),
933            None,
934        );
935        let chunking = ChunkerConfig { max_chars: 60, overlap: 0, markdown: false, parent_max_chars: Some(220) };
936        kb.set_chunking("docs", chunking).unwrap();
937        let text = "Leave policy. Every employee gets 24 days of paid annual leave. Unused leave carries over for six months. Sick leave is separate.\n\n\
938                    Expense policy. Expenses below 500 USD are approved by your manager. Larger ones need finance approval.";
939        ingest(&kb, "docs", "handbook", text).await;
940        let children = kb.document_chunks("docs", "handbook").await.unwrap();
941        assert!(children.len() > 2, "the document is split into several children");
942
943        let request = RetrievalRequest::new("annual leave days carries over", vec!["docs".into()]).top_k(5);
944        let hits = kb.retrieve(&request).await.unwrap();
945        let leave: Vec<&Hit> = hits.iter().filter(|h| h.content.contains("24 days")).collect();
946        assert_eq!(leave.len(), 1, "children of one parent collapse into one hit: {hits:?}");
947        let parent = &leave[0].content;
948        assert!(parent.contains("Leave policy") && parent.contains("Sick leave"), "whole parent returned: {parent}");
949        assert!(!parent.contains("Expense"), "parents stay separate: {parent}");
950    }
951
952    #[test]
953    fn chunking_config_is_validated() {
954        let build = |chunking: serde_json::Value| {
955            KnowledgeConfig::from_json(
956                &serde_json::json!({
957                    "embedders": { "e": { "type": "hash" } },
958                    "stores": { "m": { "type": "memory" } },
959                    "datasets": [{ "id": "d", "store": "m", "embedder": "e", "chunking": chunking }]
960                })
961                .to_string(),
962            )
963            .unwrap()
964            .build_with_env(&|_| None)
965        };
966        let kb = build(serde_json::json!({ "max_chars": 300, "parent_max_chars": 1200 })).unwrap();
967        assert_eq!(kb.chunking("d").unwrap().parent_max_chars, Some(1200));
968        let error = build(serde_json::json!({ "max_chars": 300, "parent_max_chars": 200 })).unwrap_err();
969        assert!(error.to_string().contains("parent_max_chars"), "{error}");
970    }
971
972    fn config_kb() -> KnowledgeBase {
973        KnowledgeConfig::from_json(
974            &serde_json::json!({
975                "embedders": { "e": { "type": "hash", "dimensions": 64 } },
976                "stores": { "shared": { "type": "memory" } },
977                "datasets": [{ "id": "static", "store": "shared", "embedder": "e" }]
978            })
979            .to_string(),
980        )
981        .unwrap()
982        .build_with_env(&|_| None)
983        .unwrap()
984    }
985
986    fn spec(id: &str) -> DatasetSpec {
987        DatasetSpec {
988            id: id.into(),
989            name: "Runtime".into(),
990            description: String::new(),
991            store: "shared".into(),
992            embedder: "e".into(),
993            reranker: None,
994            chunking: None,
995        }
996    }
997
998    #[tokio::test]
999    async fn runtime_datasets_are_shared_by_clones_and_removable() {
1000        let api = config_kb();
1001        let workflows = api.clone();
1002        assert_eq!(api.resource_names().stores, ["shared"]);
1003
1004        let created = api.create_dataset(spec("notes")).unwrap();
1005        assert!(created.runtime && created.managed);
1006        assert_eq!(created.store.as_deref(), Some("shared"));
1007        assert!(workflows.contains("notes"), "clones see runtime datasets");
1008
1009        ingest(&api, "notes", "n1", "Runtime datasets work like any other dataset.").await;
1010        ingest(&api, "static", "s1", "The static dataset shares the store.").await;
1011        let hits = workflows.retrieve(&RetrievalRequest::new("runtime datasets", vec!["notes".into()])).await.unwrap();
1012        assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("n1"));
1013
1014        assert_eq!(api.runtime_datasets(), vec![spec("notes")]);
1015        assert_eq!(api.remove_dataset("notes").await.unwrap(), 1);
1016        assert!(!workflows.contains("notes"));
1017        assert_eq!(api.list_documents("static", 10, None).await.unwrap().len(), 1, "other datasets keep their data");
1018        assert_eq!(api.remove_dataset("static").await.unwrap_err().kind(), "unsupported");
1019        assert_eq!(api.remove_dataset("missing").await.unwrap_err().kind(), "not_found");
1020    }
1021
1022    #[test]
1023    fn runtime_dataset_specs_are_validated() {
1024        let kb = config_kb();
1025        let error = |spec: DatasetSpec| kb.create_dataset(spec).unwrap_err().to_string();
1026        assert!(error(spec("static")).contains("already exists"));
1027        assert!(error(spec("bad/id")).contains("invalid dataset id"));
1028        assert!(error(spec("")).contains("invalid dataset id"));
1029        assert!(error(DatasetSpec { store: "nope".into(), ..spec("a") }).contains("unknown store `nope`"));
1030        assert!(error(DatasetSpec { embedder: "nope".into(), ..spec("b") }).contains("unknown embedder `nope`"));
1031        assert!(error(DatasetSpec { reranker: Some("nope".into()), ..spec("c") }).contains("unknown reranker"));
1032        let bad_chunking = ChunkerConfig { max_chars: 100, overlap: 0, markdown: true, parent_max_chars: Some(50) };
1033        assert!(error(DatasetSpec { chunking: Some(bad_chunking), ..spec("d") }).contains("parent_max_chars"));
1034        assert_eq!(kb.datasets().len(), 1, "failed creations leave nothing behind");
1035    }
1036
1037    #[tokio::test]
1038    async fn external_and_unknown_datasets_are_not_managed() {
1039        let store_kb = kb_with_two_datasets();
1040        let retriever = store_kb.entry("pets").unwrap().retriever;
1041        let kb = KnowledgeBase::new().with_retriever("remote", retriever);
1042        assert!(!kb.is_managed("remote"));
1043        assert_eq!(kb.list_documents("remote", 10, None).await.unwrap_err().kind(), "unsupported");
1044        assert_eq!(kb.delete_document("remote", "x").await.unwrap_err().kind(), "unsupported");
1045        assert_eq!(kb.list_documents("missing", 10, None).await.unwrap_err().kind(), "not_found");
1046    }
1047
1048    #[tokio::test]
1049    async fn multi_dataset_merge_and_single_mode() {
1050        let kb = kb_with_two_datasets();
1051        ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
1052        ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
1053        ingest(&kb, "space", "mars", "Mars is the red planet in our solar system.").await;
1054
1055        let req = RetrievalRequest::new("which pets purr", vec!["pets".into(), "space".into()]).top_k(2);
1056        let hits = kb.retrieve(&req).await.unwrap();
1057        assert_eq!(hits.len(), 2);
1058        assert_eq!(hits[0].document_id, "cats");
1059
1060        let single = kb.retrieve_single(&req).await.unwrap();
1061        assert!(single.iter().all(|h| h.dataset_id == "pets"));
1062
1063        let unknown = RetrievalRequest::new("x", vec!["nope".into()]);
1064        assert!(matches!(kb.retrieve(&unknown).await, Err(RagError::NotFound(_))));
1065    }
1066
1067    #[test]
1068    fn config_rejects_inline_secrets_and_resolves_env() {
1069        let cfg = KnowledgeConfig::from_json(
1070            r#"{"embedders":{"e":{"type":"hash","dimensions":16}},
1071                "stores":{"m":{"type":"memory"}},
1072                "datasets":[{"id":"a","store":"m"}]}"#,
1073        )
1074        .unwrap();
1075        let kb = cfg.build_with_env(&|_| None).unwrap();
1076        assert!(kb.contains("a"));
1077
1078        let bad = KnowledgeConfig::from_json(
1079            r#"{"rerankers":{"r":{"type":"cohere","base_url":"http://x","model":"m","api_key":"inline"}}}"#,
1080        )
1081        .unwrap();
1082        let err = bad.build_with_env(&|_| None).unwrap_err();
1083        assert!(err.to_string().contains("inline") || err.to_string().contains("unsupported"), "{err}");
1084
1085        let missing = KnowledgeConfig::from_json(r#"{"datasets":[{"id":"a","store":"nope"}]}"#).unwrap();
1086        assert!(missing.build_with_env(&|_| None).is_err());
1087    }
1088
1089    #[cfg(feature = "http")]
1090    #[test]
1091    fn config_env_secret_lookup() {
1092        let cfg = KnowledgeConfig::from_json(
1093            r#"{"embedders":{"e":{"type":"openai","base_url":"http://127.0.0.1:9","model":"m","api_key_env":"MY_KEY"}}}"#,
1094        )
1095        .unwrap();
1096        assert!(cfg.build_with_env(&|_| None).unwrap_err().to_string().contains("MY_KEY"));
1097        assert!(cfg.build_with_env(&|n| (n == "MY_KEY").then(|| "secret".to_owned())).is_ok());
1098    }
1099
1100    #[cfg(feature = "yaml")]
1101    #[test]
1102    fn yaml_config() {
1103        let cfg = KnowledgeConfig::from_yaml(
1104            "embedders:\n  e: { type: hash }\nstores:\n  m: { type: memory }\ndatasets:\n  - { id: kb, store: m, embedder: e }\n",
1105        )
1106        .unwrap();
1107        assert_eq!(cfg.datasets.len(), 1);
1108        assert!(cfg.build_with_env(&|_| None).is_ok());
1109    }
1110
1111    #[cfg(feature = "sqlite")]
1112    #[tokio::test]
1113    async fn sqlite_store_from_config_persists_between_builds() {
1114        let dir = std::env::temp_dir().join(format!("graphrt-rag-kb-sqlite-{}", std::process::id()));
1115        let _ = std::fs::remove_dir_all(&dir);
1116        std::fs::create_dir_all(&dir).unwrap();
1117        let path = dir.join("kb.db");
1118        let config = KnowledgeConfig::from_json(
1119            &serde_json::json!({
1120                "embedders": { "e": { "type": "hash", "dimensions": 64 } },
1121                "stores": { "local": { "type": "sqlite", "path": path } },
1122                "datasets": [{ "id": "faq", "store": "local", "embedder": "e" }]
1123            })
1124            .to_string(),
1125        )
1126        .unwrap();
1127
1128        let first = config.build_with_env(&|_| None).unwrap();
1129        ingest(&first, "faq", "returns", "Items can be returned within 30 days.").await;
1130        drop(first);
1131
1132        let second = config.build_with_env(&|_| None).unwrap();
1133        let hits = second.retrieve(&RetrievalRequest::new("returned within days", vec!["faq".into()])).await.unwrap();
1134        assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("returns"));
1135        let _ = std::fs::remove_dir_all(&dir);
1136    }
1137
1138    #[cfg(feature = "local-embed")]
1139    #[test]
1140    fn local_embedder_from_config_is_lazy_and_strict() {
1141        let build = |embedder: serde_json::Value| {
1142            KnowledgeConfig::from_json(&serde_json::json!({ "embedders": { "e": embedder } }).to_string())
1143                .unwrap()
1144                .build_with_env(&|_| None)
1145        };
1146        // Nothing is downloaded or loaded while building the knowledge base.
1147        assert!(build(serde_json::json!({ "type": "local" })).is_ok());
1148        assert!(build(serde_json::json!({ "type": "local", "path": "/not/loaded/yet" })).is_ok());
1149        let typo = build(serde_json::json!({ "type": "local", "modle": "x/y" })).unwrap_err();
1150        assert!(typo.to_string().contains("modle"), "{typo}");
1151        let traversal = build(serde_json::json!({ "type": "local", "model": "../../etc" })).unwrap_err();
1152        assert!(traversal.to_string().contains("`model`"), "{traversal}");
1153    }
1154
1155    #[cfg(feature = "sqlite")]
1156    #[test]
1157    fn sqlite_store_requires_a_path() {
1158        let config = KnowledgeConfig::from_json(r#"{"stores":{"local":{"type":"sqlite"}}}"#).unwrap();
1159        let error = config.build_with_env(&|_| None).unwrap_err();
1160        assert!(error.to_string().contains("`path` (the database file) is required"), "{error}");
1161    }
1162}