Skip to main content

graphrt_rag/
kb.rs

1//! [`KnowledgeBase`]: the service the `knowledge-retrieval` node uses.
2//!
3//! A knowledge base maps dataset ids to retrievers. A dataset is either backed
4//! by the built-in pipeline (a [`VectorStore`] + [`Embedder`], optionally a
5//! [`Reranker`]) or by any external [`Retriever`]. Register it in
6//! `graphrt_core::Services` as a plain `KnowledgeBase` value:
7//!
8//! ```
9//! # use std::sync::Arc;
10//! # use graphrt_core::Services;
11//! # use graphrt_rag::{KnowledgeBase, MemoryStore, HashEmbedder};
12//! let kb = KnowledgeBase::new()
13//!     .with_store_dataset("docs", Arc::new(MemoryStore::new()), Arc::new(HashEmbedder::new(64)), None);
14//! let services = Services::new().with(kb);
15//! ```
16
17use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
18
19use indexmap::IndexMap;
20use serde::{Deserialize, Serialize};
21
22use crate::chunker::ChunkerConfig;
23use crate::error::RagError;
24use crate::ingest::Ingestor;
25use crate::pipeline::{StoreRetriever, finalize, rerank};
26use crate::types::{
27    DocumentSummary, Embedder, Hit, MAX_DOCUMENT_PAGE, Reranker, RetrievalRequest, Retriever, StoredChunk, VectorStore,
28};
29
30/// Public description of a dataset.
31#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
32pub struct DatasetInfo {
33    pub id: String,
34    #[serde(default)]
35    pub name: String,
36    #[serde(default)]
37    pub description: String,
38}
39
40#[derive(Clone, Debug)]
41struct DatasetEntry {
42    info: DatasetInfo,
43    retriever: Arc<dyn Retriever>,
44    store: Option<(Arc<dyn VectorStore>, Arc<dyn Embedder>)>,
45    /// How documents ingested into this dataset are chunked.
46    chunking: ChunkerConfig,
47    /// Names of the config's store and embedder backing this dataset, when known.
48    names: Option<(String, String)>,
49    /// Set for datasets created at runtime with [`KnowledgeBase::create_dataset`].
50    runtime: Option<DatasetSpec>,
51}
52
53/// A dataset created at runtime from the stores, embedders and rerankers a
54/// knowledge config declares (see [`KnowledgeBase::create_dataset`]).
55#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
56#[serde(deny_unknown_fields)]
57pub struct DatasetSpec {
58    pub id: String,
59    #[serde(default)]
60    pub name: String,
61    #[serde(default)]
62    pub description: String,
63    /// Name of a store in the knowledge config.
64    pub store: String,
65    /// Name of an embedder in the knowledge config.
66    pub embedder: String,
67    /// Name of a reranker in the knowledge config.
68    #[serde(default)]
69    pub reranker: Option<String>,
70    #[serde(default)]
71    pub chunking: Option<ChunkerConfig>,
72}
73
74/// A dataset as listed for management UIs.
75#[derive(Clone, Debug, PartialEq, Serialize)]
76pub struct DatasetDetails {
77    #[serde(flatten)]
78    pub info: DatasetInfo,
79    /// Documents can be added, listed and removed (store-backed datasets).
80    pub managed: bool,
81    /// Created at runtime (and removable); `false` for datasets from the config.
82    pub runtime: bool,
83    pub store: Option<String>,
84    pub embedder: Option<String>,
85    pub chunking: Option<ChunkerConfig>,
86}
87
88/// Names of the stores, embedders and rerankers runtime datasets can use.
89#[derive(Clone, Debug, Default, PartialEq, Serialize)]
90pub struct ResourceNames {
91    pub stores: Vec<String>,
92    pub embedders: Vec<String>,
93    pub rerankers: Vec<String>,
94}
95
96/// Named building blocks declared by a knowledge config.
97#[derive(Clone, Debug, Default)]
98struct Resources {
99    stores: IndexMap<String, Arc<dyn VectorStore>>,
100    embedders: IndexMap<String, Arc<dyn Embedder>>,
101    rerankers: IndexMap<String, Arc<dyn Reranker>>,
102}
103
104type Registry = IndexMap<String, DatasetEntry>;
105
106/// Dataset id -> retriever registry.
107///
108/// Clones share their datasets: a dataset added to or removed from one clone
109/// (for example by the management API) is seen by every other clone (for
110/// example the one the workflows' `knowledge-retrieval` nodes hold).
111#[derive(Clone, Debug, Default)]
112pub struct KnowledgeBase {
113    datasets: Arc<RwLock<Registry>>,
114    resources: Arc<Resources>,
115    /// Reranker applied across datasets in multi-dataset retrieval.
116    reranker: Option<Arc<dyn Reranker>>,
117}
118
119impl KnowledgeBase {
120    pub fn new() -> Self {
121        Self::default()
122    }
123
124    // Every mutation replaces whole entries, so data behind a poisoned lock is still consistent.
125    fn read(&self) -> RwLockReadGuard<'_, Registry> {
126        self.datasets.read().unwrap_or_else(PoisonError::into_inner)
127    }
128
129    fn write(&self) -> RwLockWriteGuard<'_, Registry> {
130        self.datasets.write().unwrap_or_else(PoisonError::into_inner)
131    }
132
133    /// Add a dataset backed by the built-in store pipeline.
134    pub fn with_store_dataset(
135        mut self,
136        id: impl Into<String>,
137        store: Arc<dyn VectorStore>,
138        embedder: Arc<dyn Embedder>,
139        reranker: Option<Arc<dyn Reranker>>,
140    ) -> Self {
141        let id = id.into();
142        self.add_store_dataset(DatasetInfo { id, ..DatasetInfo::default() }, store, embedder, reranker);
143        self
144    }
145
146    /// Add (or replace) a store-backed dataset with descriptive info.
147    pub fn add_store_dataset(
148        &mut self,
149        info: DatasetInfo,
150        store: Arc<dyn VectorStore>,
151        embedder: Arc<dyn Embedder>,
152        reranker: Option<Arc<dyn Reranker>>,
153    ) -> &mut Self {
154        self.insert_store_dataset(info, store, embedder, reranker, ChunkerConfig::default(), None, None);
155        self
156    }
157
158    #[allow(clippy::too_many_arguments)]
159    fn insert_store_dataset(
160        &self,
161        info: DatasetInfo,
162        store: Arc<dyn VectorStore>,
163        embedder: Arc<dyn Embedder>,
164        reranker: Option<Arc<dyn Reranker>>,
165        chunking: ChunkerConfig,
166        names: Option<(String, String)>,
167        runtime: Option<DatasetSpec>,
168    ) {
169        let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
170        let entry = DatasetEntry {
171            info,
172            retriever: Arc::new(retriever),
173            store: Some((store, embedder)),
174            chunking,
175            names,
176            runtime,
177        };
178        self.write().insert(entry.info.id.clone(), entry);
179    }
180
181    /// Add a dataset served by an external retriever.
182    pub fn with_retriever(mut self, id: impl Into<String>, retriever: Arc<dyn Retriever>) -> Self {
183        let id = id.into();
184        self.add_retriever(DatasetInfo { id, ..DatasetInfo::default() }, retriever);
185        self
186    }
187
188    /// Add (or replace) a dataset served by an external retriever.
189    pub fn add_retriever(&mut self, info: DatasetInfo, retriever: Arc<dyn Retriever>) -> &mut Self {
190        let entry = DatasetEntry {
191            info,
192            retriever,
193            store: None,
194            chunking: ChunkerConfig::default(),
195            names: None,
196            runtime: None,
197        };
198        self.write().insert(entry.info.id.clone(), entry);
199        self
200    }
201
202    /// Reranker applied to the merged results of multi-dataset retrieval.
203    pub fn with_reranker(mut self, reranker: Option<Arc<dyn Reranker>>) -> Self {
204        self.reranker = reranker;
205        self
206    }
207
208    /// Every dataset's id, name and description, in registration order.
209    pub fn datasets(&self) -> Vec<DatasetInfo> {
210        self.read().values().map(|d| d.info.clone()).collect()
211    }
212
213    /// Every dataset with what a management UI shows about it.
214    pub fn dataset_details(&self) -> Vec<DatasetDetails> {
215        self.read()
216            .values()
217            .map(|d| DatasetDetails {
218                info: d.info.clone(),
219                managed: d.store.is_some(),
220                runtime: d.runtime.is_some(),
221                store: d.names.as_ref().map(|(store, _)| store.clone()),
222                embedder: d.names.as_ref().map(|(_, embedder)| embedder.clone()),
223                chunking: d.store.as_ref().map(|_| d.chunking.clone()),
224            })
225            .collect()
226    }
227
228    /// The stores, embedders and rerankers [`KnowledgeBase::create_dataset`] can use.
229    pub fn resource_names(&self) -> ResourceNames {
230        ResourceNames {
231            stores: self.resources.stores.keys().cloned().collect(),
232            embedders: self.resources.embedders.keys().cloned().collect(),
233            rerankers: self.resources.rerankers.keys().cloned().collect(),
234        }
235    }
236
237    /// Create a store-backed dataset at runtime from the named resources of the
238    /// knowledge config. Every clone of this knowledge base sees it at once.
239    ///
240    /// @throws `RagError::Input` for an invalid or taken id, an unknown store,
241    /// embedder or reranker name, or invalid chunking.
242    pub fn create_dataset(&self, spec: DatasetSpec) -> Result<DatasetDetails, RagError> {
243        validate_dataset_id(&spec.id)?;
244        let lookup = |kind: &str, name: &str, found: bool| {
245            if found {
246                Ok(())
247            } else {
248                Err(RagError::Input(format!("unknown {kind} `{name}` in the knowledge config")))
249            }
250        };
251        let store = self.resources.stores.get(&spec.store).cloned();
252        lookup("store", &spec.store, store.is_some())?;
253        let embedder = self.resources.embedders.get(&spec.embedder).cloned();
254        lookup("embedder", &spec.embedder, embedder.is_some())?;
255        let reranker = match &spec.reranker {
256            Some(name) => {
257                let reranker = self.resources.rerankers.get(name).cloned();
258                lookup("reranker", name, reranker.is_some())?;
259                reranker
260            }
261            None => None,
262        };
263        let chunking = spec.chunking.clone().unwrap_or_default();
264        validate_chunking(&spec.id, &chunking).map_err(|e| RagError::Input(e.to_string()))?;
265        let (Some(store), Some(embedder)) = (store, embedder) else {
266            return Err(RagError::Input("store and embedder are required".into()));
267        };
268        let info = DatasetInfo { id: spec.id.clone(), name: spec.name.clone(), description: spec.description.clone() };
269        {
270            // Check and insert under one lock so two requests cannot create the same id.
271            let mut registry = self.write();
272            if registry.contains_key(&spec.id) {
273                return Err(RagError::Input(format!("dataset `{}` already exists", spec.id)));
274            }
275            let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
276            registry.insert(
277                spec.id.clone(),
278                DatasetEntry {
279                    info,
280                    retriever: Arc::new(retriever),
281                    store: Some((store, embedder)),
282                    chunking,
283                    names: Some((spec.store.clone(), spec.embedder.clone())),
284                    runtime: Some(spec.clone()),
285                },
286            );
287        }
288        self.dataset_details()
289            .into_iter()
290            .find(|d| d.info.id == spec.id)
291            .ok_or_else(|| RagError::Store(format!("dataset `{}` vanished after creation", spec.id)))
292    }
293
294    /// Delete a dataset created at runtime together with its documents.
295    /// Returns the number of chunks removed.
296    ///
297    /// @throws `RagError::NotFound` for an unknown dataset, `RagError::Unsupported`
298    /// for one declared in the knowledge config (edit the config instead).
299    pub async fn remove_dataset(&self, dataset_id: &str) -> Result<usize, RagError> {
300        let entry = self.entry(dataset_id)?;
301        if entry.runtime.is_none() {
302            return Err(RagError::Unsupported(format!(
303                "dataset `{dataset_id}` is declared in the knowledge config; remove it there"
304            )));
305        }
306        // Delete the data first: if that fails the dataset stays, instead of leaving orphaned chunks.
307        let removed = match &entry.store {
308            Some((store, _)) => store.delete_dataset(dataset_id).await?,
309            None => 0,
310        };
311        self.write().shift_remove(dataset_id);
312        Ok(removed)
313    }
314
315    /// Specs of the datasets created at runtime, for persisting them.
316    pub fn runtime_datasets(&self) -> Vec<DatasetSpec> {
317        self.read().values().filter_map(|d| d.runtime.clone()).collect()
318    }
319
320    pub fn contains(&self, dataset_id: &str) -> bool {
321        self.read().contains_key(dataset_id)
322    }
323
324    /// An ingestor for a store-backed dataset (`None` for external retrievers or unknown ids),
325    /// using the dataset's chunking settings.
326    pub fn ingestor(&self, dataset_id: &str) -> Option<Ingestor> {
327        let registry = self.read();
328        let entry = registry.get(dataset_id)?;
329        let (store, embedder) = entry.store.as_ref()?;
330        Some(Ingestor::new(store.clone(), embedder.clone()).chunker(entry.chunking.clone()))
331    }
332
333    /// Set how documents ingested into `dataset_id` are chunked (e.g. parent-child).
334    ///
335    /// @throws `RagError::NotFound` for an unknown dataset, `RagError::Config` when
336    /// `parent_max_chars` is not larger than `max_chars`.
337    pub fn set_chunking(&mut self, dataset_id: &str, chunking: ChunkerConfig) -> Result<&mut Self, RagError> {
338        validate_chunking(dataset_id, &chunking)?;
339        {
340            let mut registry = self.write();
341            let entry = registry
342                .get_mut(dataset_id)
343                .ok_or_else(|| RagError::NotFound(format!("unknown dataset `{dataset_id}`")))?;
344            entry.chunking = chunking;
345        }
346        Ok(self)
347    }
348
349    /// The chunking settings of a dataset.
350    pub fn chunking(&self, dataset_id: &str) -> Option<ChunkerConfig> {
351        self.read().get(dataset_id).map(|d| d.chunking.clone())
352    }
353
354    /// A copy of a dataset's entry (cheap: shared handles), so no lock is held across awaits.
355    fn entry(&self, id: &str) -> Result<DatasetEntry, RagError> {
356        self.read().get(id).cloned().ok_or_else(|| RagError::NotFound(format!("unknown dataset `{id}`")))
357    }
358
359    /// Whether documents can be added to, listed in and removed from this dataset
360    /// (store-backed datasets; `false` for external retrievers and unknown ids).
361    pub fn is_managed(&self, dataset_id: &str) -> bool {
362        self.read().get(dataset_id).is_some_and(|d| d.store.is_some())
363    }
364
365    /// The store of a store-backed dataset.
366    ///
367    /// @throws `RagError::NotFound` for an unknown dataset, `RagError::Unsupported`
368    /// for one served by an external retriever.
369    fn managed_store(&self, dataset_id: &str) -> Result<Arc<dyn VectorStore>, RagError> {
370        let entry = self.entry(dataset_id)?;
371        entry
372            .store
373            .map(|(store, _)| store)
374            .ok_or_else(|| RagError::Unsupported(format!("dataset `{dataset_id}` is served by an external retriever")))
375    }
376
377    /// Up to `limit` documents of a dataset (capped at [`MAX_DOCUMENT_PAGE`]),
378    /// ordered by id and starting after the `after` id.
379    pub async fn list_documents(
380        &self,
381        dataset_id: &str,
382        limit: usize,
383        after: Option<&str>,
384    ) -> Result<Vec<DocumentSummary>, RagError> {
385        let limit = limit.clamp(1, MAX_DOCUMENT_PAGE);
386        self.managed_store(dataset_id)?.list_documents(dataset_id, limit, after).await
387    }
388
389    /// Chunks of one document in chunk order (vectors included).
390    pub async fn document_chunks(&self, dataset_id: &str, document_id: &str) -> Result<Vec<StoredChunk>, RagError> {
391        let mut chunks = self.managed_store(dataset_id)?.document_chunks(dataset_id, document_id).await?;
392        chunks.sort_by_key(|c| c.metadata.get("chunk_index").and_then(serde_json::Value::as_u64).unwrap_or(u64::MAX));
393        Ok(chunks)
394    }
395
396    /// Enable or disable a document: a disabled document keeps its chunks (and
397    /// stays disabled across re-ingestion) but no search returns it. Returns the
398    /// number of chunks of the document.
399    ///
400    /// @throws `RagError::NotFound` when the document does not exist.
401    pub async fn set_document_enabled(
402        &self,
403        dataset_id: &str,
404        document_id: &str,
405        enabled: bool,
406    ) -> Result<usize, RagError> {
407        let store = self.managed_store(dataset_id)?;
408        let mut chunks = store.document_chunks(dataset_id, document_id).await?;
409        if chunks.is_empty() {
410            return Err(RagError::NotFound(format!("unknown document `{document_id}` in dataset `{dataset_id}`")));
411        }
412        let count = chunks.len();
413        chunks.retain(|c| crate::types::is_disabled(&c.metadata) == enabled);
414        for chunk in &mut chunks {
415            if enabled {
416                chunk.metadata.remove(crate::types::DISABLED_KEY);
417            } else {
418                chunk.metadata.insert(crate::types::DISABLED_KEY.into(), serde_json::Value::Bool(true));
419            }
420        }
421        if !chunks.is_empty() {
422            store.upsert(chunks).await?;
423        }
424        Ok(count)
425    }
426
427    /// Remove a document. Returns the number of chunks removed (0 when it did not exist).
428    pub async fn delete_document(&self, dataset_id: &str, document_id: &str) -> Result<usize, RagError> {
429        self.managed_store(dataset_id)?.delete_document(dataset_id, document_id).await
430    }
431
432    /// Query every dataset concurrently. Returns `(dataset id, hits)` in request order.
433    async fn query_each(
434        &self,
435        request: &RetrievalRequest,
436        rerank_each: bool,
437    ) -> Result<Vec<(String, Vec<Hit>)>, RagError> {
438        let mut jobs = Vec::with_capacity(request.datasets.len());
439        for id in &request.datasets {
440            let retriever = self.entry(id)?.retriever;
441            let sub = RetrievalRequest {
442                datasets: vec![id.clone()],
443                // Threshold and top-k are applied after merging.
444                score_threshold: None,
445                rerank: rerank_each,
446                ..request.clone()
447            };
448            jobs.push(async move { retriever.retrieve(&sub).await.map(|hits| (id.clone(), hits)) });
449        }
450        futures::future::try_join_all(jobs).await
451    }
452
453    /// Retrieve across all requested datasets and merge by score.
454    ///
455    /// When `request.rerank` is set and the knowledge base has a cross-dataset
456    /// reranker, the merged list is reranked as a whole (making scores from
457    /// different datasets comparable). Otherwise each dataset's own reranker, if
458    /// any, is applied before merging.
459    pub async fn retrieve(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
460        validate_request(request)?;
461        let global_rerank = request.rerank && self.reranker.is_some() && request.datasets.len() > 1;
462        let per_dataset = self.query_each(request, request.rerank && !global_rerank).await?;
463        let mut merged: Vec<Hit> = per_dataset.into_iter().flat_map(|(_, hits)| hits).collect();
464        if global_rerank {
465            if let Some(reranker) = &self.reranker {
466                merged = rerank(reranker.as_ref(), &request.query, merged).await?;
467            }
468        }
469        Ok(finalize(merged, request.score_threshold, request.top_k))
470    }
471
472    /// Retrieve from the single best-matching dataset: every dataset is queried,
473    /// and only the hits of the dataset whose top hit scores highest are kept.
474    pub async fn retrieve_single(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
475        validate_request(request)?;
476        let per_dataset = self.query_each(request, request.rerank).await?;
477        let best = per_dataset
478            .into_iter()
479            .filter(|(_, hits)| !hits.is_empty())
480            .max_by(|a, b| top_score(&a.1).total_cmp(&top_score(&b.1)));
481        Ok(best.map(|(_, hits)| finalize(hits, request.score_threshold, request.top_k)).unwrap_or_default())
482    }
483}
484
485/// Runtime dataset ids appear in URLs and file names: 1-64 ASCII letters,
486/// digits, `-`, `_` or `.`, not starting with `.`.
487fn validate_dataset_id(id: &str) -> Result<(), RagError> {
488    let ok = !id.is_empty()
489        && id.len() <= 64
490        && !id.starts_with('.')
491        && id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'));
492    if ok {
493        Ok(())
494    } else {
495        Err(RagError::Input(format!(
496            "invalid dataset id `{id}`: use 1-64 ASCII letters, digits, `-`, `_` or `.`, not starting with `.`"
497        )))
498    }
499}
500
501fn validate_chunking(dataset_id: &str, chunking: &ChunkerConfig) -> Result<(), RagError> {
502    if chunking.max_chars == 0 {
503        return Err(RagError::Config(format!("dataset `{dataset_id}`: `chunking.max_chars` must be positive")));
504    }
505    if chunking.parent_max_chars.is_some_and(|p| p <= chunking.max_chars) {
506        return Err(RagError::Config(format!(
507            "dataset `{dataset_id}`: `chunking.parent_max_chars` must be larger than `max_chars`"
508        )));
509    }
510    Ok(())
511}
512
513/// Reject requests that would otherwise silently match nothing.
514fn validate_request(request: &RetrievalRequest) -> Result<(), RagError> {
515    if request.datasets.is_empty() {
516        return Err(RagError::Input("no dataset ids given".into()));
517    }
518    if let Some(filter) = &request.filter {
519        filter.validate().map_err(|e| RagError::Input(e.to_string()))?;
520    }
521    Ok(())
522}
523
524fn top_score(hits: &[Hit]) -> f32 {
525    hits.iter().map(|h| h.score).fold(f32::MIN, f32::max)
526}
527
528// ---------------------------------------------------------------------------
529// Configuration
530// ---------------------------------------------------------------------------
531
532/// Declarative knowledge-base configuration (`knowledge.yaml` / `.json`).
533///
534/// ```yaml
535/// embedders:
536///   default: { type: openai, base_url: "http://localhost:11434/v1", model: nomic-embed-text }
537///   inproc:  { type: local, model: sentence-transformers/all-MiniLM-L6-v2 }   # feature `local-embed`
538///   cloud:   { type: openai, model: text-embedding-3-small, api_key_env: OPENAI_API_KEY }
539/// rerankers:
540///   cohere: { type: http, base_url: "https://api.cohere.com/v2", model: rerank-v3.5, api_key_env: COHERE_API_KEY }
541/// stores:
542///   local:  { type: sqlite, path: ./kb.db }
543///   json:   { type: memory, path: ./kb.json }
544///   vector: { type: qdrant, url: "http://localhost:6333", collection: kb, api_key_env: QDRANT_API_KEY }
545/// datasets:
546///   - { id: handbook, name: Handbook, store: local, embedder: default, reranker: cohere }
547///   - { id: crm, retriever: { type: http, url: "https://rag.internal/search", api_key_env: RAG_TOKEN } }
548/// reranker: cohere   # optional, used across datasets
549/// ```
550///
551/// Secrets are never written inline: every provider takes `api_key_env`, the
552/// name of an environment variable resolved when the configuration is built.
553#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
554#[serde(deny_unknown_fields)]
555pub struct KnowledgeConfig {
556    #[serde(default)]
557    pub embedders: IndexMap<String, serde_json::Value>,
558    #[serde(default)]
559    pub rerankers: IndexMap<String, serde_json::Value>,
560    #[serde(default)]
561    pub stores: IndexMap<String, serde_json::Value>,
562    #[serde(default)]
563    pub datasets: Vec<DatasetConfig>,
564    /// Cross-dataset reranker (a key of `rerankers`).
565    #[serde(default)]
566    pub reranker: Option<String>,
567}
568
569/// One dataset entry.
570#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
571#[serde(deny_unknown_fields)]
572pub struct DatasetConfig {
573    pub id: String,
574    #[serde(default)]
575    pub name: String,
576    #[serde(default)]
577    pub description: String,
578    /// Key of `stores` (store-backed datasets).
579    #[serde(default)]
580    pub store: Option<String>,
581    /// Key of `embedders` (store-backed datasets).
582    #[serde(default)]
583    pub embedder: Option<String>,
584    /// Key of `rerankers`.
585    #[serde(default)]
586    pub reranker: Option<String>,
587    /// External retriever definition (instead of store + embedder).
588    #[serde(default)]
589    pub retriever: Option<serde_json::Value>,
590    /// Chunking of ingested documents, e.g. `{ max_chars: 400, parent_max_chars: 1500 }`
591    /// for parent-child retrieval (store-backed datasets).
592    #[serde(default)]
593    pub chunking: Option<ChunkerConfig>,
594}
595
596/// Resolves environment variable names to values (`std::env::var` in production).
597pub type EnvLookup<'a> = &'a dyn Fn(&str) -> Option<String>;
598
599impl KnowledgeConfig {
600    pub fn from_json(text: &str) -> Result<Self, RagError> {
601        serde_json::from_str(text).map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
602    }
603
604    #[cfg(feature = "yaml")]
605    pub fn from_yaml(text: &str) -> Result<Self, RagError> {
606        let options = serde_saphyr::options! { strict_booleans: true };
607        serde_saphyr::from_str_with_options(text, options)
608            .map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
609    }
610
611    /// Build the knowledge base, resolving secrets with `std::env::var`.
612    pub fn build(&self) -> Result<KnowledgeBase, RagError> {
613        self.build_with_env(&|name| std::env::var(name).ok())
614    }
615
616    /// Build the knowledge base with a custom environment lookup.
617    pub fn build_with_env(&self, env: EnvLookup<'_>) -> Result<KnowledgeBase, RagError> {
618        let mut embedders: IndexMap<&str, Arc<dyn Embedder>> = IndexMap::new();
619        for (name, raw) in &self.embedders {
620            embedders.insert(name, build_embedder(name, raw, env)?);
621        }
622        let mut rerankers: IndexMap<&str, Arc<dyn Reranker>> = IndexMap::new();
623        for (name, raw) in &self.rerankers {
624            rerankers.insert(name, build_reranker(name, raw, env)?);
625        }
626        let mut stores: IndexMap<&str, Arc<dyn VectorStore>> = IndexMap::new();
627        for (name, raw) in &self.stores {
628            stores.insert(name, build_store(name, raw, env)?);
629        }
630
631        let lookup_reranker = |key: &Option<String>| -> Result<Option<Arc<dyn Reranker>>, RagError> {
632            match key {
633                None => Ok(None),
634                Some(k) => rerankers
635                    .get(k.as_str())
636                    .cloned()
637                    .map(Some)
638                    .ok_or_else(|| RagError::Config(format!("unknown reranker `{k}`"))),
639            }
640        };
641
642        let mut kb = KnowledgeBase::new().with_reranker(lookup_reranker(&self.reranker)?);
643        for ds in &self.datasets {
644            if ds.id.trim().is_empty() {
645                return Err(RagError::Config("dataset id must not be empty".into()));
646            }
647            if kb.contains(&ds.id) {
648                return Err(RagError::Config(format!("duplicate dataset `{}`", ds.id)));
649            }
650            let info = DatasetInfo { id: ds.id.clone(), name: ds.name.clone(), description: ds.description.clone() };
651            if let Some(raw) = &ds.retriever {
652                kb.add_retriever(info, build_retriever(&ds.id, raw, env)?);
653                continue;
654            }
655            let store_key = ds.store.as_deref().ok_or_else(|| {
656                RagError::Config(format!("dataset `{}` needs either `store` + `embedder` or `retriever`", ds.id))
657            })?;
658            let store = stores
659                .get(store_key)
660                .cloned()
661                .ok_or_else(|| RagError::Config(format!("unknown store `{store_key}`")))?;
662            let embedder_key = match ds.embedder.as_deref() {
663                Some(k) => k,
664                None if embedders.len() == 1 => embedders.keys().next().copied().unwrap_or_default(),
665                None => return Err(RagError::Config(format!("dataset `{}` needs `embedder`", ds.id))),
666            };
667            let embedder = embedders
668                .get(embedder_key)
669                .cloned()
670                .ok_or_else(|| RagError::Config(format!("unknown embedder `{embedder_key}`")))?;
671            let chunking = ds.chunking.clone().unwrap_or_default();
672            validate_chunking(&ds.id, &chunking)?;
673            let names = Some((store_key.to_owned(), embedder_key.to_owned()));
674            kb.insert_store_dataset(info, store, embedder, lookup_reranker(&ds.reranker)?, chunking, names, None);
675        }
676        kb.resources = Arc::new(Resources {
677            stores: stores.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
678            embedders: embedders.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
679            rerankers: rerankers.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
680        });
681        Ok(kb)
682    }
683}
684
685/// Keys accepted by every HTTP-backed provider entry.
686#[cfg_attr(not(feature = "http"), allow(dead_code))]
687const HTTP_KEYS: &[&str] = &["type", "api_key", "api_key_env", "api_key_header", "headers", "timeout_ms"];
688
689/// Reject keys a provider entry does not understand (typos would otherwise be
690/// ignored silently, e.g. `file:` instead of `path:` turning a persistent store
691/// into an in-memory one).
692fn check_keys(name: &str, raw: &serde_json::Value, allowed: &[&str], with_http: bool) -> Result<(), RagError> {
693    let obj = raw.as_object().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
694    for key in obj.keys() {
695        let known = allowed.contains(&key.as_str()) || (with_http && HTTP_KEYS.contains(&key.as_str()));
696        if !known && key != "type" {
697            return Err(RagError::Config(format!("`{name}`: unknown key `{key}`")));
698        }
699    }
700    Ok(())
701}
702
703fn provider_type(name: &str, raw: &serde_json::Value) -> Result<String, RagError> {
704    raw.get("type")
705        .and_then(|t| t.as_str())
706        .map(str::to_owned)
707        .ok_or_else(|| RagError::Config(format!("`{name}` needs a `type`")))
708}
709
710/// Strip `type` / `api_key_env`, reject inline secrets, and inject the resolved key.
711#[cfg_attr(not(feature = "http"), allow(dead_code))]
712fn with_secret(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<serde_json::Value, RagError> {
713    let mut obj = raw.as_object().cloned().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
714    obj.remove("type");
715    if obj.contains_key("api_key") {
716        return Err(RagError::Config(format!(
717            "`{name}`: inline `api_key` is not allowed; use `api_key_env` with an environment variable name"
718        )));
719    }
720    if let Some(var) = obj.remove("api_key_env") {
721        let var = var.as_str().ok_or_else(|| RagError::Config(format!("`{name}`: `api_key_env` must be a string")))?;
722        let value =
723            env(var).ok_or_else(|| RagError::Config(format!("`{name}`: environment variable `{var}` is not set")))?;
724        obj.insert("api_key".into(), serde_json::Value::String(value));
725    }
726    Ok(serde_json::Value::Object(obj))
727}
728
729#[cfg_attr(not(feature = "http"), allow(dead_code))]
730fn parse<T: serde::de::DeserializeOwned>(name: &str, value: serde_json::Value) -> Result<T, RagError> {
731    serde_json::from_value(value).map_err(|e| RagError::Config(format!("`{name}`: {e}")))
732}
733
734fn build_embedder(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Embedder>, RagError> {
735    let kind = provider_type(name, raw)?;
736    match kind.as_str() {
737        #[cfg(feature = "http")]
738        "openai" | "openai-compatible" | "ollama" => {
739            check_keys(
740                name,
741                raw,
742                &["base_url", "model", "dimensions", "batch_size", "query_prefix", "document_prefix"],
743                true,
744            )?;
745            let mut value = with_secret(name, raw, env)?;
746            if kind == "ollama" {
747                if let Some(obj) = value.as_object_mut() {
748                    obj.entry("base_url").or_insert_with(|| "http://localhost:11434/v1".into());
749                }
750            }
751            let cfg: crate::http::OpenAiEmbedderConfig = parse(name, value)?;
752            Ok(Arc::new(crate::http::OpenAiEmbedder::new(cfg)?))
753        }
754        #[cfg(feature = "local-embed")]
755        "local" => {
756            let mut value = raw.as_object().cloned().unwrap_or_default();
757            value.remove("type");
758            let config: crate::local_embed::LocalEmbedderConfig = parse(name, serde_json::Value::Object(value))?;
759            Ok(Arc::new(crate::local_embed::LocalEmbedder::new(config)?))
760        }
761        "hash" => {
762            check_keys(name, raw, &["dimensions"], false)?;
763            let dims = match raw.get("dimensions") {
764                None => 256,
765                Some(v) => v
766                    .as_u64()
767                    .and_then(|d| usize::try_from(d).ok())
768                    .filter(|d| *d > 0)
769                    .ok_or_else(|| RagError::Config(format!("`{name}`: `dimensions` must be a positive integer")))?,
770            };
771            let _ = env;
772            Ok(Arc::new(crate::hash::HashEmbedder::new(dims)))
773        }
774        other => Err(RagError::Config(format!("embedder `{name}`: unsupported type `{other}` in this build"))),
775    }
776}
777
778fn build_reranker(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Reranker>, RagError> {
779    let kind = provider_type(name, raw)?;
780    match kind.as_str() {
781        #[cfg(feature = "http")]
782        "http" | "cohere" | "jina" | "voyage" => {
783            check_keys(name, raw, &["base_url", "model"], true)?;
784            let cfg: crate::http::HttpRerankerConfig = parse(name, with_secret(name, raw, env)?)?;
785            Ok(Arc::new(crate::http::HttpReranker::new(cfg)?))
786        }
787        other => {
788            let _ = env;
789            Err(RagError::Config(format!("reranker `{name}`: unsupported type `{other}` in this build")))
790        }
791    }
792}
793
794fn build_store(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn VectorStore>, RagError> {
795    let kind = provider_type(name, raw)?;
796    match kind.as_str() {
797        "memory" => {
798            check_keys(name, raw, &["path"], false)?;
799            let path = match raw.get("path") {
800                None => None,
801                Some(serde_json::Value::String(p)) => Some(p.as_str()),
802                Some(_) => return Err(RagError::Config(format!("store `{name}`: `path` must be a string"))),
803            };
804            match path {
805                #[cfg(feature = "fs")]
806                Some(path) => Ok(Arc::new(crate::memory::MemoryStore::open(path)?)),
807                #[cfg(not(feature = "fs"))]
808                Some(_) => Err(RagError::Config(format!("store `{name}`: `path` requires the `fs` feature"))),
809                None => Ok(Arc::new(crate::memory::MemoryStore::new())),
810            }
811        }
812        #[cfg(feature = "sqlite")]
813        "sqlite" => {
814            check_keys(name, raw, &["path"], false)?;
815            let path =
816                raw.get("path").and_then(serde_json::Value::as_str).filter(|p| !p.trim().is_empty()).ok_or_else(
817                    || RagError::Config(format!("store `{name}`: `path` (the database file) is required")),
818                )?;
819            Ok(Arc::new(crate::sqlite::SqliteStore::open(path)?))
820        }
821        #[cfg(feature = "qdrant")]
822        "qdrant" => {
823            check_keys(name, raw, &["url", "collection", "create_collection"], true)?;
824            let mut value = with_secret(name, raw, env)?;
825            if let Some(obj) = value.as_object_mut() {
826                obj.entry("api_key_header").or_insert_with(|| "api-key".into());
827            }
828            let cfg: crate::qdrant::QdrantConfig = parse(name, value)?;
829            Ok(Arc::new(crate::qdrant::QdrantStore::new(cfg)?))
830        }
831        other => {
832            let _ = env;
833            Err(RagError::Config(format!("store `{name}`: unsupported type `{other}` in this build")))
834        }
835    }
836}
837
838fn build_retriever(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Retriever>, RagError> {
839    let kind = provider_type(name, raw)?;
840    match kind.as_str() {
841        #[cfg(feature = "http")]
842        "http" => {
843            check_keys(name, raw, &["url"], true)?;
844            let cfg: crate::http::HttpRetrieverConfig = parse(name, with_secret(name, raw, env)?)?;
845            Ok(Arc::new(crate::http::HttpRetriever::new(cfg)?))
846        }
847        other => {
848            let _ = env;
849            Err(RagError::Config(format!("retriever for `{name}`: unsupported type `{other}` in this build")))
850        }
851    }
852}
853
854#[cfg(test)]
855mod tests {
856    use super::*;
857    use crate::hash::HashEmbedder;
858    use crate::ingest::Document;
859    use crate::memory::MemoryStore;
860
861    fn kb_with_two_datasets() -> KnowledgeBase {
862        let embedder: Arc<dyn Embedder> = Arc::new(HashEmbedder::new(128));
863        KnowledgeBase::new()
864            .with_store_dataset("pets", Arc::new(MemoryStore::new()), embedder.clone(), None)
865            .with_store_dataset("space", Arc::new(MemoryStore::new()), embedder, None)
866    }
867
868    async fn ingest(kb: &KnowledgeBase, ds: &str, id: &str, text: &str) {
869        kb.ingestor(ds).unwrap().ingest(&Document::new(ds, id, text)).await.unwrap();
870    }
871
872    #[tokio::test]
873    async fn documents_can_be_listed_paged_inspected_and_deleted() {
874        let kb = kb_with_two_datasets();
875        let ingestor = kb.ingestor("pets").unwrap();
876        let mut meta = crate::types::Metadata::new();
877        meta.insert("lang".into(), serde_json::json!("en"));
878        let long = "Cats purr. ".repeat(200);
879        ingestor.ingest(&Document::new("pets", "cats", long).title("Cats").metadata(meta)).await.unwrap();
880        ingest(&kb, "pets", "dogs", "Dogs bark.").await;
881        ingest(&kb, "pets", "birds", "Birds sing.").await;
882
883        let first = kb.list_documents("pets", 2, None).await.unwrap();
884        let ids: Vec<&str> = first.iter().map(|d| d.document_id.as_str()).collect();
885        assert_eq!(ids, ["birds", "cats"]);
886        let cats = &first[1];
887        assert_eq!(cats.title, "Cats");
888        assert!(cats.chunks > 1, "long document is split");
889        assert_eq!(cats.metadata.get("lang"), Some(&serde_json::json!("en")));
890        assert!(!cats.metadata.contains_key("chunk_index"), "per-chunk keys are not document metadata");
891        let next = kb.list_documents("pets", 2, Some("cats")).await.unwrap();
892        assert_eq!(next.iter().map(|d| d.document_id.as_str()).collect::<Vec<_>>(), ["dogs"]);
893
894        let chunks = kb.document_chunks("pets", "cats").await.unwrap();
895        assert_eq!(chunks.len(), cats.chunks);
896        let order: Vec<u64> = chunks.iter().map(|c| c.metadata["chunk_index"].as_u64().unwrap()).collect();
897        assert!(order.windows(2).all(|w| w[0] < w[1]), "chunks come in document order: {order:?}");
898
899        assert_eq!(kb.delete_document("pets", "cats").await.unwrap(), cats.chunks);
900        assert_eq!(kb.delete_document("pets", "cats").await.unwrap(), 0);
901        assert_eq!(kb.list_documents("pets", 50, None).await.unwrap().len(), 2);
902    }
903
904    #[tokio::test]
905    async fn disabled_documents_are_kept_but_never_retrieved() {
906        let kb = kb_with_two_datasets();
907        ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
908        ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
909        let query = RetrievalRequest::new("pets that purr", vec!["pets".into()]).top_k(5);
910        let found = |hits: &[Hit]| hits.iter().any(|h| h.document_id == "cats");
911        assert!(found(&kb.retrieve(&query).await.unwrap()));
912
913        assert!(kb.set_document_enabled("pets", "cats", false).await.unwrap() >= 1);
914        assert!(!found(&kb.retrieve(&query).await.unwrap()), "disabled document retrieved");
915        let docs = kb.list_documents("pets", 10, None).await.unwrap();
916        let cats = docs.iter().find(|d| d.document_id == "cats").unwrap();
917        assert!(!cats.enabled);
918        assert!(!cats.metadata.contains_key(crate::types::DISABLED_KEY), "flag is not document metadata");
919
920        // A new version of a disabled document stays disabled.
921        ingest(&kb, "pets", "cats", "Cats are small furry pets that purr and nap.").await;
922        assert!(!found(&kb.retrieve(&query).await.unwrap()), "re-ingestion re-enabled the document");
923
924        kb.set_document_enabled("pets", "cats", true).await.unwrap();
925        assert!(found(&kb.retrieve(&query).await.unwrap()));
926        assert_eq!(kb.set_document_enabled("pets", "nope", false).await.unwrap_err().kind(), "not_found");
927    }
928
929    #[tokio::test]
930    async fn parent_child_datasets_return_whole_parents_once() {
931        let mut kb = KnowledgeBase::new().with_store_dataset(
932            "docs",
933            Arc::new(MemoryStore::new()),
934            Arc::new(HashEmbedder::new(256)),
935            None,
936        );
937        let chunking = ChunkerConfig { max_chars: 60, overlap: 0, markdown: false, parent_max_chars: Some(220) };
938        kb.set_chunking("docs", chunking).unwrap();
939        let text = "Leave policy. Every employee gets 24 days of paid annual leave. Unused leave carries over for six months. Sick leave is separate.\n\n\
940                    Expense policy. Expenses below 500 USD are approved by your manager. Larger ones need finance approval.";
941        ingest(&kb, "docs", "handbook", text).await;
942        let children = kb.document_chunks("docs", "handbook").await.unwrap();
943        assert!(children.len() > 2, "the document is split into several children");
944
945        let request = RetrievalRequest::new("annual leave days carries over", vec!["docs".into()]).top_k(5);
946        let hits = kb.retrieve(&request).await.unwrap();
947        let leave: Vec<&Hit> = hits.iter().filter(|h| h.content.contains("24 days")).collect();
948        assert_eq!(leave.len(), 1, "children of one parent collapse into one hit: {hits:?}");
949        let parent = &leave[0].content;
950        assert!(parent.contains("Leave policy") && parent.contains("Sick leave"), "whole parent returned: {parent}");
951        assert!(!parent.contains("Expense"), "parents stay separate: {parent}");
952    }
953
954    #[test]
955    fn chunking_config_is_validated() {
956        let build = |chunking: serde_json::Value| {
957            KnowledgeConfig::from_json(
958                &serde_json::json!({
959                    "embedders": { "e": { "type": "hash" } },
960                    "stores": { "m": { "type": "memory" } },
961                    "datasets": [{ "id": "d", "store": "m", "embedder": "e", "chunking": chunking }]
962                })
963                .to_string(),
964            )
965            .unwrap()
966            .build_with_env(&|_| None)
967        };
968        let kb = build(serde_json::json!({ "max_chars": 300, "parent_max_chars": 1200 })).unwrap();
969        assert_eq!(kb.chunking("d").unwrap().parent_max_chars, Some(1200));
970        let error = build(serde_json::json!({ "max_chars": 300, "parent_max_chars": 200 })).unwrap_err();
971        assert!(error.to_string().contains("parent_max_chars"), "{error}");
972    }
973
974    fn config_kb() -> KnowledgeBase {
975        KnowledgeConfig::from_json(
976            &serde_json::json!({
977                "embedders": { "e": { "type": "hash", "dimensions": 64 } },
978                "stores": { "shared": { "type": "memory" } },
979                "datasets": [{ "id": "static", "store": "shared", "embedder": "e" }]
980            })
981            .to_string(),
982        )
983        .unwrap()
984        .build_with_env(&|_| None)
985        .unwrap()
986    }
987
988    fn spec(id: &str) -> DatasetSpec {
989        DatasetSpec {
990            id: id.into(),
991            name: "Runtime".into(),
992            description: String::new(),
993            store: "shared".into(),
994            embedder: "e".into(),
995            reranker: None,
996            chunking: None,
997        }
998    }
999
1000    #[tokio::test]
1001    async fn runtime_datasets_are_shared_by_clones_and_removable() {
1002        let api = config_kb();
1003        let workflows = api.clone();
1004        assert_eq!(api.resource_names().stores, ["shared"]);
1005
1006        let created = api.create_dataset(spec("notes")).unwrap();
1007        assert!(created.runtime && created.managed);
1008        assert_eq!(created.store.as_deref(), Some("shared"));
1009        assert!(workflows.contains("notes"), "clones see runtime datasets");
1010
1011        ingest(&api, "notes", "n1", "Runtime datasets work like any other dataset.").await;
1012        ingest(&api, "static", "s1", "The static dataset shares the store.").await;
1013        let hits = workflows.retrieve(&RetrievalRequest::new("runtime datasets", vec!["notes".into()])).await.unwrap();
1014        assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("n1"));
1015
1016        assert_eq!(api.runtime_datasets(), vec![spec("notes")]);
1017        assert_eq!(api.remove_dataset("notes").await.unwrap(), 1);
1018        assert!(!workflows.contains("notes"));
1019        assert_eq!(api.list_documents("static", 10, None).await.unwrap().len(), 1, "other datasets keep their data");
1020        assert_eq!(api.remove_dataset("static").await.unwrap_err().kind(), "unsupported");
1021        assert_eq!(api.remove_dataset("missing").await.unwrap_err().kind(), "not_found");
1022    }
1023
1024    #[test]
1025    fn runtime_dataset_specs_are_validated() {
1026        let kb = config_kb();
1027        let error = |spec: DatasetSpec| kb.create_dataset(spec).unwrap_err().to_string();
1028        assert!(error(spec("static")).contains("already exists"));
1029        assert!(error(spec("bad/id")).contains("invalid dataset id"));
1030        assert!(error(spec("")).contains("invalid dataset id"));
1031        assert!(error(DatasetSpec { store: "nope".into(), ..spec("a") }).contains("unknown store `nope`"));
1032        assert!(error(DatasetSpec { embedder: "nope".into(), ..spec("b") }).contains("unknown embedder `nope`"));
1033        assert!(error(DatasetSpec { reranker: Some("nope".into()), ..spec("c") }).contains("unknown reranker"));
1034        let bad_chunking = ChunkerConfig { max_chars: 100, overlap: 0, markdown: true, parent_max_chars: Some(50) };
1035        assert!(error(DatasetSpec { chunking: Some(bad_chunking), ..spec("d") }).contains("parent_max_chars"));
1036        assert_eq!(kb.datasets().len(), 1, "failed creations leave nothing behind");
1037    }
1038
1039    #[tokio::test]
1040    async fn external_and_unknown_datasets_are_not_managed() {
1041        let store_kb = kb_with_two_datasets();
1042        let retriever = store_kb.entry("pets").unwrap().retriever;
1043        let kb = KnowledgeBase::new().with_retriever("remote", retriever);
1044        assert!(!kb.is_managed("remote"));
1045        assert_eq!(kb.list_documents("remote", 10, None).await.unwrap_err().kind(), "unsupported");
1046        assert_eq!(kb.delete_document("remote", "x").await.unwrap_err().kind(), "unsupported");
1047        assert_eq!(kb.list_documents("missing", 10, None).await.unwrap_err().kind(), "not_found");
1048    }
1049
1050    #[tokio::test]
1051    async fn multi_dataset_merge_and_single_mode() {
1052        let kb = kb_with_two_datasets();
1053        ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
1054        ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
1055        ingest(&kb, "space", "mars", "Mars is the red planet in our solar system.").await;
1056
1057        let req = RetrievalRequest::new("which pets purr", vec!["pets".into(), "space".into()]).top_k(2);
1058        let hits = kb.retrieve(&req).await.unwrap();
1059        assert_eq!(hits.len(), 2);
1060        assert_eq!(hits[0].document_id, "cats");
1061
1062        let single = kb.retrieve_single(&req).await.unwrap();
1063        assert!(single.iter().all(|h| h.dataset_id == "pets"));
1064
1065        let unknown = RetrievalRequest::new("x", vec!["nope".into()]);
1066        assert!(matches!(kb.retrieve(&unknown).await, Err(RagError::NotFound(_))));
1067    }
1068
1069    #[test]
1070    fn config_rejects_inline_secrets_and_resolves_env() {
1071        let cfg = KnowledgeConfig::from_json(
1072            r#"{"embedders":{"e":{"type":"hash","dimensions":16}},
1073                "stores":{"m":{"type":"memory"}},
1074                "datasets":[{"id":"a","store":"m"}]}"#,
1075        )
1076        .unwrap();
1077        let kb = cfg.build_with_env(&|_| None).unwrap();
1078        assert!(kb.contains("a"));
1079
1080        let bad = KnowledgeConfig::from_json(
1081            r#"{"rerankers":{"r":{"type":"cohere","base_url":"http://x","model":"m","api_key":"inline"}}}"#,
1082        )
1083        .unwrap();
1084        let err = bad.build_with_env(&|_| None).unwrap_err();
1085        assert!(err.to_string().contains("inline") || err.to_string().contains("unsupported"), "{err}");
1086
1087        let missing = KnowledgeConfig::from_json(r#"{"datasets":[{"id":"a","store":"nope"}]}"#).unwrap();
1088        assert!(missing.build_with_env(&|_| None).is_err());
1089    }
1090
1091    #[cfg(feature = "http")]
1092    #[test]
1093    fn config_env_secret_lookup() {
1094        let cfg = KnowledgeConfig::from_json(
1095            r#"{"embedders":{"e":{"type":"openai","base_url":"http://127.0.0.1:9","model":"m","api_key_env":"MY_KEY"}}}"#,
1096        )
1097        .unwrap();
1098        assert!(cfg.build_with_env(&|_| None).unwrap_err().to_string().contains("MY_KEY"));
1099        assert!(cfg.build_with_env(&|n| (n == "MY_KEY").then(|| "secret".to_owned())).is_ok());
1100    }
1101
1102    #[cfg(feature = "yaml")]
1103    #[test]
1104    fn yaml_config() {
1105        let cfg = KnowledgeConfig::from_yaml(
1106            "embedders:\n  e: { type: hash }\nstores:\n  m: { type: memory }\ndatasets:\n  - { id: kb, store: m, embedder: e }\n",
1107        )
1108        .unwrap();
1109        assert_eq!(cfg.datasets.len(), 1);
1110        assert!(cfg.build_with_env(&|_| None).is_ok());
1111    }
1112
1113    #[cfg(feature = "sqlite")]
1114    #[tokio::test]
1115    async fn sqlite_store_from_config_persists_between_builds() {
1116        let dir = std::env::temp_dir().join(format!("graphrt-rag-kb-sqlite-{}", std::process::id()));
1117        let _ = std::fs::remove_dir_all(&dir);
1118        std::fs::create_dir_all(&dir).unwrap();
1119        let path = dir.join("kb.db");
1120        let config = KnowledgeConfig::from_json(
1121            &serde_json::json!({
1122                "embedders": { "e": { "type": "hash", "dimensions": 64 } },
1123                "stores": { "local": { "type": "sqlite", "path": path } },
1124                "datasets": [{ "id": "faq", "store": "local", "embedder": "e" }]
1125            })
1126            .to_string(),
1127        )
1128        .unwrap();
1129
1130        let first = config.build_with_env(&|_| None).unwrap();
1131        ingest(&first, "faq", "returns", "Items can be returned within 30 days.").await;
1132        drop(first);
1133
1134        let second = config.build_with_env(&|_| None).unwrap();
1135        let hits = second.retrieve(&RetrievalRequest::new("returned within days", vec!["faq".into()])).await.unwrap();
1136        assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("returns"));
1137        let _ = std::fs::remove_dir_all(&dir);
1138    }
1139
1140    #[cfg(feature = "local-embed")]
1141    #[test]
1142    fn local_embedder_from_config_is_lazy_and_strict() {
1143        let build = |embedder: serde_json::Value| {
1144            KnowledgeConfig::from_json(&serde_json::json!({ "embedders": { "e": embedder } }).to_string())
1145                .unwrap()
1146                .build_with_env(&|_| None)
1147        };
1148        // Nothing is downloaded or loaded while building the knowledge base.
1149        assert!(build(serde_json::json!({ "type": "local" })).is_ok());
1150        assert!(build(serde_json::json!({ "type": "local", "path": "/not/loaded/yet" })).is_ok());
1151        let typo = build(serde_json::json!({ "type": "local", "modle": "x/y" })).unwrap_err();
1152        assert!(typo.to_string().contains("modle"), "{typo}");
1153        let traversal = build(serde_json::json!({ "type": "local", "model": "../../etc" })).unwrap_err();
1154        assert!(traversal.to_string().contains("`model`"), "{traversal}");
1155    }
1156
1157    #[cfg(feature = "sqlite")]
1158    #[test]
1159    fn sqlite_store_requires_a_path() {
1160        let config = KnowledgeConfig::from_json(r#"{"stores":{"local":{"type":"sqlite"}}}"#).unwrap();
1161        let error = config.build_with_env(&|_| None).unwrap_err();
1162        assert!(error.to_string().contains("`path` (the database file) is required"), "{error}");
1163    }
1164}