Skip to main content

graphrt_rag/
kb.rs

1//! [`KnowledgeBase`]: the service the `knowledge-retrieval` node uses.
2//!
3//! A knowledge base maps dataset ids to retrievers. A dataset is either backed
4//! by the built-in pipeline (a [`VectorStore`] + [`Embedder`], optionally a
5//! [`Reranker`]) or by any external [`Retriever`]. Register it in
6//! `graphrt_core::Services` as a plain `KnowledgeBase` value:
7//!
8//! ```
9//! # use std::sync::Arc;
10//! # use graphrt_core::Services;
11//! # use graphrt_rag::{KnowledgeBase, MemoryStore, HashEmbedder};
12//! let kb = KnowledgeBase::new()
13//!     .with_store_dataset("docs", Arc::new(MemoryStore::new()), Arc::new(HashEmbedder::new(64)), None);
14//! let services = Services::new().with(kb);
15//! ```
16
17use std::sync::Arc;
18
19use indexmap::IndexMap;
20use serde::{Deserialize, Serialize};
21
22use crate::error::RagError;
23use crate::ingest::Ingestor;
24use crate::pipeline::{StoreRetriever, finalize, rerank};
25use crate::types::{Embedder, Hit, Reranker, RetrievalRequest, Retriever, VectorStore};
26
27/// Public description of a dataset.
28#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
29pub struct DatasetInfo {
30    pub id: String,
31    #[serde(default)]
32    pub name: String,
33    #[serde(default)]
34    pub description: String,
35}
36
37#[derive(Clone, Debug)]
38struct DatasetEntry {
39    info: DatasetInfo,
40    retriever: Arc<dyn Retriever>,
41    store: Option<(Arc<dyn VectorStore>, Arc<dyn Embedder>)>,
42}
43
44/// Dataset id -> retriever registry.
45#[derive(Clone, Debug, Default)]
46pub struct KnowledgeBase {
47    datasets: IndexMap<String, DatasetEntry>,
48    /// Reranker applied across datasets in multi-dataset retrieval.
49    reranker: Option<Arc<dyn Reranker>>,
50}
51
52impl KnowledgeBase {
53    pub fn new() -> Self {
54        Self::default()
55    }
56
57    /// Add a dataset backed by the built-in store pipeline.
58    pub fn with_store_dataset(
59        mut self,
60        id: impl Into<String>,
61        store: Arc<dyn VectorStore>,
62        embedder: Arc<dyn Embedder>,
63        reranker: Option<Arc<dyn Reranker>>,
64    ) -> Self {
65        let id = id.into();
66        self.add_store_dataset(DatasetInfo { id, ..DatasetInfo::default() }, store, embedder, reranker);
67        self
68    }
69
70    /// Add (or replace) a store-backed dataset with descriptive info.
71    pub fn add_store_dataset(
72        &mut self,
73        info: DatasetInfo,
74        store: Arc<dyn VectorStore>,
75        embedder: Arc<dyn Embedder>,
76        reranker: Option<Arc<dyn Reranker>>,
77    ) -> &mut Self {
78        let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
79        self.datasets.insert(
80            info.id.clone(),
81            DatasetEntry { info, retriever: Arc::new(retriever), store: Some((store, embedder)) },
82        );
83        self
84    }
85
86    /// Add a dataset served by an external retriever.
87    pub fn with_retriever(mut self, id: impl Into<String>, retriever: Arc<dyn Retriever>) -> Self {
88        let id = id.into();
89        self.add_retriever(DatasetInfo { id, ..DatasetInfo::default() }, retriever);
90        self
91    }
92
93    /// Add (or replace) a dataset served by an external retriever.
94    pub fn add_retriever(&mut self, info: DatasetInfo, retriever: Arc<dyn Retriever>) -> &mut Self {
95        self.datasets.insert(info.id.clone(), DatasetEntry { info, retriever, store: None });
96        self
97    }
98
99    /// Reranker applied to the merged results of multi-dataset retrieval.
100    pub fn with_reranker(mut self, reranker: Option<Arc<dyn Reranker>>) -> Self {
101        self.reranker = reranker;
102        self
103    }
104
105    pub fn datasets(&self) -> impl Iterator<Item = &DatasetInfo> {
106        self.datasets.values().map(|d| &d.info)
107    }
108
109    pub fn contains(&self, dataset_id: &str) -> bool {
110        self.datasets.contains_key(dataset_id)
111    }
112
113    /// An ingestor for a store-backed dataset (`None` for external retrievers or unknown ids).
114    pub fn ingestor(&self, dataset_id: &str) -> Option<Ingestor> {
115        let entry = self.datasets.get(dataset_id)?;
116        let (store, embedder) = entry.store.as_ref()?;
117        Some(Ingestor::new(store.clone(), embedder.clone()))
118    }
119
120    fn entry(&self, id: &str) -> Result<&DatasetEntry, RagError> {
121        self.datasets.get(id).ok_or_else(|| RagError::NotFound(format!("unknown dataset `{id}`")))
122    }
123
124    /// Query every dataset concurrently. Returns `(dataset id, hits)` in request order.
125    async fn query_each(
126        &self,
127        request: &RetrievalRequest,
128        rerank_each: bool,
129    ) -> Result<Vec<(String, Vec<Hit>)>, RagError> {
130        let mut jobs = Vec::with_capacity(request.datasets.len());
131        for id in &request.datasets {
132            let entry = self.entry(id)?;
133            let sub = RetrievalRequest {
134                datasets: vec![id.clone()],
135                // Threshold and top-k are applied after merging.
136                score_threshold: None,
137                rerank: rerank_each,
138                ..request.clone()
139            };
140            jobs.push(async move { entry.retriever.retrieve(&sub).await.map(|hits| (id.clone(), hits)) });
141        }
142        futures::future::try_join_all(jobs).await
143    }
144
145    /// Retrieve across all requested datasets and merge by score.
146    ///
147    /// When `request.rerank` is set and the knowledge base has a cross-dataset
148    /// reranker, the merged list is reranked as a whole (making scores from
149    /// different datasets comparable). Otherwise each dataset's own reranker, if
150    /// any, is applied before merging.
151    pub async fn retrieve(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
152        validate_request(request)?;
153        let global_rerank = request.rerank && self.reranker.is_some() && request.datasets.len() > 1;
154        let per_dataset = self.query_each(request, request.rerank && !global_rerank).await?;
155        let mut merged: Vec<Hit> = per_dataset.into_iter().flat_map(|(_, hits)| hits).collect();
156        if global_rerank {
157            if let Some(reranker) = &self.reranker {
158                merged = rerank(reranker.as_ref(), &request.query, merged).await?;
159            }
160        }
161        Ok(finalize(merged, request.score_threshold, request.top_k))
162    }
163
164    /// Retrieve from the single best-matching dataset: every dataset is queried,
165    /// and only the hits of the dataset whose top hit scores highest are kept.
166    pub async fn retrieve_single(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
167        validate_request(request)?;
168        let per_dataset = self.query_each(request, request.rerank).await?;
169        let best = per_dataset
170            .into_iter()
171            .filter(|(_, hits)| !hits.is_empty())
172            .max_by(|a, b| top_score(&a.1).total_cmp(&top_score(&b.1)));
173        Ok(best.map(|(_, hits)| finalize(hits, request.score_threshold, request.top_k)).unwrap_or_default())
174    }
175}
176
177/// Reject requests that would otherwise silently match nothing.
178fn validate_request(request: &RetrievalRequest) -> Result<(), RagError> {
179    if request.datasets.is_empty() {
180        return Err(RagError::Input("no dataset ids given".into()));
181    }
182    if let Some(filter) = &request.filter {
183        filter.validate().map_err(|e| RagError::Input(e.to_string()))?;
184    }
185    Ok(())
186}
187
188fn top_score(hits: &[Hit]) -> f32 {
189    hits.iter().map(|h| h.score).fold(f32::MIN, f32::max)
190}
191
192// ---------------------------------------------------------------------------
193// Configuration
194// ---------------------------------------------------------------------------
195
196/// Declarative knowledge-base configuration (`knowledge.yaml` / `.json`).
197///
198/// ```yaml
199/// embedders:
200///   default: { type: openai, base_url: "http://localhost:11434/v1", model: nomic-embed-text }
201///   cloud:   { type: openai, model: text-embedding-3-small, api_key_env: OPENAI_API_KEY }
202/// rerankers:
203///   cohere: { type: http, base_url: "https://api.cohere.com/v2", model: rerank-v3.5, api_key_env: COHERE_API_KEY }
204/// stores:
205///   local:  { type: sqlite, path: ./kb.db }
206///   json:   { type: memory, path: ./kb.json }
207///   vector: { type: qdrant, url: "http://localhost:6333", collection: kb, api_key_env: QDRANT_API_KEY }
208/// datasets:
209///   - { id: handbook, name: Handbook, store: local, embedder: default, reranker: cohere }
210///   - { id: crm, retriever: { type: http, url: "https://rag.internal/search", api_key_env: RAG_TOKEN } }
211/// reranker: cohere   # optional, used across datasets
212/// ```
213///
214/// Secrets are never written inline: every provider takes `api_key_env`, the
215/// name of an environment variable resolved when the configuration is built.
216#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
217#[serde(deny_unknown_fields)]
218pub struct KnowledgeConfig {
219    #[serde(default)]
220    pub embedders: IndexMap<String, serde_json::Value>,
221    #[serde(default)]
222    pub rerankers: IndexMap<String, serde_json::Value>,
223    #[serde(default)]
224    pub stores: IndexMap<String, serde_json::Value>,
225    #[serde(default)]
226    pub datasets: Vec<DatasetConfig>,
227    /// Cross-dataset reranker (a key of `rerankers`).
228    #[serde(default)]
229    pub reranker: Option<String>,
230}
231
232/// One dataset entry.
233#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
234#[serde(deny_unknown_fields)]
235pub struct DatasetConfig {
236    pub id: String,
237    #[serde(default)]
238    pub name: String,
239    #[serde(default)]
240    pub description: String,
241    /// Key of `stores` (store-backed datasets).
242    #[serde(default)]
243    pub store: Option<String>,
244    /// Key of `embedders` (store-backed datasets).
245    #[serde(default)]
246    pub embedder: Option<String>,
247    /// Key of `rerankers`.
248    #[serde(default)]
249    pub reranker: Option<String>,
250    /// External retriever definition (instead of store + embedder).
251    #[serde(default)]
252    pub retriever: Option<serde_json::Value>,
253}
254
255/// Resolves environment variable names to values (`std::env::var` in production).
256pub type EnvLookup<'a> = &'a dyn Fn(&str) -> Option<String>;
257
258impl KnowledgeConfig {
259    pub fn from_json(text: &str) -> Result<Self, RagError> {
260        serde_json::from_str(text).map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
261    }
262
263    #[cfg(feature = "yaml")]
264    pub fn from_yaml(text: &str) -> Result<Self, RagError> {
265        let options = serde_saphyr::options! { strict_booleans: true };
266        serde_saphyr::from_str_with_options(text, options)
267            .map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
268    }
269
270    /// Build the knowledge base, resolving secrets with `std::env::var`.
271    pub fn build(&self) -> Result<KnowledgeBase, RagError> {
272        self.build_with_env(&|name| std::env::var(name).ok())
273    }
274
275    /// Build the knowledge base with a custom environment lookup.
276    pub fn build_with_env(&self, env: EnvLookup<'_>) -> Result<KnowledgeBase, RagError> {
277        let mut embedders: IndexMap<&str, Arc<dyn Embedder>> = IndexMap::new();
278        for (name, raw) in &self.embedders {
279            embedders.insert(name, build_embedder(name, raw, env)?);
280        }
281        let mut rerankers: IndexMap<&str, Arc<dyn Reranker>> = IndexMap::new();
282        for (name, raw) in &self.rerankers {
283            rerankers.insert(name, build_reranker(name, raw, env)?);
284        }
285        let mut stores: IndexMap<&str, Arc<dyn VectorStore>> = IndexMap::new();
286        for (name, raw) in &self.stores {
287            stores.insert(name, build_store(name, raw, env)?);
288        }
289
290        let lookup_reranker = |key: &Option<String>| -> Result<Option<Arc<dyn Reranker>>, RagError> {
291            match key {
292                None => Ok(None),
293                Some(k) => rerankers
294                    .get(k.as_str())
295                    .cloned()
296                    .map(Some)
297                    .ok_or_else(|| RagError::Config(format!("unknown reranker `{k}`"))),
298            }
299        };
300
301        let mut kb = KnowledgeBase::new().with_reranker(lookup_reranker(&self.reranker)?);
302        for ds in &self.datasets {
303            if ds.id.trim().is_empty() {
304                return Err(RagError::Config("dataset id must not be empty".into()));
305            }
306            if kb.contains(&ds.id) {
307                return Err(RagError::Config(format!("duplicate dataset `{}`", ds.id)));
308            }
309            let info = DatasetInfo { id: ds.id.clone(), name: ds.name.clone(), description: ds.description.clone() };
310            if let Some(raw) = &ds.retriever {
311                kb.add_retriever(info, build_retriever(&ds.id, raw, env)?);
312                continue;
313            }
314            let store_key = ds.store.as_deref().ok_or_else(|| {
315                RagError::Config(format!("dataset `{}` needs either `store` + `embedder` or `retriever`", ds.id))
316            })?;
317            let store = stores
318                .get(store_key)
319                .cloned()
320                .ok_or_else(|| RagError::Config(format!("unknown store `{store_key}`")))?;
321            let embedder_key = match ds.embedder.as_deref() {
322                Some(k) => k,
323                None if embedders.len() == 1 => embedders.keys().next().copied().unwrap_or_default(),
324                None => return Err(RagError::Config(format!("dataset `{}` needs `embedder`", ds.id))),
325            };
326            let embedder = embedders
327                .get(embedder_key)
328                .cloned()
329                .ok_or_else(|| RagError::Config(format!("unknown embedder `{embedder_key}`")))?;
330            kb.add_store_dataset(info, store, embedder, lookup_reranker(&ds.reranker)?);
331        }
332        Ok(kb)
333    }
334}
335
336/// Keys accepted by every HTTP-backed provider entry.
337#[cfg_attr(not(feature = "http"), allow(dead_code))]
338const HTTP_KEYS: &[&str] = &["type", "api_key", "api_key_env", "api_key_header", "headers", "timeout_ms"];
339
340/// Reject keys a provider entry does not understand (typos would otherwise be
341/// ignored silently, e.g. `file:` instead of `path:` turning a persistent store
342/// into an in-memory one).
343fn check_keys(name: &str, raw: &serde_json::Value, allowed: &[&str], with_http: bool) -> Result<(), RagError> {
344    let obj = raw.as_object().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
345    for key in obj.keys() {
346        let known = allowed.contains(&key.as_str()) || (with_http && HTTP_KEYS.contains(&key.as_str()));
347        if !known && key != "type" {
348            return Err(RagError::Config(format!("`{name}`: unknown key `{key}`")));
349        }
350    }
351    Ok(())
352}
353
354fn provider_type(name: &str, raw: &serde_json::Value) -> Result<String, RagError> {
355    raw.get("type")
356        .and_then(|t| t.as_str())
357        .map(str::to_owned)
358        .ok_or_else(|| RagError::Config(format!("`{name}` needs a `type`")))
359}
360
361/// Strip `type` / `api_key_env`, reject inline secrets, and inject the resolved key.
362#[cfg_attr(not(feature = "http"), allow(dead_code))]
363fn with_secret(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<serde_json::Value, RagError> {
364    let mut obj = raw.as_object().cloned().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
365    obj.remove("type");
366    if obj.contains_key("api_key") {
367        return Err(RagError::Config(format!(
368            "`{name}`: inline `api_key` is not allowed; use `api_key_env` with an environment variable name"
369        )));
370    }
371    if let Some(var) = obj.remove("api_key_env") {
372        let var = var.as_str().ok_or_else(|| RagError::Config(format!("`{name}`: `api_key_env` must be a string")))?;
373        let value =
374            env(var).ok_or_else(|| RagError::Config(format!("`{name}`: environment variable `{var}` is not set")))?;
375        obj.insert("api_key".into(), serde_json::Value::String(value));
376    }
377    Ok(serde_json::Value::Object(obj))
378}
379
380#[cfg_attr(not(feature = "http"), allow(dead_code))]
381fn parse<T: serde::de::DeserializeOwned>(name: &str, value: serde_json::Value) -> Result<T, RagError> {
382    serde_json::from_value(value).map_err(|e| RagError::Config(format!("`{name}`: {e}")))
383}
384
385fn build_embedder(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Embedder>, RagError> {
386    let kind = provider_type(name, raw)?;
387    match kind.as_str() {
388        #[cfg(feature = "http")]
389        "openai" | "openai-compatible" | "ollama" => {
390            check_keys(
391                name,
392                raw,
393                &["base_url", "model", "dimensions", "batch_size", "query_prefix", "document_prefix"],
394                true,
395            )?;
396            let mut value = with_secret(name, raw, env)?;
397            if kind == "ollama" {
398                if let Some(obj) = value.as_object_mut() {
399                    obj.entry("base_url").or_insert_with(|| "http://localhost:11434/v1".into());
400                }
401            }
402            let cfg: crate::http::OpenAiEmbedderConfig = parse(name, value)?;
403            Ok(Arc::new(crate::http::OpenAiEmbedder::new(cfg)?))
404        }
405        "hash" => {
406            check_keys(name, raw, &["dimensions"], false)?;
407            let dims = match raw.get("dimensions") {
408                None => 256,
409                Some(v) => v
410                    .as_u64()
411                    .and_then(|d| usize::try_from(d).ok())
412                    .filter(|d| *d > 0)
413                    .ok_or_else(|| RagError::Config(format!("`{name}`: `dimensions` must be a positive integer")))?,
414            };
415            let _ = env;
416            Ok(Arc::new(crate::hash::HashEmbedder::new(dims)))
417        }
418        other => Err(RagError::Config(format!("embedder `{name}`: unsupported type `{other}` in this build"))),
419    }
420}
421
422fn build_reranker(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Reranker>, RagError> {
423    let kind = provider_type(name, raw)?;
424    match kind.as_str() {
425        #[cfg(feature = "http")]
426        "http" | "cohere" | "jina" | "voyage" => {
427            check_keys(name, raw, &["base_url", "model"], true)?;
428            let cfg: crate::http::HttpRerankerConfig = parse(name, with_secret(name, raw, env)?)?;
429            Ok(Arc::new(crate::http::HttpReranker::new(cfg)?))
430        }
431        other => {
432            let _ = env;
433            Err(RagError::Config(format!("reranker `{name}`: unsupported type `{other}` in this build")))
434        }
435    }
436}
437
438fn build_store(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn VectorStore>, RagError> {
439    let kind = provider_type(name, raw)?;
440    match kind.as_str() {
441        "memory" => {
442            check_keys(name, raw, &["path"], false)?;
443            let path = match raw.get("path") {
444                None => None,
445                Some(serde_json::Value::String(p)) => Some(p.as_str()),
446                Some(_) => return Err(RagError::Config(format!("store `{name}`: `path` must be a string"))),
447            };
448            match path {
449                #[cfg(feature = "fs")]
450                Some(path) => Ok(Arc::new(crate::memory::MemoryStore::open(path)?)),
451                #[cfg(not(feature = "fs"))]
452                Some(_) => Err(RagError::Config(format!("store `{name}`: `path` requires the `fs` feature"))),
453                None => Ok(Arc::new(crate::memory::MemoryStore::new())),
454            }
455        }
456        #[cfg(feature = "sqlite")]
457        "sqlite" => {
458            check_keys(name, raw, &["path"], false)?;
459            let path =
460                raw.get("path").and_then(serde_json::Value::as_str).filter(|p| !p.trim().is_empty()).ok_or_else(
461                    || RagError::Config(format!("store `{name}`: `path` (the database file) is required")),
462                )?;
463            Ok(Arc::new(crate::sqlite::SqliteStore::open(path)?))
464        }
465        #[cfg(feature = "qdrant")]
466        "qdrant" => {
467            check_keys(name, raw, &["url", "collection", "create_collection"], true)?;
468            let mut value = with_secret(name, raw, env)?;
469            if let Some(obj) = value.as_object_mut() {
470                obj.entry("api_key_header").or_insert_with(|| "api-key".into());
471            }
472            let cfg: crate::qdrant::QdrantConfig = parse(name, value)?;
473            Ok(Arc::new(crate::qdrant::QdrantStore::new(cfg)?))
474        }
475        other => {
476            let _ = env;
477            Err(RagError::Config(format!("store `{name}`: unsupported type `{other}` in this build")))
478        }
479    }
480}
481
482fn build_retriever(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Retriever>, RagError> {
483    let kind = provider_type(name, raw)?;
484    match kind.as_str() {
485        #[cfg(feature = "http")]
486        "http" => {
487            check_keys(name, raw, &["url"], true)?;
488            let cfg: crate::http::HttpRetrieverConfig = parse(name, with_secret(name, raw, env)?)?;
489            Ok(Arc::new(crate::http::HttpRetriever::new(cfg)?))
490        }
491        other => {
492            let _ = env;
493            Err(RagError::Config(format!("retriever for `{name}`: unsupported type `{other}` in this build")))
494        }
495    }
496}
497
498#[cfg(test)]
499mod tests {
500    use super::*;
501    use crate::hash::HashEmbedder;
502    use crate::ingest::Document;
503    use crate::memory::MemoryStore;
504
505    fn kb_with_two_datasets() -> KnowledgeBase {
506        let embedder: Arc<dyn Embedder> = Arc::new(HashEmbedder::new(128));
507        KnowledgeBase::new()
508            .with_store_dataset("pets", Arc::new(MemoryStore::new()), embedder.clone(), None)
509            .with_store_dataset("space", Arc::new(MemoryStore::new()), embedder, None)
510    }
511
512    async fn ingest(kb: &KnowledgeBase, ds: &str, id: &str, text: &str) {
513        kb.ingestor(ds).unwrap().ingest(&Document::new(ds, id, text)).await.unwrap();
514    }
515
516    #[tokio::test]
517    async fn multi_dataset_merge_and_single_mode() {
518        let kb = kb_with_two_datasets();
519        ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
520        ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
521        ingest(&kb, "space", "mars", "Mars is the red planet in our solar system.").await;
522
523        let req = RetrievalRequest::new("which pets purr", vec!["pets".into(), "space".into()]).top_k(2);
524        let hits = kb.retrieve(&req).await.unwrap();
525        assert_eq!(hits.len(), 2);
526        assert_eq!(hits[0].document_id, "cats");
527
528        let single = kb.retrieve_single(&req).await.unwrap();
529        assert!(single.iter().all(|h| h.dataset_id == "pets"));
530
531        let unknown = RetrievalRequest::new("x", vec!["nope".into()]);
532        assert!(matches!(kb.retrieve(&unknown).await, Err(RagError::NotFound(_))));
533    }
534
535    #[test]
536    fn config_rejects_inline_secrets_and_resolves_env() {
537        let cfg = KnowledgeConfig::from_json(
538            r#"{"embedders":{"e":{"type":"hash","dimensions":16}},
539                "stores":{"m":{"type":"memory"}},
540                "datasets":[{"id":"a","store":"m"}]}"#,
541        )
542        .unwrap();
543        let kb = cfg.build_with_env(&|_| None).unwrap();
544        assert!(kb.contains("a"));
545
546        let bad = KnowledgeConfig::from_json(
547            r#"{"rerankers":{"r":{"type":"cohere","base_url":"http://x","model":"m","api_key":"inline"}}}"#,
548        )
549        .unwrap();
550        let err = bad.build_with_env(&|_| None).unwrap_err();
551        assert!(err.to_string().contains("inline") || err.to_string().contains("unsupported"), "{err}");
552
553        let missing = KnowledgeConfig::from_json(r#"{"datasets":[{"id":"a","store":"nope"}]}"#).unwrap();
554        assert!(missing.build_with_env(&|_| None).is_err());
555    }
556
557    #[cfg(feature = "http")]
558    #[test]
559    fn config_env_secret_lookup() {
560        let cfg = KnowledgeConfig::from_json(
561            r#"{"embedders":{"e":{"type":"openai","base_url":"http://127.0.0.1:9","model":"m","api_key_env":"MY_KEY"}}}"#,
562        )
563        .unwrap();
564        assert!(cfg.build_with_env(&|_| None).unwrap_err().to_string().contains("MY_KEY"));
565        assert!(cfg.build_with_env(&|n| (n == "MY_KEY").then(|| "secret".to_owned())).is_ok());
566    }
567
568    #[cfg(feature = "yaml")]
569    #[test]
570    fn yaml_config() {
571        let cfg = KnowledgeConfig::from_yaml(
572            "embedders:\n  e: { type: hash }\nstores:\n  m: { type: memory }\ndatasets:\n  - { id: kb, store: m, embedder: e }\n",
573        )
574        .unwrap();
575        assert_eq!(cfg.datasets.len(), 1);
576        assert!(cfg.build_with_env(&|_| None).is_ok());
577    }
578
579    #[cfg(feature = "sqlite")]
580    #[tokio::test]
581    async fn sqlite_store_from_config_persists_between_builds() {
582        let dir = std::env::temp_dir().join(format!("graphrt-rag-kb-sqlite-{}", std::process::id()));
583        let _ = std::fs::remove_dir_all(&dir);
584        std::fs::create_dir_all(&dir).unwrap();
585        let path = dir.join("kb.db");
586        let config = KnowledgeConfig::from_json(
587            &serde_json::json!({
588                "embedders": { "e": { "type": "hash", "dimensions": 64 } },
589                "stores": { "local": { "type": "sqlite", "path": path } },
590                "datasets": [{ "id": "faq", "store": "local", "embedder": "e" }]
591            })
592            .to_string(),
593        )
594        .unwrap();
595
596        let first = config.build_with_env(&|_| None).unwrap();
597        ingest(&first, "faq", "returns", "Items can be returned within 30 days.").await;
598        drop(first);
599
600        let second = config.build_with_env(&|_| None).unwrap();
601        let hits = second.retrieve(&RetrievalRequest::new("returned within days", vec!["faq".into()])).await.unwrap();
602        assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("returns"));
603        let _ = std::fs::remove_dir_all(&dir);
604    }
605
606    #[cfg(feature = "sqlite")]
607    #[test]
608    fn sqlite_store_requires_a_path() {
609        let config = KnowledgeConfig::from_json(r#"{"stores":{"local":{"type":"sqlite"}}}"#).unwrap();
610        let error = config.build_with_env(&|_| None).unwrap_err();
611        assert!(error.to_string().contains("`path` (the database file) is required"), "{error}");
612    }
613}