1use std::sync::Arc;
18
19use indexmap::IndexMap;
20use serde::{Deserialize, Serialize};
21
22use crate::error::RagError;
23use crate::ingest::Ingestor;
24use crate::pipeline::{StoreRetriever, finalize, rerank};
25use crate::types::{Embedder, Hit, Reranker, RetrievalRequest, Retriever, VectorStore};
26
27#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
29pub struct DatasetInfo {
30 pub id: String,
31 #[serde(default)]
32 pub name: String,
33 #[serde(default)]
34 pub description: String,
35}
36
37#[derive(Clone, Debug)]
38struct DatasetEntry {
39 info: DatasetInfo,
40 retriever: Arc<dyn Retriever>,
41 store: Option<(Arc<dyn VectorStore>, Arc<dyn Embedder>)>,
42}
43
44#[derive(Clone, Debug, Default)]
46pub struct KnowledgeBase {
47 datasets: IndexMap<String, DatasetEntry>,
48 reranker: Option<Arc<dyn Reranker>>,
50}
51
52impl KnowledgeBase {
53 pub fn new() -> Self {
54 Self::default()
55 }
56
57 pub fn with_store_dataset(
59 mut self,
60 id: impl Into<String>,
61 store: Arc<dyn VectorStore>,
62 embedder: Arc<dyn Embedder>,
63 reranker: Option<Arc<dyn Reranker>>,
64 ) -> Self {
65 let id = id.into();
66 self.add_store_dataset(DatasetInfo { id, ..DatasetInfo::default() }, store, embedder, reranker);
67 self
68 }
69
70 pub fn add_store_dataset(
72 &mut self,
73 info: DatasetInfo,
74 store: Arc<dyn VectorStore>,
75 embedder: Arc<dyn Embedder>,
76 reranker: Option<Arc<dyn Reranker>>,
77 ) -> &mut Self {
78 let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
79 self.datasets.insert(
80 info.id.clone(),
81 DatasetEntry { info, retriever: Arc::new(retriever), store: Some((store, embedder)) },
82 );
83 self
84 }
85
86 pub fn with_retriever(mut self, id: impl Into<String>, retriever: Arc<dyn Retriever>) -> Self {
88 let id = id.into();
89 self.add_retriever(DatasetInfo { id, ..DatasetInfo::default() }, retriever);
90 self
91 }
92
93 pub fn add_retriever(&mut self, info: DatasetInfo, retriever: Arc<dyn Retriever>) -> &mut Self {
95 self.datasets.insert(info.id.clone(), DatasetEntry { info, retriever, store: None });
96 self
97 }
98
99 pub fn with_reranker(mut self, reranker: Option<Arc<dyn Reranker>>) -> Self {
101 self.reranker = reranker;
102 self
103 }
104
105 pub fn datasets(&self) -> impl Iterator<Item = &DatasetInfo> {
106 self.datasets.values().map(|d| &d.info)
107 }
108
109 pub fn contains(&self, dataset_id: &str) -> bool {
110 self.datasets.contains_key(dataset_id)
111 }
112
113 pub fn ingestor(&self, dataset_id: &str) -> Option<Ingestor> {
115 let entry = self.datasets.get(dataset_id)?;
116 let (store, embedder) = entry.store.as_ref()?;
117 Some(Ingestor::new(store.clone(), embedder.clone()))
118 }
119
120 fn entry(&self, id: &str) -> Result<&DatasetEntry, RagError> {
121 self.datasets.get(id).ok_or_else(|| RagError::NotFound(format!("unknown dataset `{id}`")))
122 }
123
124 async fn query_each(
126 &self,
127 request: &RetrievalRequest,
128 rerank_each: bool,
129 ) -> Result<Vec<(String, Vec<Hit>)>, RagError> {
130 let mut jobs = Vec::with_capacity(request.datasets.len());
131 for id in &request.datasets {
132 let entry = self.entry(id)?;
133 let sub = RetrievalRequest {
134 datasets: vec![id.clone()],
135 score_threshold: None,
137 rerank: rerank_each,
138 ..request.clone()
139 };
140 jobs.push(async move { entry.retriever.retrieve(&sub).await.map(|hits| (id.clone(), hits)) });
141 }
142 futures::future::try_join_all(jobs).await
143 }
144
145 pub async fn retrieve(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
152 validate_request(request)?;
153 let global_rerank = request.rerank && self.reranker.is_some() && request.datasets.len() > 1;
154 let per_dataset = self.query_each(request, request.rerank && !global_rerank).await?;
155 let mut merged: Vec<Hit> = per_dataset.into_iter().flat_map(|(_, hits)| hits).collect();
156 if global_rerank {
157 if let Some(reranker) = &self.reranker {
158 merged = rerank(reranker.as_ref(), &request.query, merged).await?;
159 }
160 }
161 Ok(finalize(merged, request.score_threshold, request.top_k))
162 }
163
164 pub async fn retrieve_single(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
167 validate_request(request)?;
168 let per_dataset = self.query_each(request, request.rerank).await?;
169 let best = per_dataset
170 .into_iter()
171 .filter(|(_, hits)| !hits.is_empty())
172 .max_by(|a, b| top_score(&a.1).total_cmp(&top_score(&b.1)));
173 Ok(best.map(|(_, hits)| finalize(hits, request.score_threshold, request.top_k)).unwrap_or_default())
174 }
175}
176
177fn validate_request(request: &RetrievalRequest) -> Result<(), RagError> {
179 if request.datasets.is_empty() {
180 return Err(RagError::Input("no dataset ids given".into()));
181 }
182 if let Some(filter) = &request.filter {
183 filter.validate().map_err(|e| RagError::Input(e.to_string()))?;
184 }
185 Ok(())
186}
187
188fn top_score(hits: &[Hit]) -> f32 {
189 hits.iter().map(|h| h.score).fold(f32::MIN, f32::max)
190}
191
192#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
217#[serde(deny_unknown_fields)]
218pub struct KnowledgeConfig {
219 #[serde(default)]
220 pub embedders: IndexMap<String, serde_json::Value>,
221 #[serde(default)]
222 pub rerankers: IndexMap<String, serde_json::Value>,
223 #[serde(default)]
224 pub stores: IndexMap<String, serde_json::Value>,
225 #[serde(default)]
226 pub datasets: Vec<DatasetConfig>,
227 #[serde(default)]
229 pub reranker: Option<String>,
230}
231
232#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
234#[serde(deny_unknown_fields)]
235pub struct DatasetConfig {
236 pub id: String,
237 #[serde(default)]
238 pub name: String,
239 #[serde(default)]
240 pub description: String,
241 #[serde(default)]
243 pub store: Option<String>,
244 #[serde(default)]
246 pub embedder: Option<String>,
247 #[serde(default)]
249 pub reranker: Option<String>,
250 #[serde(default)]
252 pub retriever: Option<serde_json::Value>,
253}
254
255pub type EnvLookup<'a> = &'a dyn Fn(&str) -> Option<String>;
257
258impl KnowledgeConfig {
259 pub fn from_json(text: &str) -> Result<Self, RagError> {
260 serde_json::from_str(text).map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
261 }
262
263 #[cfg(feature = "yaml")]
264 pub fn from_yaml(text: &str) -> Result<Self, RagError> {
265 let options = serde_saphyr::options! { strict_booleans: true };
266 serde_saphyr::from_str_with_options(text, options)
267 .map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
268 }
269
270 pub fn build(&self) -> Result<KnowledgeBase, RagError> {
272 self.build_with_env(&|name| std::env::var(name).ok())
273 }
274
275 pub fn build_with_env(&self, env: EnvLookup<'_>) -> Result<KnowledgeBase, RagError> {
277 let mut embedders: IndexMap<&str, Arc<dyn Embedder>> = IndexMap::new();
278 for (name, raw) in &self.embedders {
279 embedders.insert(name, build_embedder(name, raw, env)?);
280 }
281 let mut rerankers: IndexMap<&str, Arc<dyn Reranker>> = IndexMap::new();
282 for (name, raw) in &self.rerankers {
283 rerankers.insert(name, build_reranker(name, raw, env)?);
284 }
285 let mut stores: IndexMap<&str, Arc<dyn VectorStore>> = IndexMap::new();
286 for (name, raw) in &self.stores {
287 stores.insert(name, build_store(name, raw, env)?);
288 }
289
290 let lookup_reranker = |key: &Option<String>| -> Result<Option<Arc<dyn Reranker>>, RagError> {
291 match key {
292 None => Ok(None),
293 Some(k) => rerankers
294 .get(k.as_str())
295 .cloned()
296 .map(Some)
297 .ok_or_else(|| RagError::Config(format!("unknown reranker `{k}`"))),
298 }
299 };
300
301 let mut kb = KnowledgeBase::new().with_reranker(lookup_reranker(&self.reranker)?);
302 for ds in &self.datasets {
303 if ds.id.trim().is_empty() {
304 return Err(RagError::Config("dataset id must not be empty".into()));
305 }
306 if kb.contains(&ds.id) {
307 return Err(RagError::Config(format!("duplicate dataset `{}`", ds.id)));
308 }
309 let info = DatasetInfo { id: ds.id.clone(), name: ds.name.clone(), description: ds.description.clone() };
310 if let Some(raw) = &ds.retriever {
311 kb.add_retriever(info, build_retriever(&ds.id, raw, env)?);
312 continue;
313 }
314 let store_key = ds.store.as_deref().ok_or_else(|| {
315 RagError::Config(format!("dataset `{}` needs either `store` + `embedder` or `retriever`", ds.id))
316 })?;
317 let store = stores
318 .get(store_key)
319 .cloned()
320 .ok_or_else(|| RagError::Config(format!("unknown store `{store_key}`")))?;
321 let embedder_key = match ds.embedder.as_deref() {
322 Some(k) => k,
323 None if embedders.len() == 1 => embedders.keys().next().copied().unwrap_or_default(),
324 None => return Err(RagError::Config(format!("dataset `{}` needs `embedder`", ds.id))),
325 };
326 let embedder = embedders
327 .get(embedder_key)
328 .cloned()
329 .ok_or_else(|| RagError::Config(format!("unknown embedder `{embedder_key}`")))?;
330 kb.add_store_dataset(info, store, embedder, lookup_reranker(&ds.reranker)?);
331 }
332 Ok(kb)
333 }
334}
335
336#[cfg_attr(not(feature = "http"), allow(dead_code))]
338const HTTP_KEYS: &[&str] = &["type", "api_key", "api_key_env", "api_key_header", "headers", "timeout_ms"];
339
340fn check_keys(name: &str, raw: &serde_json::Value, allowed: &[&str], with_http: bool) -> Result<(), RagError> {
344 let obj = raw.as_object().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
345 for key in obj.keys() {
346 let known = allowed.contains(&key.as_str()) || (with_http && HTTP_KEYS.contains(&key.as_str()));
347 if !known && key != "type" {
348 return Err(RagError::Config(format!("`{name}`: unknown key `{key}`")));
349 }
350 }
351 Ok(())
352}
353
354fn provider_type(name: &str, raw: &serde_json::Value) -> Result<String, RagError> {
355 raw.get("type")
356 .and_then(|t| t.as_str())
357 .map(str::to_owned)
358 .ok_or_else(|| RagError::Config(format!("`{name}` needs a `type`")))
359}
360
361#[cfg_attr(not(feature = "http"), allow(dead_code))]
363fn with_secret(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<serde_json::Value, RagError> {
364 let mut obj = raw.as_object().cloned().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
365 obj.remove("type");
366 if obj.contains_key("api_key") {
367 return Err(RagError::Config(format!(
368 "`{name}`: inline `api_key` is not allowed; use `api_key_env` with an environment variable name"
369 )));
370 }
371 if let Some(var) = obj.remove("api_key_env") {
372 let var = var.as_str().ok_or_else(|| RagError::Config(format!("`{name}`: `api_key_env` must be a string")))?;
373 let value =
374 env(var).ok_or_else(|| RagError::Config(format!("`{name}`: environment variable `{var}` is not set")))?;
375 obj.insert("api_key".into(), serde_json::Value::String(value));
376 }
377 Ok(serde_json::Value::Object(obj))
378}
379
380#[cfg_attr(not(feature = "http"), allow(dead_code))]
381fn parse<T: serde::de::DeserializeOwned>(name: &str, value: serde_json::Value) -> Result<T, RagError> {
382 serde_json::from_value(value).map_err(|e| RagError::Config(format!("`{name}`: {e}")))
383}
384
385fn build_embedder(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Embedder>, RagError> {
386 let kind = provider_type(name, raw)?;
387 match kind.as_str() {
388 #[cfg(feature = "http")]
389 "openai" | "openai-compatible" | "ollama" => {
390 check_keys(
391 name,
392 raw,
393 &["base_url", "model", "dimensions", "batch_size", "query_prefix", "document_prefix"],
394 true,
395 )?;
396 let mut value = with_secret(name, raw, env)?;
397 if kind == "ollama" {
398 if let Some(obj) = value.as_object_mut() {
399 obj.entry("base_url").or_insert_with(|| "http://localhost:11434/v1".into());
400 }
401 }
402 let cfg: crate::http::OpenAiEmbedderConfig = parse(name, value)?;
403 Ok(Arc::new(crate::http::OpenAiEmbedder::new(cfg)?))
404 }
405 "hash" => {
406 check_keys(name, raw, &["dimensions"], false)?;
407 let dims = match raw.get("dimensions") {
408 None => 256,
409 Some(v) => v
410 .as_u64()
411 .and_then(|d| usize::try_from(d).ok())
412 .filter(|d| *d > 0)
413 .ok_or_else(|| RagError::Config(format!("`{name}`: `dimensions` must be a positive integer")))?,
414 };
415 let _ = env;
416 Ok(Arc::new(crate::hash::HashEmbedder::new(dims)))
417 }
418 other => Err(RagError::Config(format!("embedder `{name}`: unsupported type `{other}` in this build"))),
419 }
420}
421
422fn build_reranker(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Reranker>, RagError> {
423 let kind = provider_type(name, raw)?;
424 match kind.as_str() {
425 #[cfg(feature = "http")]
426 "http" | "cohere" | "jina" | "voyage" => {
427 check_keys(name, raw, &["base_url", "model"], true)?;
428 let cfg: crate::http::HttpRerankerConfig = parse(name, with_secret(name, raw, env)?)?;
429 Ok(Arc::new(crate::http::HttpReranker::new(cfg)?))
430 }
431 other => {
432 let _ = env;
433 Err(RagError::Config(format!("reranker `{name}`: unsupported type `{other}` in this build")))
434 }
435 }
436}
437
438fn build_store(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn VectorStore>, RagError> {
439 let kind = provider_type(name, raw)?;
440 match kind.as_str() {
441 "memory" => {
442 check_keys(name, raw, &["path"], false)?;
443 let path = match raw.get("path") {
444 None => None,
445 Some(serde_json::Value::String(p)) => Some(p.as_str()),
446 Some(_) => return Err(RagError::Config(format!("store `{name}`: `path` must be a string"))),
447 };
448 match path {
449 #[cfg(feature = "fs")]
450 Some(path) => Ok(Arc::new(crate::memory::MemoryStore::open(path)?)),
451 #[cfg(not(feature = "fs"))]
452 Some(_) => Err(RagError::Config(format!("store `{name}`: `path` requires the `fs` feature"))),
453 None => Ok(Arc::new(crate::memory::MemoryStore::new())),
454 }
455 }
456 #[cfg(feature = "sqlite")]
457 "sqlite" => {
458 check_keys(name, raw, &["path"], false)?;
459 let path =
460 raw.get("path").and_then(serde_json::Value::as_str).filter(|p| !p.trim().is_empty()).ok_or_else(
461 || RagError::Config(format!("store `{name}`: `path` (the database file) is required")),
462 )?;
463 Ok(Arc::new(crate::sqlite::SqliteStore::open(path)?))
464 }
465 #[cfg(feature = "qdrant")]
466 "qdrant" => {
467 check_keys(name, raw, &["url", "collection", "create_collection"], true)?;
468 let mut value = with_secret(name, raw, env)?;
469 if let Some(obj) = value.as_object_mut() {
470 obj.entry("api_key_header").or_insert_with(|| "api-key".into());
471 }
472 let cfg: crate::qdrant::QdrantConfig = parse(name, value)?;
473 Ok(Arc::new(crate::qdrant::QdrantStore::new(cfg)?))
474 }
475 other => {
476 let _ = env;
477 Err(RagError::Config(format!("store `{name}`: unsupported type `{other}` in this build")))
478 }
479 }
480}
481
482fn build_retriever(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Retriever>, RagError> {
483 let kind = provider_type(name, raw)?;
484 match kind.as_str() {
485 #[cfg(feature = "http")]
486 "http" => {
487 check_keys(name, raw, &["url"], true)?;
488 let cfg: crate::http::HttpRetrieverConfig = parse(name, with_secret(name, raw, env)?)?;
489 Ok(Arc::new(crate::http::HttpRetriever::new(cfg)?))
490 }
491 other => {
492 let _ = env;
493 Err(RagError::Config(format!("retriever for `{name}`: unsupported type `{other}` in this build")))
494 }
495 }
496}
497
498#[cfg(test)]
499mod tests {
500 use super::*;
501 use crate::hash::HashEmbedder;
502 use crate::ingest::Document;
503 use crate::memory::MemoryStore;
504
505 fn kb_with_two_datasets() -> KnowledgeBase {
506 let embedder: Arc<dyn Embedder> = Arc::new(HashEmbedder::new(128));
507 KnowledgeBase::new()
508 .with_store_dataset("pets", Arc::new(MemoryStore::new()), embedder.clone(), None)
509 .with_store_dataset("space", Arc::new(MemoryStore::new()), embedder, None)
510 }
511
512 async fn ingest(kb: &KnowledgeBase, ds: &str, id: &str, text: &str) {
513 kb.ingestor(ds).unwrap().ingest(&Document::new(ds, id, text)).await.unwrap();
514 }
515
516 #[tokio::test]
517 async fn multi_dataset_merge_and_single_mode() {
518 let kb = kb_with_two_datasets();
519 ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
520 ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
521 ingest(&kb, "space", "mars", "Mars is the red planet in our solar system.").await;
522
523 let req = RetrievalRequest::new("which pets purr", vec!["pets".into(), "space".into()]).top_k(2);
524 let hits = kb.retrieve(&req).await.unwrap();
525 assert_eq!(hits.len(), 2);
526 assert_eq!(hits[0].document_id, "cats");
527
528 let single = kb.retrieve_single(&req).await.unwrap();
529 assert!(single.iter().all(|h| h.dataset_id == "pets"));
530
531 let unknown = RetrievalRequest::new("x", vec!["nope".into()]);
532 assert!(matches!(kb.retrieve(&unknown).await, Err(RagError::NotFound(_))));
533 }
534
535 #[test]
536 fn config_rejects_inline_secrets_and_resolves_env() {
537 let cfg = KnowledgeConfig::from_json(
538 r#"{"embedders":{"e":{"type":"hash","dimensions":16}},
539 "stores":{"m":{"type":"memory"}},
540 "datasets":[{"id":"a","store":"m"}]}"#,
541 )
542 .unwrap();
543 let kb = cfg.build_with_env(&|_| None).unwrap();
544 assert!(kb.contains("a"));
545
546 let bad = KnowledgeConfig::from_json(
547 r#"{"rerankers":{"r":{"type":"cohere","base_url":"http://x","model":"m","api_key":"inline"}}}"#,
548 )
549 .unwrap();
550 let err = bad.build_with_env(&|_| None).unwrap_err();
551 assert!(err.to_string().contains("inline") || err.to_string().contains("unsupported"), "{err}");
552
553 let missing = KnowledgeConfig::from_json(r#"{"datasets":[{"id":"a","store":"nope"}]}"#).unwrap();
554 assert!(missing.build_with_env(&|_| None).is_err());
555 }
556
557 #[cfg(feature = "http")]
558 #[test]
559 fn config_env_secret_lookup() {
560 let cfg = KnowledgeConfig::from_json(
561 r#"{"embedders":{"e":{"type":"openai","base_url":"http://127.0.0.1:9","model":"m","api_key_env":"MY_KEY"}}}"#,
562 )
563 .unwrap();
564 assert!(cfg.build_with_env(&|_| None).unwrap_err().to_string().contains("MY_KEY"));
565 assert!(cfg.build_with_env(&|n| (n == "MY_KEY").then(|| "secret".to_owned())).is_ok());
566 }
567
568 #[cfg(feature = "yaml")]
569 #[test]
570 fn yaml_config() {
571 let cfg = KnowledgeConfig::from_yaml(
572 "embedders:\n e: { type: hash }\nstores:\n m: { type: memory }\ndatasets:\n - { id: kb, store: m, embedder: e }\n",
573 )
574 .unwrap();
575 assert_eq!(cfg.datasets.len(), 1);
576 assert!(cfg.build_with_env(&|_| None).is_ok());
577 }
578
579 #[cfg(feature = "sqlite")]
580 #[tokio::test]
581 async fn sqlite_store_from_config_persists_between_builds() {
582 let dir = std::env::temp_dir().join(format!("graphrt-rag-kb-sqlite-{}", std::process::id()));
583 let _ = std::fs::remove_dir_all(&dir);
584 std::fs::create_dir_all(&dir).unwrap();
585 let path = dir.join("kb.db");
586 let config = KnowledgeConfig::from_json(
587 &serde_json::json!({
588 "embedders": { "e": { "type": "hash", "dimensions": 64 } },
589 "stores": { "local": { "type": "sqlite", "path": path } },
590 "datasets": [{ "id": "faq", "store": "local", "embedder": "e" }]
591 })
592 .to_string(),
593 )
594 .unwrap();
595
596 let first = config.build_with_env(&|_| None).unwrap();
597 ingest(&first, "faq", "returns", "Items can be returned within 30 days.").await;
598 drop(first);
599
600 let second = config.build_with_env(&|_| None).unwrap();
601 let hits = second.retrieve(&RetrievalRequest::new("returned within days", vec!["faq".into()])).await.unwrap();
602 assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("returns"));
603 let _ = std::fs::remove_dir_all(&dir);
604 }
605
606 #[cfg(feature = "sqlite")]
607 #[test]
608 fn sqlite_store_requires_a_path() {
609 let config = KnowledgeConfig::from_json(r#"{"stores":{"local":{"type":"sqlite"}}}"#).unwrap();
610 let error = config.build_with_env(&|_| None).unwrap_err();
611 assert!(error.to_string().contains("`path` (the database file) is required"), "{error}");
612 }
613}