1use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
18
19use indexmap::IndexMap;
20use serde::{Deserialize, Serialize};
21
22use crate::chunker::ChunkerConfig;
23use crate::error::RagError;
24use crate::ingest::Ingestor;
25use crate::pipeline::{StoreRetriever, finalize, rerank};
26use crate::types::{
27 DocumentSummary, Embedder, Hit, MAX_DOCUMENT_PAGE, Reranker, RetrievalRequest, Retriever, StoredChunk, VectorStore,
28};
29
30#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
32pub struct DatasetInfo {
33 pub id: String,
34 #[serde(default)]
35 pub name: String,
36 #[serde(default)]
37 pub description: String,
38}
39
40#[derive(Clone, Debug)]
41struct DatasetEntry {
42 info: DatasetInfo,
43 retriever: Arc<dyn Retriever>,
44 store: Option<(Arc<dyn VectorStore>, Arc<dyn Embedder>)>,
45 chunking: ChunkerConfig,
47 names: Option<(String, String)>,
49 runtime: Option<DatasetSpec>,
51}
52
53#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
56#[serde(deny_unknown_fields)]
57pub struct DatasetSpec {
58 pub id: String,
59 #[serde(default)]
60 pub name: String,
61 #[serde(default)]
62 pub description: String,
63 pub store: String,
65 pub embedder: String,
67 #[serde(default)]
69 pub reranker: Option<String>,
70 #[serde(default)]
71 pub chunking: Option<ChunkerConfig>,
72}
73
74#[derive(Clone, Debug, PartialEq, Serialize)]
76pub struct DatasetDetails {
77 #[serde(flatten)]
78 pub info: DatasetInfo,
79 pub managed: bool,
81 pub runtime: bool,
83 pub store: Option<String>,
84 pub embedder: Option<String>,
85 pub chunking: Option<ChunkerConfig>,
86}
87
88#[derive(Clone, Debug, Default, PartialEq, Serialize)]
90pub struct ResourceNames {
91 pub stores: Vec<String>,
92 pub embedders: Vec<String>,
93 pub rerankers: Vec<String>,
94}
95
96#[derive(Clone, Debug, Default)]
98struct Resources {
99 stores: IndexMap<String, Arc<dyn VectorStore>>,
100 embedders: IndexMap<String, Arc<dyn Embedder>>,
101 rerankers: IndexMap<String, Arc<dyn Reranker>>,
102}
103
104type Registry = IndexMap<String, DatasetEntry>;
105
106#[derive(Clone, Debug, Default)]
112pub struct KnowledgeBase {
113 datasets: Arc<RwLock<Registry>>,
114 resources: Arc<Resources>,
115 reranker: Option<Arc<dyn Reranker>>,
117}
118
119impl KnowledgeBase {
120 pub fn new() -> Self {
121 Self::default()
122 }
123
124 fn read(&self) -> RwLockReadGuard<'_, Registry> {
126 self.datasets.read().unwrap_or_else(PoisonError::into_inner)
127 }
128
129 fn write(&self) -> RwLockWriteGuard<'_, Registry> {
130 self.datasets.write().unwrap_or_else(PoisonError::into_inner)
131 }
132
133 pub fn with_store_dataset(
135 mut self,
136 id: impl Into<String>,
137 store: Arc<dyn VectorStore>,
138 embedder: Arc<dyn Embedder>,
139 reranker: Option<Arc<dyn Reranker>>,
140 ) -> Self {
141 let id = id.into();
142 self.add_store_dataset(DatasetInfo { id, ..DatasetInfo::default() }, store, embedder, reranker);
143 self
144 }
145
146 pub fn add_store_dataset(
148 &mut self,
149 info: DatasetInfo,
150 store: Arc<dyn VectorStore>,
151 embedder: Arc<dyn Embedder>,
152 reranker: Option<Arc<dyn Reranker>>,
153 ) -> &mut Self {
154 self.insert_store_dataset(info, store, embedder, reranker, ChunkerConfig::default(), None, None);
155 self
156 }
157
158 #[allow(clippy::too_many_arguments)]
159 fn insert_store_dataset(
160 &self,
161 info: DatasetInfo,
162 store: Arc<dyn VectorStore>,
163 embedder: Arc<dyn Embedder>,
164 reranker: Option<Arc<dyn Reranker>>,
165 chunking: ChunkerConfig,
166 names: Option<(String, String)>,
167 runtime: Option<DatasetSpec>,
168 ) {
169 let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
170 let entry = DatasetEntry {
171 info,
172 retriever: Arc::new(retriever),
173 store: Some((store, embedder)),
174 chunking,
175 names,
176 runtime,
177 };
178 self.write().insert(entry.info.id.clone(), entry);
179 }
180
181 pub fn with_retriever(mut self, id: impl Into<String>, retriever: Arc<dyn Retriever>) -> Self {
183 let id = id.into();
184 self.add_retriever(DatasetInfo { id, ..DatasetInfo::default() }, retriever);
185 self
186 }
187
188 pub fn add_retriever(&mut self, info: DatasetInfo, retriever: Arc<dyn Retriever>) -> &mut Self {
190 let entry = DatasetEntry {
191 info,
192 retriever,
193 store: None,
194 chunking: ChunkerConfig::default(),
195 names: None,
196 runtime: None,
197 };
198 self.write().insert(entry.info.id.clone(), entry);
199 self
200 }
201
202 pub fn with_reranker(mut self, reranker: Option<Arc<dyn Reranker>>) -> Self {
204 self.reranker = reranker;
205 self
206 }
207
208 pub fn datasets(&self) -> Vec<DatasetInfo> {
210 self.read().values().map(|d| d.info.clone()).collect()
211 }
212
213 pub fn dataset_details(&self) -> Vec<DatasetDetails> {
215 self.read()
216 .values()
217 .map(|d| DatasetDetails {
218 info: d.info.clone(),
219 managed: d.store.is_some(),
220 runtime: d.runtime.is_some(),
221 store: d.names.as_ref().map(|(store, _)| store.clone()),
222 embedder: d.names.as_ref().map(|(_, embedder)| embedder.clone()),
223 chunking: d.store.as_ref().map(|_| d.chunking.clone()),
224 })
225 .collect()
226 }
227
228 pub fn resource_names(&self) -> ResourceNames {
230 ResourceNames {
231 stores: self.resources.stores.keys().cloned().collect(),
232 embedders: self.resources.embedders.keys().cloned().collect(),
233 rerankers: self.resources.rerankers.keys().cloned().collect(),
234 }
235 }
236
237 pub fn create_dataset(&self, spec: DatasetSpec) -> Result<DatasetDetails, RagError> {
243 validate_dataset_id(&spec.id)?;
244 let lookup = |kind: &str, name: &str, found: bool| {
245 if found {
246 Ok(())
247 } else {
248 Err(RagError::Input(format!("unknown {kind} `{name}` in the knowledge config")))
249 }
250 };
251 let store = self.resources.stores.get(&spec.store).cloned();
252 lookup("store", &spec.store, store.is_some())?;
253 let embedder = self.resources.embedders.get(&spec.embedder).cloned();
254 lookup("embedder", &spec.embedder, embedder.is_some())?;
255 let reranker = match &spec.reranker {
256 Some(name) => {
257 let reranker = self.resources.rerankers.get(name).cloned();
258 lookup("reranker", name, reranker.is_some())?;
259 reranker
260 }
261 None => None,
262 };
263 let chunking = spec.chunking.clone().unwrap_or_default();
264 validate_chunking(&spec.id, &chunking).map_err(|e| RagError::Input(e.to_string()))?;
265 let (Some(store), Some(embedder)) = (store, embedder) else {
266 return Err(RagError::Input("store and embedder are required".into()));
267 };
268 let info = DatasetInfo { id: spec.id.clone(), name: spec.name.clone(), description: spec.description.clone() };
269 {
270 let mut registry = self.write();
272 if registry.contains_key(&spec.id) {
273 return Err(RagError::Input(format!("dataset `{}` already exists", spec.id)));
274 }
275 let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
276 registry.insert(
277 spec.id.clone(),
278 DatasetEntry {
279 info,
280 retriever: Arc::new(retriever),
281 store: Some((store, embedder)),
282 chunking,
283 names: Some((spec.store.clone(), spec.embedder.clone())),
284 runtime: Some(spec.clone()),
285 },
286 );
287 }
288 self.dataset_details()
289 .into_iter()
290 .find(|d| d.info.id == spec.id)
291 .ok_or_else(|| RagError::Store(format!("dataset `{}` vanished after creation", spec.id)))
292 }
293
294 pub async fn remove_dataset(&self, dataset_id: &str) -> Result<usize, RagError> {
300 let entry = self.entry(dataset_id)?;
301 if entry.runtime.is_none() {
302 return Err(RagError::Unsupported(format!(
303 "dataset `{dataset_id}` is declared in the knowledge config; remove it there"
304 )));
305 }
306 let removed = match &entry.store {
308 Some((store, _)) => store.delete_dataset(dataset_id).await?,
309 None => 0,
310 };
311 self.write().shift_remove(dataset_id);
312 Ok(removed)
313 }
314
315 pub fn runtime_datasets(&self) -> Vec<DatasetSpec> {
317 self.read().values().filter_map(|d| d.runtime.clone()).collect()
318 }
319
320 pub fn contains(&self, dataset_id: &str) -> bool {
321 self.read().contains_key(dataset_id)
322 }
323
324 pub fn ingestor(&self, dataset_id: &str) -> Option<Ingestor> {
327 let registry = self.read();
328 let entry = registry.get(dataset_id)?;
329 let (store, embedder) = entry.store.as_ref()?;
330 Some(Ingestor::new(store.clone(), embedder.clone()).chunker(entry.chunking.clone()))
331 }
332
333 pub fn set_chunking(&mut self, dataset_id: &str, chunking: ChunkerConfig) -> Result<&mut Self, RagError> {
338 validate_chunking(dataset_id, &chunking)?;
339 {
340 let mut registry = self.write();
341 let entry = registry
342 .get_mut(dataset_id)
343 .ok_or_else(|| RagError::NotFound(format!("unknown dataset `{dataset_id}`")))?;
344 entry.chunking = chunking;
345 }
346 Ok(self)
347 }
348
349 pub fn chunking(&self, dataset_id: &str) -> Option<ChunkerConfig> {
351 self.read().get(dataset_id).map(|d| d.chunking.clone())
352 }
353
354 fn entry(&self, id: &str) -> Result<DatasetEntry, RagError> {
356 self.read().get(id).cloned().ok_or_else(|| RagError::NotFound(format!("unknown dataset `{id}`")))
357 }
358
359 pub fn is_managed(&self, dataset_id: &str) -> bool {
362 self.read().get(dataset_id).is_some_and(|d| d.store.is_some())
363 }
364
365 fn managed_store(&self, dataset_id: &str) -> Result<Arc<dyn VectorStore>, RagError> {
370 let entry = self.entry(dataset_id)?;
371 entry
372 .store
373 .map(|(store, _)| store)
374 .ok_or_else(|| RagError::Unsupported(format!("dataset `{dataset_id}` is served by an external retriever")))
375 }
376
377 pub async fn list_documents(
380 &self,
381 dataset_id: &str,
382 limit: usize,
383 after: Option<&str>,
384 ) -> Result<Vec<DocumentSummary>, RagError> {
385 let limit = limit.clamp(1, MAX_DOCUMENT_PAGE);
386 self.managed_store(dataset_id)?.list_documents(dataset_id, limit, after).await
387 }
388
389 pub async fn document_chunks(&self, dataset_id: &str, document_id: &str) -> Result<Vec<StoredChunk>, RagError> {
391 let mut chunks = self.managed_store(dataset_id)?.document_chunks(dataset_id, document_id).await?;
392 chunks.sort_by_key(|c| c.metadata.get("chunk_index").and_then(serde_json::Value::as_u64).unwrap_or(u64::MAX));
393 Ok(chunks)
394 }
395
396 pub async fn set_document_enabled(
402 &self,
403 dataset_id: &str,
404 document_id: &str,
405 enabled: bool,
406 ) -> Result<usize, RagError> {
407 let store = self.managed_store(dataset_id)?;
408 let mut chunks = store.document_chunks(dataset_id, document_id).await?;
409 if chunks.is_empty() {
410 return Err(RagError::NotFound(format!("unknown document `{document_id}` in dataset `{dataset_id}`")));
411 }
412 let count = chunks.len();
413 chunks.retain(|c| crate::types::is_disabled(&c.metadata) == enabled);
414 for chunk in &mut chunks {
415 if enabled {
416 chunk.metadata.remove(crate::types::DISABLED_KEY);
417 } else {
418 chunk.metadata.insert(crate::types::DISABLED_KEY.into(), serde_json::Value::Bool(true));
419 }
420 }
421 if !chunks.is_empty() {
422 store.upsert(chunks).await?;
423 }
424 Ok(count)
425 }
426
427 pub async fn delete_document(&self, dataset_id: &str, document_id: &str) -> Result<usize, RagError> {
429 self.managed_store(dataset_id)?.delete_document(dataset_id, document_id).await
430 }
431
432 async fn query_each(
434 &self,
435 request: &RetrievalRequest,
436 rerank_each: bool,
437 ) -> Result<Vec<(String, Vec<Hit>)>, RagError> {
438 let mut jobs = Vec::with_capacity(request.datasets.len());
439 for id in &request.datasets {
440 let retriever = self.entry(id)?.retriever;
441 let sub = RetrievalRequest {
442 datasets: vec![id.clone()],
443 score_threshold: None,
445 rerank: rerank_each,
446 ..request.clone()
447 };
448 jobs.push(async move { retriever.retrieve(&sub).await.map(|hits| (id.clone(), hits)) });
449 }
450 futures::future::try_join_all(jobs).await
451 }
452
453 pub async fn retrieve(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
460 validate_request(request)?;
461 let global_rerank = request.rerank && self.reranker.is_some() && request.datasets.len() > 1;
462 let per_dataset = self.query_each(request, request.rerank && !global_rerank).await?;
463 let mut merged: Vec<Hit> = per_dataset.into_iter().flat_map(|(_, hits)| hits).collect();
464 if global_rerank {
465 if let Some(reranker) = &self.reranker {
466 merged = rerank(reranker.as_ref(), &request.query, merged).await?;
467 }
468 }
469 Ok(finalize(merged, request.score_threshold, request.top_k))
470 }
471
472 pub async fn retrieve_single(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
475 validate_request(request)?;
476 let per_dataset = self.query_each(request, request.rerank).await?;
477 let best = per_dataset
478 .into_iter()
479 .filter(|(_, hits)| !hits.is_empty())
480 .max_by(|a, b| top_score(&a.1).total_cmp(&top_score(&b.1)));
481 Ok(best.map(|(_, hits)| finalize(hits, request.score_threshold, request.top_k)).unwrap_or_default())
482 }
483}
484
485fn validate_dataset_id(id: &str) -> Result<(), RagError> {
488 let ok = !id.is_empty()
489 && id.len() <= 64
490 && !id.starts_with('.')
491 && id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'));
492 if ok {
493 Ok(())
494 } else {
495 Err(RagError::Input(format!(
496 "invalid dataset id `{id}`: use 1-64 ASCII letters, digits, `-`, `_` or `.`, not starting with `.`"
497 )))
498 }
499}
500
501fn validate_chunking(dataset_id: &str, chunking: &ChunkerConfig) -> Result<(), RagError> {
502 if chunking.max_chars == 0 {
503 return Err(RagError::Config(format!("dataset `{dataset_id}`: `chunking.max_chars` must be positive")));
504 }
505 if chunking.parent_max_chars.is_some_and(|p| p <= chunking.max_chars) {
506 return Err(RagError::Config(format!(
507 "dataset `{dataset_id}`: `chunking.parent_max_chars` must be larger than `max_chars`"
508 )));
509 }
510 Ok(())
511}
512
513fn validate_request(request: &RetrievalRequest) -> Result<(), RagError> {
515 if request.datasets.is_empty() {
516 return Err(RagError::Input("no dataset ids given".into()));
517 }
518 if let Some(filter) = &request.filter {
519 filter.validate().map_err(|e| RagError::Input(e.to_string()))?;
520 }
521 Ok(())
522}
523
524fn top_score(hits: &[Hit]) -> f32 {
525 hits.iter().map(|h| h.score).fold(f32::MIN, f32::max)
526}
527
528#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
554#[serde(deny_unknown_fields)]
555pub struct KnowledgeConfig {
556 #[serde(default)]
557 pub embedders: IndexMap<String, serde_json::Value>,
558 #[serde(default)]
559 pub rerankers: IndexMap<String, serde_json::Value>,
560 #[serde(default)]
561 pub stores: IndexMap<String, serde_json::Value>,
562 #[serde(default)]
563 pub datasets: Vec<DatasetConfig>,
564 #[serde(default)]
566 pub reranker: Option<String>,
567}
568
569#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
571#[serde(deny_unknown_fields)]
572pub struct DatasetConfig {
573 pub id: String,
574 #[serde(default)]
575 pub name: String,
576 #[serde(default)]
577 pub description: String,
578 #[serde(default)]
580 pub store: Option<String>,
581 #[serde(default)]
583 pub embedder: Option<String>,
584 #[serde(default)]
586 pub reranker: Option<String>,
587 #[serde(default)]
589 pub retriever: Option<serde_json::Value>,
590 #[serde(default)]
593 pub chunking: Option<ChunkerConfig>,
594}
595
596pub type EnvLookup<'a> = &'a dyn Fn(&str) -> Option<String>;
598
599impl KnowledgeConfig {
600 pub fn from_json(text: &str) -> Result<Self, RagError> {
601 serde_json::from_str(text).map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
602 }
603
604 #[cfg(feature = "yaml")]
605 pub fn from_yaml(text: &str) -> Result<Self, RagError> {
606 let options = serde_saphyr::options! { strict_booleans: true };
607 serde_saphyr::from_str_with_options(text, options)
608 .map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
609 }
610
611 pub fn build(&self) -> Result<KnowledgeBase, RagError> {
613 self.build_with_env(&|name| std::env::var(name).ok())
614 }
615
616 pub fn build_with_env(&self, env: EnvLookup<'_>) -> Result<KnowledgeBase, RagError> {
618 let mut embedders: IndexMap<&str, Arc<dyn Embedder>> = IndexMap::new();
619 for (name, raw) in &self.embedders {
620 embedders.insert(name, build_embedder(name, raw, env)?);
621 }
622 let mut rerankers: IndexMap<&str, Arc<dyn Reranker>> = IndexMap::new();
623 for (name, raw) in &self.rerankers {
624 rerankers.insert(name, build_reranker(name, raw, env)?);
625 }
626 let mut stores: IndexMap<&str, Arc<dyn VectorStore>> = IndexMap::new();
627 for (name, raw) in &self.stores {
628 stores.insert(name, build_store(name, raw, env)?);
629 }
630
631 let lookup_reranker = |key: &Option<String>| -> Result<Option<Arc<dyn Reranker>>, RagError> {
632 match key {
633 None => Ok(None),
634 Some(k) => rerankers
635 .get(k.as_str())
636 .cloned()
637 .map(Some)
638 .ok_or_else(|| RagError::Config(format!("unknown reranker `{k}`"))),
639 }
640 };
641
642 let mut kb = KnowledgeBase::new().with_reranker(lookup_reranker(&self.reranker)?);
643 for ds in &self.datasets {
644 if ds.id.trim().is_empty() {
645 return Err(RagError::Config("dataset id must not be empty".into()));
646 }
647 if kb.contains(&ds.id) {
648 return Err(RagError::Config(format!("duplicate dataset `{}`", ds.id)));
649 }
650 let info = DatasetInfo { id: ds.id.clone(), name: ds.name.clone(), description: ds.description.clone() };
651 if let Some(raw) = &ds.retriever {
652 kb.add_retriever(info, build_retriever(&ds.id, raw, env)?);
653 continue;
654 }
655 let store_key = ds.store.as_deref().ok_or_else(|| {
656 RagError::Config(format!("dataset `{}` needs either `store` + `embedder` or `retriever`", ds.id))
657 })?;
658 let store = stores
659 .get(store_key)
660 .cloned()
661 .ok_or_else(|| RagError::Config(format!("unknown store `{store_key}`")))?;
662 let embedder_key = match ds.embedder.as_deref() {
663 Some(k) => k,
664 None if embedders.len() == 1 => embedders.keys().next().copied().unwrap_or_default(),
665 None => return Err(RagError::Config(format!("dataset `{}` needs `embedder`", ds.id))),
666 };
667 let embedder = embedders
668 .get(embedder_key)
669 .cloned()
670 .ok_or_else(|| RagError::Config(format!("unknown embedder `{embedder_key}`")))?;
671 let chunking = ds.chunking.clone().unwrap_or_default();
672 validate_chunking(&ds.id, &chunking)?;
673 let names = Some((store_key.to_owned(), embedder_key.to_owned()));
674 kb.insert_store_dataset(info, store, embedder, lookup_reranker(&ds.reranker)?, chunking, names, None);
675 }
676 kb.resources = Arc::new(Resources {
677 stores: stores.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
678 embedders: embedders.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
679 rerankers: rerankers.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
680 });
681 Ok(kb)
682 }
683}
684
685#[cfg_attr(not(feature = "http"), allow(dead_code))]
687const HTTP_KEYS: &[&str] = &["type", "api_key", "api_key_env", "api_key_header", "headers", "timeout_ms"];
688
689fn check_keys(name: &str, raw: &serde_json::Value, allowed: &[&str], with_http: bool) -> Result<(), RagError> {
693 let obj = raw.as_object().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
694 for key in obj.keys() {
695 let known = allowed.contains(&key.as_str()) || (with_http && HTTP_KEYS.contains(&key.as_str()));
696 if !known && key != "type" {
697 return Err(RagError::Config(format!("`{name}`: unknown key `{key}`")));
698 }
699 }
700 Ok(())
701}
702
703fn provider_type(name: &str, raw: &serde_json::Value) -> Result<String, RagError> {
704 raw.get("type")
705 .and_then(|t| t.as_str())
706 .map(str::to_owned)
707 .ok_or_else(|| RagError::Config(format!("`{name}` needs a `type`")))
708}
709
710#[cfg_attr(not(feature = "http"), allow(dead_code))]
712fn with_secret(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<serde_json::Value, RagError> {
713 let mut obj = raw.as_object().cloned().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
714 obj.remove("type");
715 if obj.contains_key("api_key") {
716 return Err(RagError::Config(format!(
717 "`{name}`: inline `api_key` is not allowed; use `api_key_env` with an environment variable name"
718 )));
719 }
720 if let Some(var) = obj.remove("api_key_env") {
721 let var = var.as_str().ok_or_else(|| RagError::Config(format!("`{name}`: `api_key_env` must be a string")))?;
722 let value =
723 env(var).ok_or_else(|| RagError::Config(format!("`{name}`: environment variable `{var}` is not set")))?;
724 obj.insert("api_key".into(), serde_json::Value::String(value));
725 }
726 Ok(serde_json::Value::Object(obj))
727}
728
729#[cfg_attr(not(feature = "http"), allow(dead_code))]
730fn parse<T: serde::de::DeserializeOwned>(name: &str, value: serde_json::Value) -> Result<T, RagError> {
731 serde_json::from_value(value).map_err(|e| RagError::Config(format!("`{name}`: {e}")))
732}
733
734fn build_embedder(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Embedder>, RagError> {
735 let kind = provider_type(name, raw)?;
736 match kind.as_str() {
737 #[cfg(feature = "http")]
738 "openai" | "openai-compatible" | "ollama" => {
739 check_keys(
740 name,
741 raw,
742 &["base_url", "model", "dimensions", "batch_size", "query_prefix", "document_prefix"],
743 true,
744 )?;
745 let mut value = with_secret(name, raw, env)?;
746 if kind == "ollama" {
747 if let Some(obj) = value.as_object_mut() {
748 obj.entry("base_url").or_insert_with(|| "http://localhost:11434/v1".into());
749 }
750 }
751 let cfg: crate::http::OpenAiEmbedderConfig = parse(name, value)?;
752 Ok(Arc::new(crate::http::OpenAiEmbedder::new(cfg)?))
753 }
754 #[cfg(feature = "local-embed")]
755 "local" => {
756 let mut value = raw.as_object().cloned().unwrap_or_default();
757 value.remove("type");
758 let config: crate::local_embed::LocalEmbedderConfig = parse(name, serde_json::Value::Object(value))?;
759 Ok(Arc::new(crate::local_embed::LocalEmbedder::new(config)?))
760 }
761 "hash" => {
762 check_keys(name, raw, &["dimensions"], false)?;
763 let dims = match raw.get("dimensions") {
764 None => 256,
765 Some(v) => v
766 .as_u64()
767 .and_then(|d| usize::try_from(d).ok())
768 .filter(|d| *d > 0)
769 .ok_or_else(|| RagError::Config(format!("`{name}`: `dimensions` must be a positive integer")))?,
770 };
771 let _ = env;
772 Ok(Arc::new(crate::hash::HashEmbedder::new(dims)))
773 }
774 other => Err(RagError::Config(format!("embedder `{name}`: unsupported type `{other}` in this build"))),
775 }
776}
777
778fn build_reranker(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Reranker>, RagError> {
779 let kind = provider_type(name, raw)?;
780 match kind.as_str() {
781 #[cfg(feature = "http")]
782 "http" | "cohere" | "jina" | "voyage" => {
783 check_keys(name, raw, &["base_url", "model"], true)?;
784 let cfg: crate::http::HttpRerankerConfig = parse(name, with_secret(name, raw, env)?)?;
785 Ok(Arc::new(crate::http::HttpReranker::new(cfg)?))
786 }
787 other => {
788 let _ = env;
789 Err(RagError::Config(format!("reranker `{name}`: unsupported type `{other}` in this build")))
790 }
791 }
792}
793
794fn build_store(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn VectorStore>, RagError> {
795 let kind = provider_type(name, raw)?;
796 match kind.as_str() {
797 "memory" => {
798 check_keys(name, raw, &["path"], false)?;
799 let path = match raw.get("path") {
800 None => None,
801 Some(serde_json::Value::String(p)) => Some(p.as_str()),
802 Some(_) => return Err(RagError::Config(format!("store `{name}`: `path` must be a string"))),
803 };
804 match path {
805 #[cfg(feature = "fs")]
806 Some(path) => Ok(Arc::new(crate::memory::MemoryStore::open(path)?)),
807 #[cfg(not(feature = "fs"))]
808 Some(_) => Err(RagError::Config(format!("store `{name}`: `path` requires the `fs` feature"))),
809 None => Ok(Arc::new(crate::memory::MemoryStore::new())),
810 }
811 }
812 #[cfg(feature = "sqlite")]
813 "sqlite" => {
814 check_keys(name, raw, &["path"], false)?;
815 let path =
816 raw.get("path").and_then(serde_json::Value::as_str).filter(|p| !p.trim().is_empty()).ok_or_else(
817 || RagError::Config(format!("store `{name}`: `path` (the database file) is required")),
818 )?;
819 Ok(Arc::new(crate::sqlite::SqliteStore::open(path)?))
820 }
821 #[cfg(feature = "qdrant")]
822 "qdrant" => {
823 check_keys(name, raw, &["url", "collection", "create_collection"], true)?;
824 let mut value = with_secret(name, raw, env)?;
825 if let Some(obj) = value.as_object_mut() {
826 obj.entry("api_key_header").or_insert_with(|| "api-key".into());
827 }
828 let cfg: crate::qdrant::QdrantConfig = parse(name, value)?;
829 Ok(Arc::new(crate::qdrant::QdrantStore::new(cfg)?))
830 }
831 other => {
832 let _ = env;
833 Err(RagError::Config(format!("store `{name}`: unsupported type `{other}` in this build")))
834 }
835 }
836}
837
838fn build_retriever(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Retriever>, RagError> {
839 let kind = provider_type(name, raw)?;
840 match kind.as_str() {
841 #[cfg(feature = "http")]
842 "http" => {
843 check_keys(name, raw, &["url"], true)?;
844 let cfg: crate::http::HttpRetrieverConfig = parse(name, with_secret(name, raw, env)?)?;
845 Ok(Arc::new(crate::http::HttpRetriever::new(cfg)?))
846 }
847 other => {
848 let _ = env;
849 Err(RagError::Config(format!("retriever for `{name}`: unsupported type `{other}` in this build")))
850 }
851 }
852}
853
854#[cfg(test)]
855mod tests {
856 use super::*;
857 use crate::hash::HashEmbedder;
858 use crate::ingest::Document;
859 use crate::memory::MemoryStore;
860
861 fn kb_with_two_datasets() -> KnowledgeBase {
862 let embedder: Arc<dyn Embedder> = Arc::new(HashEmbedder::new(128));
863 KnowledgeBase::new()
864 .with_store_dataset("pets", Arc::new(MemoryStore::new()), embedder.clone(), None)
865 .with_store_dataset("space", Arc::new(MemoryStore::new()), embedder, None)
866 }
867
868 async fn ingest(kb: &KnowledgeBase, ds: &str, id: &str, text: &str) {
869 kb.ingestor(ds).unwrap().ingest(&Document::new(ds, id, text)).await.unwrap();
870 }
871
872 #[tokio::test]
873 async fn documents_can_be_listed_paged_inspected_and_deleted() {
874 let kb = kb_with_two_datasets();
875 let ingestor = kb.ingestor("pets").unwrap();
876 let mut meta = crate::types::Metadata::new();
877 meta.insert("lang".into(), serde_json::json!("en"));
878 let long = "Cats purr. ".repeat(200);
879 ingestor.ingest(&Document::new("pets", "cats", long).title("Cats").metadata(meta)).await.unwrap();
880 ingest(&kb, "pets", "dogs", "Dogs bark.").await;
881 ingest(&kb, "pets", "birds", "Birds sing.").await;
882
883 let first = kb.list_documents("pets", 2, None).await.unwrap();
884 let ids: Vec<&str> = first.iter().map(|d| d.document_id.as_str()).collect();
885 assert_eq!(ids, ["birds", "cats"]);
886 let cats = &first[1];
887 assert_eq!(cats.title, "Cats");
888 assert!(cats.chunks > 1, "long document is split");
889 assert_eq!(cats.metadata.get("lang"), Some(&serde_json::json!("en")));
890 assert!(!cats.metadata.contains_key("chunk_index"), "per-chunk keys are not document metadata");
891 let next = kb.list_documents("pets", 2, Some("cats")).await.unwrap();
892 assert_eq!(next.iter().map(|d| d.document_id.as_str()).collect::<Vec<_>>(), ["dogs"]);
893
894 let chunks = kb.document_chunks("pets", "cats").await.unwrap();
895 assert_eq!(chunks.len(), cats.chunks);
896 let order: Vec<u64> = chunks.iter().map(|c| c.metadata["chunk_index"].as_u64().unwrap()).collect();
897 assert!(order.windows(2).all(|w| w[0] < w[1]), "chunks come in document order: {order:?}");
898
899 assert_eq!(kb.delete_document("pets", "cats").await.unwrap(), cats.chunks);
900 assert_eq!(kb.delete_document("pets", "cats").await.unwrap(), 0);
901 assert_eq!(kb.list_documents("pets", 50, None).await.unwrap().len(), 2);
902 }
903
904 #[tokio::test]
905 async fn disabled_documents_are_kept_but_never_retrieved() {
906 let kb = kb_with_two_datasets();
907 ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
908 ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
909 let query = RetrievalRequest::new("pets that purr", vec!["pets".into()]).top_k(5);
910 let found = |hits: &[Hit]| hits.iter().any(|h| h.document_id == "cats");
911 assert!(found(&kb.retrieve(&query).await.unwrap()));
912
913 assert!(kb.set_document_enabled("pets", "cats", false).await.unwrap() >= 1);
914 assert!(!found(&kb.retrieve(&query).await.unwrap()), "disabled document retrieved");
915 let docs = kb.list_documents("pets", 10, None).await.unwrap();
916 let cats = docs.iter().find(|d| d.document_id == "cats").unwrap();
917 assert!(!cats.enabled);
918 assert!(!cats.metadata.contains_key(crate::types::DISABLED_KEY), "flag is not document metadata");
919
920 ingest(&kb, "pets", "cats", "Cats are small furry pets that purr and nap.").await;
922 assert!(!found(&kb.retrieve(&query).await.unwrap()), "re-ingestion re-enabled the document");
923
924 kb.set_document_enabled("pets", "cats", true).await.unwrap();
925 assert!(found(&kb.retrieve(&query).await.unwrap()));
926 assert_eq!(kb.set_document_enabled("pets", "nope", false).await.unwrap_err().kind(), "not_found");
927 }
928
929 #[tokio::test]
930 async fn parent_child_datasets_return_whole_parents_once() {
931 let mut kb = KnowledgeBase::new().with_store_dataset(
932 "docs",
933 Arc::new(MemoryStore::new()),
934 Arc::new(HashEmbedder::new(256)),
935 None,
936 );
937 let chunking = ChunkerConfig { max_chars: 60, overlap: 0, markdown: false, parent_max_chars: Some(220) };
938 kb.set_chunking("docs", chunking).unwrap();
939 let text = "Leave policy. Every employee gets 24 days of paid annual leave. Unused leave carries over for six months. Sick leave is separate.\n\n\
940 Expense policy. Expenses below 500 USD are approved by your manager. Larger ones need finance approval.";
941 ingest(&kb, "docs", "handbook", text).await;
942 let children = kb.document_chunks("docs", "handbook").await.unwrap();
943 assert!(children.len() > 2, "the document is split into several children");
944
945 let request = RetrievalRequest::new("annual leave days carries over", vec!["docs".into()]).top_k(5);
946 let hits = kb.retrieve(&request).await.unwrap();
947 let leave: Vec<&Hit> = hits.iter().filter(|h| h.content.contains("24 days")).collect();
948 assert_eq!(leave.len(), 1, "children of one parent collapse into one hit: {hits:?}");
949 let parent = &leave[0].content;
950 assert!(parent.contains("Leave policy") && parent.contains("Sick leave"), "whole parent returned: {parent}");
951 assert!(!parent.contains("Expense"), "parents stay separate: {parent}");
952 }
953
954 #[test]
955 fn chunking_config_is_validated() {
956 let build = |chunking: serde_json::Value| {
957 KnowledgeConfig::from_json(
958 &serde_json::json!({
959 "embedders": { "e": { "type": "hash" } },
960 "stores": { "m": { "type": "memory" } },
961 "datasets": [{ "id": "d", "store": "m", "embedder": "e", "chunking": chunking }]
962 })
963 .to_string(),
964 )
965 .unwrap()
966 .build_with_env(&|_| None)
967 };
968 let kb = build(serde_json::json!({ "max_chars": 300, "parent_max_chars": 1200 })).unwrap();
969 assert_eq!(kb.chunking("d").unwrap().parent_max_chars, Some(1200));
970 let error = build(serde_json::json!({ "max_chars": 300, "parent_max_chars": 200 })).unwrap_err();
971 assert!(error.to_string().contains("parent_max_chars"), "{error}");
972 }
973
974 fn config_kb() -> KnowledgeBase {
975 KnowledgeConfig::from_json(
976 &serde_json::json!({
977 "embedders": { "e": { "type": "hash", "dimensions": 64 } },
978 "stores": { "shared": { "type": "memory" } },
979 "datasets": [{ "id": "static", "store": "shared", "embedder": "e" }]
980 })
981 .to_string(),
982 )
983 .unwrap()
984 .build_with_env(&|_| None)
985 .unwrap()
986 }
987
988 fn spec(id: &str) -> DatasetSpec {
989 DatasetSpec {
990 id: id.into(),
991 name: "Runtime".into(),
992 description: String::new(),
993 store: "shared".into(),
994 embedder: "e".into(),
995 reranker: None,
996 chunking: None,
997 }
998 }
999
1000 #[tokio::test]
1001 async fn runtime_datasets_are_shared_by_clones_and_removable() {
1002 let api = config_kb();
1003 let workflows = api.clone();
1004 assert_eq!(api.resource_names().stores, ["shared"]);
1005
1006 let created = api.create_dataset(spec("notes")).unwrap();
1007 assert!(created.runtime && created.managed);
1008 assert_eq!(created.store.as_deref(), Some("shared"));
1009 assert!(workflows.contains("notes"), "clones see runtime datasets");
1010
1011 ingest(&api, "notes", "n1", "Runtime datasets work like any other dataset.").await;
1012 ingest(&api, "static", "s1", "The static dataset shares the store.").await;
1013 let hits = workflows.retrieve(&RetrievalRequest::new("runtime datasets", vec!["notes".into()])).await.unwrap();
1014 assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("n1"));
1015
1016 assert_eq!(api.runtime_datasets(), vec![spec("notes")]);
1017 assert_eq!(api.remove_dataset("notes").await.unwrap(), 1);
1018 assert!(!workflows.contains("notes"));
1019 assert_eq!(api.list_documents("static", 10, None).await.unwrap().len(), 1, "other datasets keep their data");
1020 assert_eq!(api.remove_dataset("static").await.unwrap_err().kind(), "unsupported");
1021 assert_eq!(api.remove_dataset("missing").await.unwrap_err().kind(), "not_found");
1022 }
1023
1024 #[test]
1025 fn runtime_dataset_specs_are_validated() {
1026 let kb = config_kb();
1027 let error = |spec: DatasetSpec| kb.create_dataset(spec).unwrap_err().to_string();
1028 assert!(error(spec("static")).contains("already exists"));
1029 assert!(error(spec("bad/id")).contains("invalid dataset id"));
1030 assert!(error(spec("")).contains("invalid dataset id"));
1031 assert!(error(DatasetSpec { store: "nope".into(), ..spec("a") }).contains("unknown store `nope`"));
1032 assert!(error(DatasetSpec { embedder: "nope".into(), ..spec("b") }).contains("unknown embedder `nope`"));
1033 assert!(error(DatasetSpec { reranker: Some("nope".into()), ..spec("c") }).contains("unknown reranker"));
1034 let bad_chunking = ChunkerConfig { max_chars: 100, overlap: 0, markdown: true, parent_max_chars: Some(50) };
1035 assert!(error(DatasetSpec { chunking: Some(bad_chunking), ..spec("d") }).contains("parent_max_chars"));
1036 assert_eq!(kb.datasets().len(), 1, "failed creations leave nothing behind");
1037 }
1038
1039 #[tokio::test]
1040 async fn external_and_unknown_datasets_are_not_managed() {
1041 let store_kb = kb_with_two_datasets();
1042 let retriever = store_kb.entry("pets").unwrap().retriever;
1043 let kb = KnowledgeBase::new().with_retriever("remote", retriever);
1044 assert!(!kb.is_managed("remote"));
1045 assert_eq!(kb.list_documents("remote", 10, None).await.unwrap_err().kind(), "unsupported");
1046 assert_eq!(kb.delete_document("remote", "x").await.unwrap_err().kind(), "unsupported");
1047 assert_eq!(kb.list_documents("missing", 10, None).await.unwrap_err().kind(), "not_found");
1048 }
1049
1050 #[tokio::test]
1051 async fn multi_dataset_merge_and_single_mode() {
1052 let kb = kb_with_two_datasets();
1053 ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
1054 ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
1055 ingest(&kb, "space", "mars", "Mars is the red planet in our solar system.").await;
1056
1057 let req = RetrievalRequest::new("which pets purr", vec!["pets".into(), "space".into()]).top_k(2);
1058 let hits = kb.retrieve(&req).await.unwrap();
1059 assert_eq!(hits.len(), 2);
1060 assert_eq!(hits[0].document_id, "cats");
1061
1062 let single = kb.retrieve_single(&req).await.unwrap();
1063 assert!(single.iter().all(|h| h.dataset_id == "pets"));
1064
1065 let unknown = RetrievalRequest::new("x", vec!["nope".into()]);
1066 assert!(matches!(kb.retrieve(&unknown).await, Err(RagError::NotFound(_))));
1067 }
1068
1069 #[test]
1070 fn config_rejects_inline_secrets_and_resolves_env() {
1071 let cfg = KnowledgeConfig::from_json(
1072 r#"{"embedders":{"e":{"type":"hash","dimensions":16}},
1073 "stores":{"m":{"type":"memory"}},
1074 "datasets":[{"id":"a","store":"m"}]}"#,
1075 )
1076 .unwrap();
1077 let kb = cfg.build_with_env(&|_| None).unwrap();
1078 assert!(kb.contains("a"));
1079
1080 let bad = KnowledgeConfig::from_json(
1081 r#"{"rerankers":{"r":{"type":"cohere","base_url":"http://x","model":"m","api_key":"inline"}}}"#,
1082 )
1083 .unwrap();
1084 let err = bad.build_with_env(&|_| None).unwrap_err();
1085 assert!(err.to_string().contains("inline") || err.to_string().contains("unsupported"), "{err}");
1086
1087 let missing = KnowledgeConfig::from_json(r#"{"datasets":[{"id":"a","store":"nope"}]}"#).unwrap();
1088 assert!(missing.build_with_env(&|_| None).is_err());
1089 }
1090
1091 #[cfg(feature = "http")]
1092 #[test]
1093 fn config_env_secret_lookup() {
1094 let cfg = KnowledgeConfig::from_json(
1095 r#"{"embedders":{"e":{"type":"openai","base_url":"http://127.0.0.1:9","model":"m","api_key_env":"MY_KEY"}}}"#,
1096 )
1097 .unwrap();
1098 assert!(cfg.build_with_env(&|_| None).unwrap_err().to_string().contains("MY_KEY"));
1099 assert!(cfg.build_with_env(&|n| (n == "MY_KEY").then(|| "secret".to_owned())).is_ok());
1100 }
1101
1102 #[cfg(feature = "yaml")]
1103 #[test]
1104 fn yaml_config() {
1105 let cfg = KnowledgeConfig::from_yaml(
1106 "embedders:\n e: { type: hash }\nstores:\n m: { type: memory }\ndatasets:\n - { id: kb, store: m, embedder: e }\n",
1107 )
1108 .unwrap();
1109 assert_eq!(cfg.datasets.len(), 1);
1110 assert!(cfg.build_with_env(&|_| None).is_ok());
1111 }
1112
1113 #[cfg(feature = "sqlite")]
1114 #[tokio::test]
1115 async fn sqlite_store_from_config_persists_between_builds() {
1116 let dir = std::env::temp_dir().join(format!("graphrt-rag-kb-sqlite-{}", std::process::id()));
1117 let _ = std::fs::remove_dir_all(&dir);
1118 std::fs::create_dir_all(&dir).unwrap();
1119 let path = dir.join("kb.db");
1120 let config = KnowledgeConfig::from_json(
1121 &serde_json::json!({
1122 "embedders": { "e": { "type": "hash", "dimensions": 64 } },
1123 "stores": { "local": { "type": "sqlite", "path": path } },
1124 "datasets": [{ "id": "faq", "store": "local", "embedder": "e" }]
1125 })
1126 .to_string(),
1127 )
1128 .unwrap();
1129
1130 let first = config.build_with_env(&|_| None).unwrap();
1131 ingest(&first, "faq", "returns", "Items can be returned within 30 days.").await;
1132 drop(first);
1133
1134 let second = config.build_with_env(&|_| None).unwrap();
1135 let hits = second.retrieve(&RetrievalRequest::new("returned within days", vec!["faq".into()])).await.unwrap();
1136 assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("returns"));
1137 let _ = std::fs::remove_dir_all(&dir);
1138 }
1139
1140 #[cfg(feature = "local-embed")]
1141 #[test]
1142 fn local_embedder_from_config_is_lazy_and_strict() {
1143 let build = |embedder: serde_json::Value| {
1144 KnowledgeConfig::from_json(&serde_json::json!({ "embedders": { "e": embedder } }).to_string())
1145 .unwrap()
1146 .build_with_env(&|_| None)
1147 };
1148 assert!(build(serde_json::json!({ "type": "local" })).is_ok());
1150 assert!(build(serde_json::json!({ "type": "local", "path": "/not/loaded/yet" })).is_ok());
1151 let typo = build(serde_json::json!({ "type": "local", "modle": "x/y" })).unwrap_err();
1152 assert!(typo.to_string().contains("modle"), "{typo}");
1153 let traversal = build(serde_json::json!({ "type": "local", "model": "../../etc" })).unwrap_err();
1154 assert!(traversal.to_string().contains("`model`"), "{traversal}");
1155 }
1156
1157 #[cfg(feature = "sqlite")]
1158 #[test]
1159 fn sqlite_store_requires_a_path() {
1160 let config = KnowledgeConfig::from_json(r#"{"stores":{"local":{"type":"sqlite"}}}"#).unwrap();
1161 let error = config.build_with_env(&|_| None).unwrap_err();
1162 assert!(error.to_string().contains("`path` (the database file) is required"), "{error}");
1163 }
1164}