1use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
18
19use indexmap::IndexMap;
20use serde::{Deserialize, Serialize};
21
22use crate::chunker::ChunkerConfig;
23use crate::error::RagError;
24use crate::ingest::Ingestor;
25use crate::pipeline::{StoreRetriever, finalize, rerank};
26use crate::types::{
27 DocumentSummary, Embedder, Hit, MAX_DOCUMENT_PAGE, Reranker, RetrievalRequest, Retriever, StoredChunk, VectorStore,
28};
29
30#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
32pub struct DatasetInfo {
33 pub id: String,
34 #[serde(default)]
35 pub name: String,
36 #[serde(default)]
37 pub description: String,
38}
39
40#[derive(Clone, Debug)]
41struct DatasetEntry {
42 info: DatasetInfo,
43 retriever: Arc<dyn Retriever>,
44 store: Option<(Arc<dyn VectorStore>, Arc<dyn Embedder>)>,
45 chunking: ChunkerConfig,
47 names: Option<(String, String)>,
49 runtime: Option<DatasetSpec>,
51}
52
53#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
56#[serde(deny_unknown_fields)]
57pub struct DatasetSpec {
58 pub id: String,
59 #[serde(default)]
60 pub name: String,
61 #[serde(default)]
62 pub description: String,
63 pub store: String,
65 pub embedder: String,
67 #[serde(default)]
69 pub reranker: Option<String>,
70 #[serde(default)]
71 pub chunking: Option<ChunkerConfig>,
72}
73
74#[derive(Clone, Debug, PartialEq, Serialize)]
76pub struct DatasetDetails {
77 #[serde(flatten)]
78 pub info: DatasetInfo,
79 pub managed: bool,
81 pub runtime: bool,
83 pub store: Option<String>,
84 pub embedder: Option<String>,
85 pub chunking: Option<ChunkerConfig>,
86}
87
88#[derive(Clone, Debug, Default, PartialEq, Serialize)]
90pub struct ResourceNames {
91 pub stores: Vec<String>,
92 pub embedders: Vec<String>,
93 pub rerankers: Vec<String>,
94}
95
96#[derive(Clone, Debug, Default)]
98struct Resources {
99 stores: IndexMap<String, Arc<dyn VectorStore>>,
100 embedders: IndexMap<String, Arc<dyn Embedder>>,
101 rerankers: IndexMap<String, Arc<dyn Reranker>>,
102}
103
104type Registry = IndexMap<String, DatasetEntry>;
105
106#[derive(Clone, Debug, Default)]
112pub struct KnowledgeBase {
113 datasets: Arc<RwLock<Registry>>,
114 resources: Arc<Resources>,
115 reranker: Option<Arc<dyn Reranker>>,
117}
118
119impl KnowledgeBase {
120 pub fn new() -> Self {
121 Self::default()
122 }
123
124 fn read(&self) -> RwLockReadGuard<'_, Registry> {
126 self.datasets.read().unwrap_or_else(PoisonError::into_inner)
127 }
128
129 fn write(&self) -> RwLockWriteGuard<'_, Registry> {
130 self.datasets.write().unwrap_or_else(PoisonError::into_inner)
131 }
132
133 pub fn with_store_dataset(
135 mut self,
136 id: impl Into<String>,
137 store: Arc<dyn VectorStore>,
138 embedder: Arc<dyn Embedder>,
139 reranker: Option<Arc<dyn Reranker>>,
140 ) -> Self {
141 let id = id.into();
142 self.add_store_dataset(DatasetInfo { id, ..DatasetInfo::default() }, store, embedder, reranker);
143 self
144 }
145
146 pub fn add_store_dataset(
148 &mut self,
149 info: DatasetInfo,
150 store: Arc<dyn VectorStore>,
151 embedder: Arc<dyn Embedder>,
152 reranker: Option<Arc<dyn Reranker>>,
153 ) -> &mut Self {
154 self.insert_store_dataset(info, store, embedder, reranker, ChunkerConfig::default(), None, None);
155 self
156 }
157
158 #[allow(clippy::too_many_arguments)]
159 fn insert_store_dataset(
160 &self,
161 info: DatasetInfo,
162 store: Arc<dyn VectorStore>,
163 embedder: Arc<dyn Embedder>,
164 reranker: Option<Arc<dyn Reranker>>,
165 chunking: ChunkerConfig,
166 names: Option<(String, String)>,
167 runtime: Option<DatasetSpec>,
168 ) {
169 let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
170 let entry = DatasetEntry {
171 info,
172 retriever: Arc::new(retriever),
173 store: Some((store, embedder)),
174 chunking,
175 names,
176 runtime,
177 };
178 self.write().insert(entry.info.id.clone(), entry);
179 }
180
181 pub fn with_retriever(mut self, id: impl Into<String>, retriever: Arc<dyn Retriever>) -> Self {
183 let id = id.into();
184 self.add_retriever(DatasetInfo { id, ..DatasetInfo::default() }, retriever);
185 self
186 }
187
188 pub fn add_retriever(&mut self, info: DatasetInfo, retriever: Arc<dyn Retriever>) -> &mut Self {
190 let entry = DatasetEntry {
191 info,
192 retriever,
193 store: None,
194 chunking: ChunkerConfig::default(),
195 names: None,
196 runtime: None,
197 };
198 self.write().insert(entry.info.id.clone(), entry);
199 self
200 }
201
202 pub fn with_reranker(mut self, reranker: Option<Arc<dyn Reranker>>) -> Self {
204 self.reranker = reranker;
205 self
206 }
207
208 pub fn datasets(&self) -> Vec<DatasetInfo> {
210 self.read().values().map(|d| d.info.clone()).collect()
211 }
212
213 pub fn dataset_details(&self) -> Vec<DatasetDetails> {
215 self.read()
216 .values()
217 .map(|d| DatasetDetails {
218 info: d.info.clone(),
219 managed: d.store.is_some(),
220 runtime: d.runtime.is_some(),
221 store: d.names.as_ref().map(|(store, _)| store.clone()),
222 embedder: d.names.as_ref().map(|(_, embedder)| embedder.clone()),
223 chunking: d.store.as_ref().map(|_| d.chunking.clone()),
224 })
225 .collect()
226 }
227
228 pub fn resource_names(&self) -> ResourceNames {
230 ResourceNames {
231 stores: self.resources.stores.keys().cloned().collect(),
232 embedders: self.resources.embedders.keys().cloned().collect(),
233 rerankers: self.resources.rerankers.keys().cloned().collect(),
234 }
235 }
236
237 pub fn create_dataset(&self, spec: DatasetSpec) -> Result<DatasetDetails, RagError> {
243 validate_dataset_id(&spec.id)?;
244 let lookup = |kind: &str, name: &str, found: bool| {
245 if found {
246 Ok(())
247 } else {
248 Err(RagError::Input(format!("unknown {kind} `{name}` in the knowledge config")))
249 }
250 };
251 let store = self.resources.stores.get(&spec.store).cloned();
252 lookup("store", &spec.store, store.is_some())?;
253 let embedder = self.resources.embedders.get(&spec.embedder).cloned();
254 lookup("embedder", &spec.embedder, embedder.is_some())?;
255 let reranker = match &spec.reranker {
256 Some(name) => {
257 let reranker = self.resources.rerankers.get(name).cloned();
258 lookup("reranker", name, reranker.is_some())?;
259 reranker
260 }
261 None => None,
262 };
263 let chunking = spec.chunking.clone().unwrap_or_default();
264 validate_chunking(&spec.id, &chunking).map_err(|e| RagError::Input(e.to_string()))?;
265 let (Some(store), Some(embedder)) = (store, embedder) else {
266 return Err(RagError::Input("store and embedder are required".into()));
267 };
268 let info = DatasetInfo { id: spec.id.clone(), name: spec.name.clone(), description: spec.description.clone() };
269 {
270 let mut registry = self.write();
272 if registry.contains_key(&spec.id) {
273 return Err(RagError::Input(format!("dataset `{}` already exists", spec.id)));
274 }
275 let retriever = StoreRetriever::new(store.clone(), embedder.clone()).with_reranker(reranker);
276 registry.insert(
277 spec.id.clone(),
278 DatasetEntry {
279 info,
280 retriever: Arc::new(retriever),
281 store: Some((store, embedder)),
282 chunking,
283 names: Some((spec.store.clone(), spec.embedder.clone())),
284 runtime: Some(spec.clone()),
285 },
286 );
287 }
288 self.dataset_details()
289 .into_iter()
290 .find(|d| d.info.id == spec.id)
291 .ok_or_else(|| RagError::Store(format!("dataset `{}` vanished after creation", spec.id)))
292 }
293
294 pub async fn remove_dataset(&self, dataset_id: &str) -> Result<usize, RagError> {
300 let entry = self.entry(dataset_id)?;
301 if entry.runtime.is_none() {
302 return Err(RagError::Unsupported(format!(
303 "dataset `{dataset_id}` is declared in the knowledge config; remove it there"
304 )));
305 }
306 let removed = match &entry.store {
308 Some((store, _)) => store.delete_dataset(dataset_id).await?,
309 None => 0,
310 };
311 self.write().shift_remove(dataset_id);
312 Ok(removed)
313 }
314
315 pub fn runtime_datasets(&self) -> Vec<DatasetSpec> {
317 self.read().values().filter_map(|d| d.runtime.clone()).collect()
318 }
319
320 pub fn contains(&self, dataset_id: &str) -> bool {
321 self.read().contains_key(dataset_id)
322 }
323
324 pub fn ingestor(&self, dataset_id: &str) -> Option<Ingestor> {
327 let registry = self.read();
328 let entry = registry.get(dataset_id)?;
329 let (store, embedder) = entry.store.as_ref()?;
330 Some(Ingestor::new(store.clone(), embedder.clone()).chunker(entry.chunking.clone()))
331 }
332
333 pub fn set_chunking(&mut self, dataset_id: &str, chunking: ChunkerConfig) -> Result<&mut Self, RagError> {
338 validate_chunking(dataset_id, &chunking)?;
339 {
340 let mut registry = self.write();
341 let entry = registry
342 .get_mut(dataset_id)
343 .ok_or_else(|| RagError::NotFound(format!("unknown dataset `{dataset_id}`")))?;
344 entry.chunking = chunking;
345 }
346 Ok(self)
347 }
348
349 pub fn chunking(&self, dataset_id: &str) -> Option<ChunkerConfig> {
351 self.read().get(dataset_id).map(|d| d.chunking.clone())
352 }
353
354 fn entry(&self, id: &str) -> Result<DatasetEntry, RagError> {
356 self.read().get(id).cloned().ok_or_else(|| RagError::NotFound(format!("unknown dataset `{id}`")))
357 }
358
359 pub fn is_managed(&self, dataset_id: &str) -> bool {
362 self.read().get(dataset_id).is_some_and(|d| d.store.is_some())
363 }
364
365 fn managed_store(&self, dataset_id: &str) -> Result<Arc<dyn VectorStore>, RagError> {
370 let entry = self.entry(dataset_id)?;
371 entry
372 .store
373 .map(|(store, _)| store)
374 .ok_or_else(|| RagError::Unsupported(format!("dataset `{dataset_id}` is served by an external retriever")))
375 }
376
377 pub async fn list_documents(
380 &self,
381 dataset_id: &str,
382 limit: usize,
383 after: Option<&str>,
384 ) -> Result<Vec<DocumentSummary>, RagError> {
385 let limit = limit.clamp(1, MAX_DOCUMENT_PAGE);
386 self.managed_store(dataset_id)?.list_documents(dataset_id, limit, after).await
387 }
388
389 pub async fn document_chunks(&self, dataset_id: &str, document_id: &str) -> Result<Vec<StoredChunk>, RagError> {
391 let mut chunks = self.managed_store(dataset_id)?.document_chunks(dataset_id, document_id).await?;
392 chunks.sort_by_key(|c| c.metadata.get("chunk_index").and_then(serde_json::Value::as_u64).unwrap_or(u64::MAX));
393 Ok(chunks)
394 }
395
396 pub async fn set_document_enabled(
402 &self,
403 dataset_id: &str,
404 document_id: &str,
405 enabled: bool,
406 ) -> Result<usize, RagError> {
407 let store = self.managed_store(dataset_id)?;
408 let mut chunks = store.document_chunks(dataset_id, document_id).await?;
409 if chunks.is_empty() {
410 return Err(RagError::NotFound(format!("unknown document `{document_id}` in dataset `{dataset_id}`")));
411 }
412 let count = chunks.len();
413 chunks.retain(|c| crate::types::is_disabled(&c.metadata) == enabled);
414 for chunk in &mut chunks {
415 if enabled {
416 chunk.metadata.remove(crate::types::DISABLED_KEY);
417 } else {
418 chunk.metadata.insert(crate::types::DISABLED_KEY.into(), serde_json::Value::Bool(true));
419 }
420 }
421 if !chunks.is_empty() {
422 store.upsert(chunks).await?;
423 }
424 Ok(count)
425 }
426
427 pub async fn delete_document(&self, dataset_id: &str, document_id: &str) -> Result<usize, RagError> {
429 self.managed_store(dataset_id)?.delete_document(dataset_id, document_id).await
430 }
431
432 async fn query_each(
434 &self,
435 request: &RetrievalRequest,
436 rerank_each: bool,
437 ) -> Result<Vec<(String, Vec<Hit>)>, RagError> {
438 let mut jobs = Vec::with_capacity(request.datasets.len());
439 for id in &request.datasets {
440 let retriever = self.entry(id)?.retriever;
441 let sub = RetrievalRequest {
442 datasets: vec![id.clone()],
443 score_threshold: None,
445 rerank: rerank_each,
446 ..request.clone()
447 };
448 jobs.push(async move { retriever.retrieve(&sub).await.map(|hits| (id.clone(), hits)) });
449 }
450 futures::future::try_join_all(jobs).await
451 }
452
453 pub async fn retrieve(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
460 validate_request(request)?;
461 let global_rerank = request.rerank && self.reranker.is_some() && request.datasets.len() > 1;
462 let per_dataset = self.query_each(request, request.rerank && !global_rerank).await?;
463 let mut merged: Vec<Hit> = per_dataset.into_iter().flat_map(|(_, hits)| hits).collect();
464 if global_rerank && let Some(reranker) = &self.reranker {
465 merged = rerank(reranker.as_ref(), &request.query, merged).await?;
466 }
467 Ok(finalize(merged, request.score_threshold, request.top_k))
468 }
469
470 pub async fn retrieve_single(&self, request: &RetrievalRequest) -> Result<Vec<Hit>, RagError> {
473 validate_request(request)?;
474 let per_dataset = self.query_each(request, request.rerank).await?;
475 let best = per_dataset
476 .into_iter()
477 .filter(|(_, hits)| !hits.is_empty())
478 .max_by(|a, b| top_score(&a.1).total_cmp(&top_score(&b.1)));
479 Ok(best.map(|(_, hits)| finalize(hits, request.score_threshold, request.top_k)).unwrap_or_default())
480 }
481}
482
483fn validate_dataset_id(id: &str) -> Result<(), RagError> {
486 let ok = !id.is_empty()
487 && id.len() <= 64
488 && !id.starts_with('.')
489 && id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'));
490 if ok {
491 Ok(())
492 } else {
493 Err(RagError::Input(format!(
494 "invalid dataset id `{id}`: use 1-64 ASCII letters, digits, `-`, `_` or `.`, not starting with `.`"
495 )))
496 }
497}
498
499fn validate_chunking(dataset_id: &str, chunking: &ChunkerConfig) -> Result<(), RagError> {
500 if chunking.max_chars == 0 {
501 return Err(RagError::Config(format!("dataset `{dataset_id}`: `chunking.max_chars` must be positive")));
502 }
503 if chunking.parent_max_chars.is_some_and(|p| p <= chunking.max_chars) {
504 return Err(RagError::Config(format!(
505 "dataset `{dataset_id}`: `chunking.parent_max_chars` must be larger than `max_chars`"
506 )));
507 }
508 Ok(())
509}
510
511fn validate_request(request: &RetrievalRequest) -> Result<(), RagError> {
513 if request.datasets.is_empty() {
514 return Err(RagError::Input("no dataset ids given".into()));
515 }
516 if let Some(filter) = &request.filter {
517 filter.validate().map_err(|e| RagError::Input(e.to_string()))?;
518 }
519 Ok(())
520}
521
522fn top_score(hits: &[Hit]) -> f32 {
523 hits.iter().map(|h| h.score).fold(f32::MIN, f32::max)
524}
525
526#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
552#[serde(deny_unknown_fields)]
553pub struct KnowledgeConfig {
554 #[serde(default)]
555 pub embedders: IndexMap<String, serde_json::Value>,
556 #[serde(default)]
557 pub rerankers: IndexMap<String, serde_json::Value>,
558 #[serde(default)]
559 pub stores: IndexMap<String, serde_json::Value>,
560 #[serde(default)]
561 pub datasets: Vec<DatasetConfig>,
562 #[serde(default)]
564 pub reranker: Option<String>,
565}
566
567#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
569#[serde(deny_unknown_fields)]
570pub struct DatasetConfig {
571 pub id: String,
572 #[serde(default)]
573 pub name: String,
574 #[serde(default)]
575 pub description: String,
576 #[serde(default)]
578 pub store: Option<String>,
579 #[serde(default)]
581 pub embedder: Option<String>,
582 #[serde(default)]
584 pub reranker: Option<String>,
585 #[serde(default)]
587 pub retriever: Option<serde_json::Value>,
588 #[serde(default)]
591 pub chunking: Option<ChunkerConfig>,
592}
593
594pub type EnvLookup<'a> = &'a dyn Fn(&str) -> Option<String>;
596
597impl KnowledgeConfig {
598 pub fn from_json(text: &str) -> Result<Self, RagError> {
599 serde_json::from_str(text).map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
600 }
601
602 #[cfg(feature = "yaml")]
603 pub fn from_yaml(text: &str) -> Result<Self, RagError> {
604 let options = serde_saphyr::options! { strict_booleans: true };
605 serde_saphyr::from_str_with_options(text, options)
606 .map_err(|e| RagError::Config(format!("invalid knowledge config: {e}")))
607 }
608
609 pub fn build(&self) -> Result<KnowledgeBase, RagError> {
611 self.build_with_env(&|name| std::env::var(name).ok())
612 }
613
614 pub fn build_with_env(&self, env: EnvLookup<'_>) -> Result<KnowledgeBase, RagError> {
616 let mut embedders: IndexMap<&str, Arc<dyn Embedder>> = IndexMap::new();
617 for (name, raw) in &self.embedders {
618 embedders.insert(name, build_embedder(name, raw, env)?);
619 }
620 let mut rerankers: IndexMap<&str, Arc<dyn Reranker>> = IndexMap::new();
621 for (name, raw) in &self.rerankers {
622 rerankers.insert(name, build_reranker(name, raw, env)?);
623 }
624 let mut stores: IndexMap<&str, Arc<dyn VectorStore>> = IndexMap::new();
625 for (name, raw) in &self.stores {
626 stores.insert(name, build_store(name, raw, env)?);
627 }
628
629 let lookup_reranker = |key: &Option<String>| -> Result<Option<Arc<dyn Reranker>>, RagError> {
630 match key {
631 None => Ok(None),
632 Some(k) => rerankers
633 .get(k.as_str())
634 .cloned()
635 .map(Some)
636 .ok_or_else(|| RagError::Config(format!("unknown reranker `{k}`"))),
637 }
638 };
639
640 let mut kb = KnowledgeBase::new().with_reranker(lookup_reranker(&self.reranker)?);
641 for ds in &self.datasets {
642 if ds.id.trim().is_empty() {
643 return Err(RagError::Config("dataset id must not be empty".into()));
644 }
645 if kb.contains(&ds.id) {
646 return Err(RagError::Config(format!("duplicate dataset `{}`", ds.id)));
647 }
648 let info = DatasetInfo { id: ds.id.clone(), name: ds.name.clone(), description: ds.description.clone() };
649 if let Some(raw) = &ds.retriever {
650 kb.add_retriever(info, build_retriever(&ds.id, raw, env)?);
651 continue;
652 }
653 let store_key = ds.store.as_deref().ok_or_else(|| {
654 RagError::Config(format!("dataset `{}` needs either `store` + `embedder` or `retriever`", ds.id))
655 })?;
656 let store = stores
657 .get(store_key)
658 .cloned()
659 .ok_or_else(|| RagError::Config(format!("unknown store `{store_key}`")))?;
660 let embedder_key = match ds.embedder.as_deref() {
661 Some(k) => k,
662 None if embedders.len() == 1 => embedders.keys().next().copied().unwrap_or_default(),
663 None => return Err(RagError::Config(format!("dataset `{}` needs `embedder`", ds.id))),
664 };
665 let embedder = embedders
666 .get(embedder_key)
667 .cloned()
668 .ok_or_else(|| RagError::Config(format!("unknown embedder `{embedder_key}`")))?;
669 let chunking = ds.chunking.clone().unwrap_or_default();
670 validate_chunking(&ds.id, &chunking)?;
671 let names = Some((store_key.to_owned(), embedder_key.to_owned()));
672 kb.insert_store_dataset(info, store, embedder, lookup_reranker(&ds.reranker)?, chunking, names, None);
673 }
674 kb.resources = Arc::new(Resources {
675 stores: stores.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
676 embedders: embedders.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
677 rerankers: rerankers.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(),
678 });
679 Ok(kb)
680 }
681}
682
683#[cfg_attr(not(feature = "http"), allow(dead_code))]
685const HTTP_KEYS: &[&str] = &["type", "api_key", "api_key_env", "api_key_header", "headers", "timeout_ms"];
686
687fn check_keys(name: &str, raw: &serde_json::Value, allowed: &[&str], with_http: bool) -> Result<(), RagError> {
691 let obj = raw.as_object().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
692 for key in obj.keys() {
693 let known = allowed.contains(&key.as_str()) || (with_http && HTTP_KEYS.contains(&key.as_str()));
694 if !known && key != "type" {
695 return Err(RagError::Config(format!("`{name}`: unknown key `{key}`")));
696 }
697 }
698 Ok(())
699}
700
701fn provider_type(name: &str, raw: &serde_json::Value) -> Result<String, RagError> {
702 raw.get("type")
703 .and_then(|t| t.as_str())
704 .map(str::to_owned)
705 .ok_or_else(|| RagError::Config(format!("`{name}` needs a `type`")))
706}
707
708#[cfg_attr(not(feature = "http"), allow(dead_code))]
710fn with_secret(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<serde_json::Value, RagError> {
711 let mut obj = raw.as_object().cloned().ok_or_else(|| RagError::Config(format!("`{name}` must be an object")))?;
712 obj.remove("type");
713 if obj.contains_key("api_key") {
714 return Err(RagError::Config(format!(
715 "`{name}`: inline `api_key` is not allowed; use `api_key_env` with an environment variable name"
716 )));
717 }
718 if let Some(var) = obj.remove("api_key_env") {
719 let var = var.as_str().ok_or_else(|| RagError::Config(format!("`{name}`: `api_key_env` must be a string")))?;
720 let value =
721 env(var).ok_or_else(|| RagError::Config(format!("`{name}`: environment variable `{var}` is not set")))?;
722 obj.insert("api_key".into(), serde_json::Value::String(value));
723 }
724 Ok(serde_json::Value::Object(obj))
725}
726
727#[cfg_attr(not(feature = "http"), allow(dead_code))]
728fn parse<T: serde::de::DeserializeOwned>(name: &str, value: serde_json::Value) -> Result<T, RagError> {
729 serde_json::from_value(value).map_err(|e| RagError::Config(format!("`{name}`: {e}")))
730}
731
732fn build_embedder(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Embedder>, RagError> {
733 let kind = provider_type(name, raw)?;
734 match kind.as_str() {
735 #[cfg(feature = "http")]
736 "openai" | "openai-compatible" | "ollama" => {
737 check_keys(
738 name,
739 raw,
740 &["base_url", "model", "dimensions", "batch_size", "query_prefix", "document_prefix"],
741 true,
742 )?;
743 let mut value = with_secret(name, raw, env)?;
744 if kind == "ollama"
745 && let Some(obj) = value.as_object_mut()
746 {
747 obj.entry("base_url").or_insert_with(|| "http://localhost:11434/v1".into());
748 }
749 let cfg: crate::http::OpenAiEmbedderConfig = parse(name, value)?;
750 Ok(Arc::new(crate::http::OpenAiEmbedder::new(cfg)?))
751 }
752 #[cfg(feature = "local-embed")]
753 "local" => {
754 let mut value = raw.as_object().cloned().unwrap_or_default();
755 value.remove("type");
756 let config: crate::local_embed::LocalEmbedderConfig = parse(name, serde_json::Value::Object(value))?;
757 Ok(Arc::new(crate::local_embed::LocalEmbedder::new(config)?))
758 }
759 "hash" => {
760 check_keys(name, raw, &["dimensions"], false)?;
761 let dims = match raw.get("dimensions") {
762 None => 256,
763 Some(v) => v
764 .as_u64()
765 .and_then(|d| usize::try_from(d).ok())
766 .filter(|d| *d > 0)
767 .ok_or_else(|| RagError::Config(format!("`{name}`: `dimensions` must be a positive integer")))?,
768 };
769 let _ = env;
770 Ok(Arc::new(crate::hash::HashEmbedder::new(dims)))
771 }
772 other => Err(RagError::Config(format!("embedder `{name}`: unsupported type `{other}` in this build"))),
773 }
774}
775
776fn build_reranker(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Reranker>, RagError> {
777 let kind = provider_type(name, raw)?;
778 match kind.as_str() {
779 #[cfg(feature = "http")]
780 "http" | "cohere" | "jina" | "voyage" => {
781 check_keys(name, raw, &["base_url", "model"], true)?;
782 let cfg: crate::http::HttpRerankerConfig = parse(name, with_secret(name, raw, env)?)?;
783 Ok(Arc::new(crate::http::HttpReranker::new(cfg)?))
784 }
785 other => {
786 let _ = env;
787 Err(RagError::Config(format!("reranker `{name}`: unsupported type `{other}` in this build")))
788 }
789 }
790}
791
792fn build_store(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn VectorStore>, RagError> {
793 let kind = provider_type(name, raw)?;
794 match kind.as_str() {
795 "memory" => {
796 check_keys(name, raw, &["path"], false)?;
797 let path = match raw.get("path") {
798 None => None,
799 Some(serde_json::Value::String(p)) => Some(p.as_str()),
800 Some(_) => return Err(RagError::Config(format!("store `{name}`: `path` must be a string"))),
801 };
802 match path {
803 #[cfg(feature = "fs")]
804 Some(path) => Ok(Arc::new(crate::memory::MemoryStore::open(path)?)),
805 #[cfg(not(feature = "fs"))]
806 Some(_) => Err(RagError::Config(format!("store `{name}`: `path` requires the `fs` feature"))),
807 None => Ok(Arc::new(crate::memory::MemoryStore::new())),
808 }
809 }
810 #[cfg(feature = "sqlite")]
811 "sqlite" => {
812 check_keys(name, raw, &["path"], false)?;
813 let path =
814 raw.get("path").and_then(serde_json::Value::as_str).filter(|p| !p.trim().is_empty()).ok_or_else(
815 || RagError::Config(format!("store `{name}`: `path` (the database file) is required")),
816 )?;
817 Ok(Arc::new(crate::sqlite::SqliteStore::open(path)?))
818 }
819 #[cfg(feature = "qdrant")]
820 "qdrant" => {
821 check_keys(name, raw, &["url", "collection", "create_collection"], true)?;
822 let mut value = with_secret(name, raw, env)?;
823 if let Some(obj) = value.as_object_mut() {
824 obj.entry("api_key_header").or_insert_with(|| "api-key".into());
825 }
826 let cfg: crate::qdrant::QdrantConfig = parse(name, value)?;
827 Ok(Arc::new(crate::qdrant::QdrantStore::new(cfg)?))
828 }
829 other => {
830 let _ = env;
831 Err(RagError::Config(format!("store `{name}`: unsupported type `{other}` in this build")))
832 }
833 }
834}
835
836fn build_retriever(name: &str, raw: &serde_json::Value, env: EnvLookup<'_>) -> Result<Arc<dyn Retriever>, RagError> {
837 let kind = provider_type(name, raw)?;
838 match kind.as_str() {
839 #[cfg(feature = "http")]
840 "http" => {
841 check_keys(name, raw, &["url"], true)?;
842 let cfg: crate::http::HttpRetrieverConfig = parse(name, with_secret(name, raw, env)?)?;
843 Ok(Arc::new(crate::http::HttpRetriever::new(cfg)?))
844 }
845 other => {
846 let _ = env;
847 Err(RagError::Config(format!("retriever for `{name}`: unsupported type `{other}` in this build")))
848 }
849 }
850}
851
852#[cfg(test)]
853mod tests {
854 use super::*;
855 use crate::hash::HashEmbedder;
856 use crate::ingest::Document;
857 use crate::memory::MemoryStore;
858
859 fn kb_with_two_datasets() -> KnowledgeBase {
860 let embedder: Arc<dyn Embedder> = Arc::new(HashEmbedder::new(128));
861 KnowledgeBase::new()
862 .with_store_dataset("pets", Arc::new(MemoryStore::new()), embedder.clone(), None)
863 .with_store_dataset("space", Arc::new(MemoryStore::new()), embedder, None)
864 }
865
866 async fn ingest(kb: &KnowledgeBase, ds: &str, id: &str, text: &str) {
867 kb.ingestor(ds).unwrap().ingest(&Document::new(ds, id, text)).await.unwrap();
868 }
869
870 #[tokio::test]
871 async fn documents_can_be_listed_paged_inspected_and_deleted() {
872 let kb = kb_with_two_datasets();
873 let ingestor = kb.ingestor("pets").unwrap();
874 let mut meta = crate::types::Metadata::new();
875 meta.insert("lang".into(), serde_json::json!("en"));
876 let long = "Cats purr. ".repeat(200);
877 ingestor.ingest(&Document::new("pets", "cats", long).title("Cats").metadata(meta)).await.unwrap();
878 ingest(&kb, "pets", "dogs", "Dogs bark.").await;
879 ingest(&kb, "pets", "birds", "Birds sing.").await;
880
881 let first = kb.list_documents("pets", 2, None).await.unwrap();
882 let ids: Vec<&str> = first.iter().map(|d| d.document_id.as_str()).collect();
883 assert_eq!(ids, ["birds", "cats"]);
884 let cats = &first[1];
885 assert_eq!(cats.title, "Cats");
886 assert!(cats.chunks > 1, "long document is split");
887 assert_eq!(cats.metadata.get("lang"), Some(&serde_json::json!("en")));
888 assert!(!cats.metadata.contains_key("chunk_index"), "per-chunk keys are not document metadata");
889 let next = kb.list_documents("pets", 2, Some("cats")).await.unwrap();
890 assert_eq!(next.iter().map(|d| d.document_id.as_str()).collect::<Vec<_>>(), ["dogs"]);
891
892 let chunks = kb.document_chunks("pets", "cats").await.unwrap();
893 assert_eq!(chunks.len(), cats.chunks);
894 let order: Vec<u64> = chunks.iter().map(|c| c.metadata["chunk_index"].as_u64().unwrap()).collect();
895 assert!(order.windows(2).all(|w| w[0] < w[1]), "chunks come in document order: {order:?}");
896
897 assert_eq!(kb.delete_document("pets", "cats").await.unwrap(), cats.chunks);
898 assert_eq!(kb.delete_document("pets", "cats").await.unwrap(), 0);
899 assert_eq!(kb.list_documents("pets", 50, None).await.unwrap().len(), 2);
900 }
901
902 #[tokio::test]
903 async fn disabled_documents_are_kept_but_never_retrieved() {
904 let kb = kb_with_two_datasets();
905 ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
906 ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
907 let query = RetrievalRequest::new("pets that purr", vec!["pets".into()]).top_k(5);
908 let found = |hits: &[Hit]| hits.iter().any(|h| h.document_id == "cats");
909 assert!(found(&kb.retrieve(&query).await.unwrap()));
910
911 assert!(kb.set_document_enabled("pets", "cats", false).await.unwrap() >= 1);
912 assert!(!found(&kb.retrieve(&query).await.unwrap()), "disabled document retrieved");
913 let docs = kb.list_documents("pets", 10, None).await.unwrap();
914 let cats = docs.iter().find(|d| d.document_id == "cats").unwrap();
915 assert!(!cats.enabled);
916 assert!(!cats.metadata.contains_key(crate::types::DISABLED_KEY), "flag is not document metadata");
917
918 ingest(&kb, "pets", "cats", "Cats are small furry pets that purr and nap.").await;
920 assert!(!found(&kb.retrieve(&query).await.unwrap()), "re-ingestion re-enabled the document");
921
922 kb.set_document_enabled("pets", "cats", true).await.unwrap();
923 assert!(found(&kb.retrieve(&query).await.unwrap()));
924 assert_eq!(kb.set_document_enabled("pets", "nope", false).await.unwrap_err().kind(), "not_found");
925 }
926
927 #[tokio::test]
928 async fn parent_child_datasets_return_whole_parents_once() {
929 let mut kb = KnowledgeBase::new().with_store_dataset(
930 "docs",
931 Arc::new(MemoryStore::new()),
932 Arc::new(HashEmbedder::new(256)),
933 None,
934 );
935 let chunking = ChunkerConfig { max_chars: 60, overlap: 0, markdown: false, parent_max_chars: Some(220) };
936 kb.set_chunking("docs", chunking).unwrap();
937 let text = "Leave policy. Every employee gets 24 days of paid annual leave. Unused leave carries over for six months. Sick leave is separate.\n\n\
938 Expense policy. Expenses below 500 USD are approved by your manager. Larger ones need finance approval.";
939 ingest(&kb, "docs", "handbook", text).await;
940 let children = kb.document_chunks("docs", "handbook").await.unwrap();
941 assert!(children.len() > 2, "the document is split into several children");
942
943 let request = RetrievalRequest::new("annual leave days carries over", vec!["docs".into()]).top_k(5);
944 let hits = kb.retrieve(&request).await.unwrap();
945 let leave: Vec<&Hit> = hits.iter().filter(|h| h.content.contains("24 days")).collect();
946 assert_eq!(leave.len(), 1, "children of one parent collapse into one hit: {hits:?}");
947 let parent = &leave[0].content;
948 assert!(parent.contains("Leave policy") && parent.contains("Sick leave"), "whole parent returned: {parent}");
949 assert!(!parent.contains("Expense"), "parents stay separate: {parent}");
950 }
951
952 #[test]
953 fn chunking_config_is_validated() {
954 let build = |chunking: serde_json::Value| {
955 KnowledgeConfig::from_json(
956 &serde_json::json!({
957 "embedders": { "e": { "type": "hash" } },
958 "stores": { "m": { "type": "memory" } },
959 "datasets": [{ "id": "d", "store": "m", "embedder": "e", "chunking": chunking }]
960 })
961 .to_string(),
962 )
963 .unwrap()
964 .build_with_env(&|_| None)
965 };
966 let kb = build(serde_json::json!({ "max_chars": 300, "parent_max_chars": 1200 })).unwrap();
967 assert_eq!(kb.chunking("d").unwrap().parent_max_chars, Some(1200));
968 let error = build(serde_json::json!({ "max_chars": 300, "parent_max_chars": 200 })).unwrap_err();
969 assert!(error.to_string().contains("parent_max_chars"), "{error}");
970 }
971
972 fn config_kb() -> KnowledgeBase {
973 KnowledgeConfig::from_json(
974 &serde_json::json!({
975 "embedders": { "e": { "type": "hash", "dimensions": 64 } },
976 "stores": { "shared": { "type": "memory" } },
977 "datasets": [{ "id": "static", "store": "shared", "embedder": "e" }]
978 })
979 .to_string(),
980 )
981 .unwrap()
982 .build_with_env(&|_| None)
983 .unwrap()
984 }
985
986 fn spec(id: &str) -> DatasetSpec {
987 DatasetSpec {
988 id: id.into(),
989 name: "Runtime".into(),
990 description: String::new(),
991 store: "shared".into(),
992 embedder: "e".into(),
993 reranker: None,
994 chunking: None,
995 }
996 }
997
998 #[tokio::test]
999 async fn runtime_datasets_are_shared_by_clones_and_removable() {
1000 let api = config_kb();
1001 let workflows = api.clone();
1002 assert_eq!(api.resource_names().stores, ["shared"]);
1003
1004 let created = api.create_dataset(spec("notes")).unwrap();
1005 assert!(created.runtime && created.managed);
1006 assert_eq!(created.store.as_deref(), Some("shared"));
1007 assert!(workflows.contains("notes"), "clones see runtime datasets");
1008
1009 ingest(&api, "notes", "n1", "Runtime datasets work like any other dataset.").await;
1010 ingest(&api, "static", "s1", "The static dataset shares the store.").await;
1011 let hits = workflows.retrieve(&RetrievalRequest::new("runtime datasets", vec!["notes".into()])).await.unwrap();
1012 assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("n1"));
1013
1014 assert_eq!(api.runtime_datasets(), vec![spec("notes")]);
1015 assert_eq!(api.remove_dataset("notes").await.unwrap(), 1);
1016 assert!(!workflows.contains("notes"));
1017 assert_eq!(api.list_documents("static", 10, None).await.unwrap().len(), 1, "other datasets keep their data");
1018 assert_eq!(api.remove_dataset("static").await.unwrap_err().kind(), "unsupported");
1019 assert_eq!(api.remove_dataset("missing").await.unwrap_err().kind(), "not_found");
1020 }
1021
1022 #[test]
1023 fn runtime_dataset_specs_are_validated() {
1024 let kb = config_kb();
1025 let error = |spec: DatasetSpec| kb.create_dataset(spec).unwrap_err().to_string();
1026 assert!(error(spec("static")).contains("already exists"));
1027 assert!(error(spec("bad/id")).contains("invalid dataset id"));
1028 assert!(error(spec("")).contains("invalid dataset id"));
1029 assert!(error(DatasetSpec { store: "nope".into(), ..spec("a") }).contains("unknown store `nope`"));
1030 assert!(error(DatasetSpec { embedder: "nope".into(), ..spec("b") }).contains("unknown embedder `nope`"));
1031 assert!(error(DatasetSpec { reranker: Some("nope".into()), ..spec("c") }).contains("unknown reranker"));
1032 let bad_chunking = ChunkerConfig { max_chars: 100, overlap: 0, markdown: true, parent_max_chars: Some(50) };
1033 assert!(error(DatasetSpec { chunking: Some(bad_chunking), ..spec("d") }).contains("parent_max_chars"));
1034 assert_eq!(kb.datasets().len(), 1, "failed creations leave nothing behind");
1035 }
1036
1037 #[tokio::test]
1038 async fn external_and_unknown_datasets_are_not_managed() {
1039 let store_kb = kb_with_two_datasets();
1040 let retriever = store_kb.entry("pets").unwrap().retriever;
1041 let kb = KnowledgeBase::new().with_retriever("remote", retriever);
1042 assert!(!kb.is_managed("remote"));
1043 assert_eq!(kb.list_documents("remote", 10, None).await.unwrap_err().kind(), "unsupported");
1044 assert_eq!(kb.delete_document("remote", "x").await.unwrap_err().kind(), "unsupported");
1045 assert_eq!(kb.list_documents("missing", 10, None).await.unwrap_err().kind(), "not_found");
1046 }
1047
1048 #[tokio::test]
1049 async fn multi_dataset_merge_and_single_mode() {
1050 let kb = kb_with_two_datasets();
1051 ingest(&kb, "pets", "cats", "Cats are small furry pets that purr.").await;
1052 ingest(&kb, "pets", "dogs", "Dogs are loyal pets that bark.").await;
1053 ingest(&kb, "space", "mars", "Mars is the red planet in our solar system.").await;
1054
1055 let req = RetrievalRequest::new("which pets purr", vec!["pets".into(), "space".into()]).top_k(2);
1056 let hits = kb.retrieve(&req).await.unwrap();
1057 assert_eq!(hits.len(), 2);
1058 assert_eq!(hits[0].document_id, "cats");
1059
1060 let single = kb.retrieve_single(&req).await.unwrap();
1061 assert!(single.iter().all(|h| h.dataset_id == "pets"));
1062
1063 let unknown = RetrievalRequest::new("x", vec!["nope".into()]);
1064 assert!(matches!(kb.retrieve(&unknown).await, Err(RagError::NotFound(_))));
1065 }
1066
1067 #[test]
1068 fn config_rejects_inline_secrets_and_resolves_env() {
1069 let cfg = KnowledgeConfig::from_json(
1070 r#"{"embedders":{"e":{"type":"hash","dimensions":16}},
1071 "stores":{"m":{"type":"memory"}},
1072 "datasets":[{"id":"a","store":"m"}]}"#,
1073 )
1074 .unwrap();
1075 let kb = cfg.build_with_env(&|_| None).unwrap();
1076 assert!(kb.contains("a"));
1077
1078 let bad = KnowledgeConfig::from_json(
1079 r#"{"rerankers":{"r":{"type":"cohere","base_url":"http://x","model":"m","api_key":"inline"}}}"#,
1080 )
1081 .unwrap();
1082 let err = bad.build_with_env(&|_| None).unwrap_err();
1083 assert!(err.to_string().contains("inline") || err.to_string().contains("unsupported"), "{err}");
1084
1085 let missing = KnowledgeConfig::from_json(r#"{"datasets":[{"id":"a","store":"nope"}]}"#).unwrap();
1086 assert!(missing.build_with_env(&|_| None).is_err());
1087 }
1088
1089 #[cfg(feature = "http")]
1090 #[test]
1091 fn config_env_secret_lookup() {
1092 let cfg = KnowledgeConfig::from_json(
1093 r#"{"embedders":{"e":{"type":"openai","base_url":"http://127.0.0.1:9","model":"m","api_key_env":"MY_KEY"}}}"#,
1094 )
1095 .unwrap();
1096 assert!(cfg.build_with_env(&|_| None).unwrap_err().to_string().contains("MY_KEY"));
1097 assert!(cfg.build_with_env(&|n| (n == "MY_KEY").then(|| "secret".to_owned())).is_ok());
1098 }
1099
1100 #[cfg(feature = "yaml")]
1101 #[test]
1102 fn yaml_config() {
1103 let cfg = KnowledgeConfig::from_yaml(
1104 "embedders:\n e: { type: hash }\nstores:\n m: { type: memory }\ndatasets:\n - { id: kb, store: m, embedder: e }\n",
1105 )
1106 .unwrap();
1107 assert_eq!(cfg.datasets.len(), 1);
1108 assert!(cfg.build_with_env(&|_| None).is_ok());
1109 }
1110
1111 #[cfg(feature = "sqlite")]
1112 #[tokio::test]
1113 async fn sqlite_store_from_config_persists_between_builds() {
1114 let dir = std::env::temp_dir().join(format!("graphrt-rag-kb-sqlite-{}", std::process::id()));
1115 let _ = std::fs::remove_dir_all(&dir);
1116 std::fs::create_dir_all(&dir).unwrap();
1117 let path = dir.join("kb.db");
1118 let config = KnowledgeConfig::from_json(
1119 &serde_json::json!({
1120 "embedders": { "e": { "type": "hash", "dimensions": 64 } },
1121 "stores": { "local": { "type": "sqlite", "path": path } },
1122 "datasets": [{ "id": "faq", "store": "local", "embedder": "e" }]
1123 })
1124 .to_string(),
1125 )
1126 .unwrap();
1127
1128 let first = config.build_with_env(&|_| None).unwrap();
1129 ingest(&first, "faq", "returns", "Items can be returned within 30 days.").await;
1130 drop(first);
1131
1132 let second = config.build_with_env(&|_| None).unwrap();
1133 let hits = second.retrieve(&RetrievalRequest::new("returned within days", vec!["faq".into()])).await.unwrap();
1134 assert_eq!(hits.first().map(|h| h.document_id.as_str()), Some("returns"));
1135 let _ = std::fs::remove_dir_all(&dir);
1136 }
1137
1138 #[cfg(feature = "local-embed")]
1139 #[test]
1140 fn local_embedder_from_config_is_lazy_and_strict() {
1141 let build = |embedder: serde_json::Value| {
1142 KnowledgeConfig::from_json(&serde_json::json!({ "embedders": { "e": embedder } }).to_string())
1143 .unwrap()
1144 .build_with_env(&|_| None)
1145 };
1146 assert!(build(serde_json::json!({ "type": "local" })).is_ok());
1148 assert!(build(serde_json::json!({ "type": "local", "path": "/not/loaded/yet" })).is_ok());
1149 let typo = build(serde_json::json!({ "type": "local", "modle": "x/y" })).unwrap_err();
1150 assert!(typo.to_string().contains("modle"), "{typo}");
1151 let traversal = build(serde_json::json!({ "type": "local", "model": "../../etc" })).unwrap_err();
1152 assert!(traversal.to_string().contains("`model`"), "{traversal}");
1153 }
1154
1155 #[cfg(feature = "sqlite")]
1156 #[test]
1157 fn sqlite_store_requires_a_path() {
1158 let config = KnowledgeConfig::from_json(r#"{"stores":{"local":{"type":"sqlite"}}}"#).unwrap();
1159 let error = config.build_with_env(&|_| None).unwrap_err();
1160 assert!(error.to_string().contains("`path` (the database file) is required"), "{error}");
1161 }
1162}