1use std::fmt;
4
5use futures::future::BoxFuture;
6use serde::{Deserialize, Serialize};
7
8use crate::error::RagError;
9use crate::filter::MetadataFilter;
10
11pub type Metadata = serde_json::Map<String, serde_json::Value>;
13
14#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
16pub struct StoredChunk {
17 pub id: String,
19 pub dataset_id: String,
20 pub document_id: String,
21 #[serde(default)]
22 pub title: String,
23 pub text: String,
24 #[serde(default)]
25 pub metadata: Metadata,
26 #[serde(default)]
28 pub content_hash: String,
29 pub vector: Vec<f32>,
30}
31
32#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
34pub struct Hit {
35 pub id: String,
36 pub dataset_id: String,
37 #[serde(default)]
38 pub document_id: String,
39 #[serde(default)]
40 pub title: String,
41 pub content: String,
42 pub score: f32,
43 #[serde(default)]
44 pub metadata: Metadata,
45}
46
47impl Hit {
48 pub fn from_chunk(chunk: &StoredChunk, score: f32) -> Self {
50 Self {
51 id: chunk.id.clone(),
52 dataset_id: chunk.dataset_id.clone(),
53 document_id: chunk.document_id.clone(),
54 title: chunk.title.clone(),
55 content: chunk.text.clone(),
56 score,
57 metadata: chunk.metadata.clone(),
58 }
59 }
60}
61
62#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
64pub struct DocumentSummary {
65 pub document_id: String,
66 #[serde(default)]
67 pub title: String,
68 pub chunks: usize,
70 pub characters: usize,
72 pub enabled: bool,
74 #[serde(default)]
76 pub metadata: Metadata,
77}
78
79pub const DISABLED_KEY: &str = "_disabled";
82
83pub fn is_disabled(metadata: &Metadata) -> bool {
85 metadata.get(DISABLED_KEY).and_then(serde_json::Value::as_bool).unwrap_or(false)
86}
87
88pub const PARENT_KEY: &str = "parent_index";
90
91pub(crate) const PER_CHUNK_KEYS: [&str; 4] = ["chunk_index", "heading", DISABLED_KEY, PARENT_KEY];
93
94pub const DEFAULT_DOCUMENT_PAGE: usize = 50;
96pub const MAX_DOCUMENT_PAGE: usize = 500;
97
98#[derive(Clone, Debug)]
100pub struct VectorQuery<'a> {
101 pub datasets: &'a [String],
102 pub vector: &'a [f32],
103 pub top_k: usize,
104 pub filter: Option<&'a MetadataFilter>,
105}
106
107#[derive(Clone, Debug)]
109pub struct KeywordQuery<'a> {
110 pub datasets: &'a [String],
111 pub text: &'a str,
112 pub top_k: usize,
113 pub filter: Option<&'a MetadataFilter>,
114}
115
116pub trait Embedder: Send + Sync + fmt::Debug {
118 fn model(&self) -> &str;
120
121 fn dimensions(&self) -> Option<usize> {
123 None
124 }
125
126 fn embed_documents<'a>(&'a self, texts: &'a [String]) -> BoxFuture<'a, Result<Vec<Vec<f32>>, RagError>>;
128
129 fn embed_query<'a>(&'a self, text: &'a str) -> BoxFuture<'a, Result<Vec<f32>, RagError>> {
132 Box::pin(async move {
133 let texts = [text.to_owned()];
134 let mut vectors = self.embed_documents(&texts).await?;
135 vectors.pop().ok_or_else(|| RagError::Decode("embedder returned no vector".into()))
136 })
137 }
138}
139
140pub trait VectorStore: Send + Sync + fmt::Debug {
142 fn upsert(&self, chunks: Vec<StoredChunk>) -> BoxFuture<'_, Result<(), RagError>>;
144
145 fn delete_document<'a>(
147 &'a self,
148 dataset_id: &'a str,
149 document_id: &'a str,
150 ) -> BoxFuture<'a, Result<usize, RagError>>;
151
152 fn delete_dataset<'a>(&'a self, dataset_id: &'a str) -> BoxFuture<'a, Result<usize, RagError>>;
154
155 fn document_chunks<'a>(
159 &'a self,
160 dataset_id: &'a str,
161 document_id: &'a str,
162 ) -> BoxFuture<'a, Result<Vec<StoredChunk>, RagError>>;
163
164 fn delete_chunks<'a>(&'a self, dataset_id: &'a str, ids: &'a [String]) -> BoxFuture<'a, Result<usize, RagError>>;
166
167 fn list_documents<'a>(
171 &'a self,
172 dataset_id: &'a str,
173 limit: usize,
174 after: Option<&'a str>,
175 ) -> BoxFuture<'a, Result<Vec<DocumentSummary>, RagError>> {
176 let _ = (dataset_id, limit, after);
177 Box::pin(futures::future::ready(Err(RagError::Unsupported("this store cannot list documents".into()))))
178 }
179
180 fn search<'a>(&'a self, query: VectorQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>>;
182
183 fn supports_keyword(&self) -> bool {
185 false
186 }
187
188 fn keyword_search<'a>(&'a self, query: KeywordQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>> {
190 let _ = query;
191 Box::pin(futures::future::ready(Ok(Vec::new())))
192 }
193
194 fn contains_hash<'a>(
198 &'a self,
199 dataset_id: &'a str,
200 content_hash: &'a str,
201 ) -> BoxFuture<'a, Result<bool, RagError>> {
202 let _ = (dataset_id, content_hash);
203 Box::pin(futures::future::ready(Ok(false)))
204 }
205}
206
207pub trait Reranker: Send + Sync + fmt::Debug {
209 fn rerank<'a>(&'a self, query: &'a str, documents: &'a [String]) -> BoxFuture<'a, Result<Vec<f32>, RagError>>;
211}
212
213#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
215pub struct HybridWeights {
216 pub vector: f32,
217 pub keyword: f32,
218}
219
220impl Default for HybridWeights {
221 fn default() -> Self {
222 Self { vector: 1.0, keyword: 0.0 }
223 }
224}
225
226#[derive(Clone, Debug, PartialEq)]
228pub struct RetrievalRequest {
229 pub query: String,
230 pub datasets: Vec<String>,
231 pub top_k: usize,
232 pub score_threshold: Option<f32>,
234 pub filter: Option<MetadataFilter>,
235 pub weights: HybridWeights,
236 pub rerank: bool,
238}
239
240impl RetrievalRequest {
241 pub fn new(query: impl Into<String>, datasets: Vec<String>) -> Self {
242 Self {
243 query: query.into(),
244 datasets,
245 top_k: 4,
246 score_threshold: None,
247 filter: None,
248 weights: HybridWeights::default(),
249 rerank: false,
250 }
251 }
252
253 pub fn top_k(mut self, top_k: usize) -> Self {
254 self.top_k = top_k;
255 self
256 }
257
258 pub fn threshold(mut self, threshold: Option<f32>) -> Self {
259 self.score_threshold = threshold;
260 self
261 }
262
263 pub fn filter(mut self, filter: Option<MetadataFilter>) -> Self {
264 self.filter = filter;
265 self
266 }
267
268 pub fn weights(mut self, weights: HybridWeights) -> Self {
269 self.weights = weights;
270 self
271 }
272
273 pub fn rerank(mut self, rerank: bool) -> Self {
274 self.rerank = rerank;
275 self
276 }
277}
278
279pub trait Retriever: Send + Sync + fmt::Debug {
282 fn retrieve<'a>(&'a self, request: &'a RetrievalRequest) -> BoxFuture<'a, Result<Vec<Hit>, RagError>>;
283}
284
285pub trait Extractor: Send + Sync + fmt::Debug {
287 fn extract(&self, bytes: &[u8], mime_type: Option<&str>, file_name: &str) -> Result<String, RagError>;
288}
289
290#[derive(Clone, Copy, Debug, Default)]
292pub struct PlainTextExtractor;
293
294impl Extractor for PlainTextExtractor {
295 fn extract(&self, bytes: &[u8], mime_type: Option<&str>, file_name: &str) -> Result<String, RagError> {
296 let textual = mime_type.is_none_or(|m| {
297 m.starts_with("text/")
298 || m.contains("json")
299 || m.contains("xml")
300 || m.contains("markdown")
301 || m.contains("csv")
302 });
303 if !textual {
304 return Err(RagError::Input(format!(
305 "`{file_name}` ({}) is not a text format; register a document extractor",
306 mime_type.unwrap_or("unknown")
307 )));
308 }
309 String::from_utf8(bytes.to_vec()).map_err(|_| RagError::Input(format!("`{file_name}` is not valid UTF-8")))
310 }
311}
312
313pub fn check_vector(vector: &[f32], what: &str) -> Result<(), RagError> {
315 if vector.is_empty() {
316 return Err(RagError::Input(format!("{what} has an empty embedding vector")));
317 }
318 if vector.iter().any(|x| !x.is_finite()) {
319 return Err(RagError::Input(format!("{what} has a non-finite (NaN or infinite) embedding value")));
320 }
321 Ok(())
322}
323
324pub fn content_hash(dataset_id: &str, text: &str) -> String {
326 let mut h: u64 = 0xcbf2_9ce4_8422_2325;
327 for b in dataset_id.bytes().chain(std::iter::once(0)).chain(text.bytes()) {
328 h ^= u64::from(b);
329 h = h.wrapping_mul(0x0100_0000_01b3);
330 }
331 format!("{h:016x}")
332}