use std::fmt;
use futures::future::BoxFuture;
use serde::{Deserialize, Serialize};
use crate::error::RagError;
use crate::filter::MetadataFilter;
pub type Metadata = serde_json::Map<String, serde_json::Value>;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct StoredChunk {
pub id: String,
pub dataset_id: String,
pub document_id: String,
#[serde(default)]
pub title: String,
pub text: String,
#[serde(default)]
pub metadata: Metadata,
#[serde(default)]
pub content_hash: String,
pub vector: Vec<f32>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Hit {
pub id: String,
pub dataset_id: String,
#[serde(default)]
pub document_id: String,
#[serde(default)]
pub title: String,
pub content: String,
pub score: f32,
#[serde(default)]
pub metadata: Metadata,
}
impl Hit {
pub fn from_chunk(chunk: &StoredChunk, score: f32) -> Self {
Self {
id: chunk.id.clone(),
dataset_id: chunk.dataset_id.clone(),
document_id: chunk.document_id.clone(),
title: chunk.title.clone(),
content: chunk.text.clone(),
score,
metadata: chunk.metadata.clone(),
}
}
}
#[derive(Clone, Debug)]
pub struct VectorQuery<'a> {
pub datasets: &'a [String],
pub vector: &'a [f32],
pub top_k: usize,
pub filter: Option<&'a MetadataFilter>,
}
#[derive(Clone, Debug)]
pub struct KeywordQuery<'a> {
pub datasets: &'a [String],
pub text: &'a str,
pub top_k: usize,
pub filter: Option<&'a MetadataFilter>,
}
pub trait Embedder: Send + Sync + fmt::Debug {
fn model(&self) -> &str;
fn dimensions(&self) -> Option<usize> {
None
}
fn embed_documents<'a>(&'a self, texts: &'a [String]) -> BoxFuture<'a, Result<Vec<Vec<f32>>, RagError>>;
fn embed_query<'a>(&'a self, text: &'a str) -> BoxFuture<'a, Result<Vec<f32>, RagError>> {
Box::pin(async move {
let texts = [text.to_owned()];
let mut vectors = self.embed_documents(&texts).await?;
vectors.pop().ok_or_else(|| RagError::Decode("embedder returned no vector".into()))
})
}
}
pub trait VectorStore: Send + Sync + fmt::Debug {
fn upsert(&self, chunks: Vec<StoredChunk>) -> BoxFuture<'_, Result<(), RagError>>;
fn delete_document<'a>(
&'a self,
dataset_id: &'a str,
document_id: &'a str,
) -> BoxFuture<'a, Result<usize, RagError>>;
fn delete_dataset<'a>(&'a self, dataset_id: &'a str) -> BoxFuture<'a, Result<usize, RagError>>;
fn document_chunks<'a>(
&'a self,
dataset_id: &'a str,
document_id: &'a str,
) -> BoxFuture<'a, Result<Vec<StoredChunk>, RagError>>;
fn delete_chunks<'a>(&'a self, dataset_id: &'a str, ids: &'a [String]) -> BoxFuture<'a, Result<usize, RagError>>;
fn search<'a>(&'a self, query: VectorQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>>;
fn supports_keyword(&self) -> bool {
false
}
fn keyword_search<'a>(&'a self, query: KeywordQuery<'a>) -> BoxFuture<'a, Result<Vec<Hit>, RagError>> {
let _ = query;
Box::pin(futures::future::ready(Ok(Vec::new())))
}
fn contains_hash<'a>(
&'a self,
dataset_id: &'a str,
content_hash: &'a str,
) -> BoxFuture<'a, Result<bool, RagError>> {
let _ = (dataset_id, content_hash);
Box::pin(futures::future::ready(Ok(false)))
}
}
pub trait Reranker: Send + Sync + fmt::Debug {
fn rerank<'a>(&'a self, query: &'a str, documents: &'a [String]) -> BoxFuture<'a, Result<Vec<f32>, RagError>>;
}
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
pub struct HybridWeights {
pub vector: f32,
pub keyword: f32,
}
impl Default for HybridWeights {
fn default() -> Self {
Self { vector: 1.0, keyword: 0.0 }
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct RetrievalRequest {
pub query: String,
pub datasets: Vec<String>,
pub top_k: usize,
pub score_threshold: Option<f32>,
pub filter: Option<MetadataFilter>,
pub weights: HybridWeights,
pub rerank: bool,
}
impl RetrievalRequest {
pub fn new(query: impl Into<String>, datasets: Vec<String>) -> Self {
Self {
query: query.into(),
datasets,
top_k: 4,
score_threshold: None,
filter: None,
weights: HybridWeights::default(),
rerank: false,
}
}
pub fn top_k(mut self, top_k: usize) -> Self {
self.top_k = top_k;
self
}
pub fn threshold(mut self, threshold: Option<f32>) -> Self {
self.score_threshold = threshold;
self
}
pub fn filter(mut self, filter: Option<MetadataFilter>) -> Self {
self.filter = filter;
self
}
pub fn weights(mut self, weights: HybridWeights) -> Self {
self.weights = weights;
self
}
pub fn rerank(mut self, rerank: bool) -> Self {
self.rerank = rerank;
self
}
}
pub trait Retriever: Send + Sync + fmt::Debug {
fn retrieve<'a>(&'a self, request: &'a RetrievalRequest) -> BoxFuture<'a, Result<Vec<Hit>, RagError>>;
}
pub trait Extractor: Send + Sync + fmt::Debug {
fn extract(&self, bytes: &[u8], mime_type: Option<&str>, file_name: &str) -> Result<String, RagError>;
}
#[derive(Clone, Copy, Debug, Default)]
pub struct PlainTextExtractor;
impl Extractor for PlainTextExtractor {
fn extract(&self, bytes: &[u8], mime_type: Option<&str>, file_name: &str) -> Result<String, RagError> {
let textual = mime_type.is_none_or(|m| {
m.starts_with("text/")
|| m.contains("json")
|| m.contains("xml")
|| m.contains("markdown")
|| m.contains("csv")
});
if !textual {
return Err(RagError::Input(format!(
"`{file_name}` ({}) is not a text format; register a document extractor",
mime_type.unwrap_or("unknown")
)));
}
String::from_utf8(bytes.to_vec()).map_err(|_| RagError::Input(format!("`{file_name}` is not valid UTF-8")))
}
}
pub fn check_vector(vector: &[f32], what: &str) -> Result<(), RagError> {
if vector.is_empty() {
return Err(RagError::Input(format!("{what} has an empty embedding vector")));
}
if vector.iter().any(|x| !x.is_finite()) {
return Err(RagError::Input(format!("{what} has a non-finite (NaN or infinite) embedding value")));
}
Ok(())
}
pub fn content_hash(dataset_id: &str, text: &str) -> String {
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
for b in dataset_id.bytes().chain(std::iter::once(0)).chain(text.bytes()) {
h ^= u64::from(b);
h = h.wrapping_mul(0x0100_0000_01b3);
}
format!("{h:016x}")
}