use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use crate::chunker::{ChunkerConfig, chunk_text};
use crate::error::RagError;
use crate::types::{Embedder, Extractor, Metadata, StoredChunk, VectorStore, check_vector, content_hash};
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Document {
pub dataset_id: String,
pub document_id: String,
pub title: String,
pub text: String,
pub metadata: Metadata,
}
impl Document {
pub fn new(dataset_id: impl Into<String>, document_id: impl Into<String>, text: impl Into<String>) -> Self {
Self {
dataset_id: dataset_id.into(),
document_id: document_id.into(),
title: String::new(),
text: text.into(),
metadata: Metadata::new(),
}
}
pub fn title(mut self, title: impl Into<String>) -> Self {
self.title = title.into();
self
}
pub fn metadata(mut self, metadata: Metadata) -> Self {
self.metadata = metadata;
self
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct IngestReport {
pub stored: usize,
pub skipped: usize,
pub removed: usize,
}
#[derive(Clone, Debug)]
pub struct Ingestor {
store: Arc<dyn VectorStore>,
embedder: Arc<dyn Embedder>,
chunker: ChunkerConfig,
batch_size: usize,
replace_existing: bool,
}
impl Ingestor {
pub fn new(store: Arc<dyn VectorStore>, embedder: Arc<dyn Embedder>) -> Self {
Self { store, embedder, chunker: ChunkerConfig::default(), batch_size: 32, replace_existing: true }
}
pub fn chunker(mut self, config: ChunkerConfig) -> Self {
self.chunker = config;
self
}
pub fn batch_size(mut self, size: usize) -> Self {
self.batch_size = size.max(1);
self
}
pub fn replace_existing(mut self, replace: bool) -> Self {
self.replace_existing = replace;
self
}
pub async fn ingest(&self, doc: &Document) -> Result<IngestReport, RagError> {
if doc.dataset_id.is_empty() || doc.document_id.is_empty() {
return Err(RagError::Input("document needs a dataset_id and a document_id".into()));
}
let existing: HashMap<String, StoredChunk> = self
.store
.document_chunks(&doc.dataset_id, &doc.document_id)
.await?
.into_iter()
.map(|c| (c.id.clone(), c))
.collect();
let mut report = IngestReport::default();
let mut seen: HashSet<String> = HashSet::new();
let mut fresh: Vec<StoredChunk> = Vec::new();
let mut refreshed: Vec<StoredChunk> = Vec::new();
for chunk in chunk_text(&doc.text, &self.chunker) {
let hash = content_hash(&doc.dataset_id, &chunk.text);
let id = format!("{}:{}:{hash}", doc.dataset_id, doc.document_id);
if !seen.insert(id.clone()) {
report.skipped += 1;
continue;
}
let mut metadata = doc.metadata.clone();
metadata.insert("chunk_index".into(), serde_json::Value::from(chunk.index));
if let Some(h) = &chunk.heading {
metadata.insert("heading".into(), serde_json::Value::String(h.clone()));
}
let mut stored = StoredChunk {
id,
dataset_id: doc.dataset_id.clone(),
document_id: doc.document_id.clone(),
title: doc.title.clone(),
text: chunk.text,
metadata,
content_hash: hash,
vector: Vec::new(),
};
match existing.get(&stored.id).filter(|old| !old.vector.is_empty() && old.text == stored.text) {
Some(old) => {
report.skipped += 1;
if old.title != stored.title || old.metadata != stored.metadata {
stored.vector.clone_from(&old.vector);
refreshed.push(stored);
}
}
None => fresh.push(stored),
}
}
for batch in fresh.chunks_mut(self.batch_size) {
let texts: Vec<String> = batch.iter().map(|c| c.text.clone()).collect();
let vectors = self.embedder.embed_documents(&texts).await?;
if vectors.len() != batch.len() {
return Err(RagError::Decode(format!(
"embedder returned {} vectors for {} texts",
vectors.len(),
batch.len()
)));
}
for (chunk, vector) in batch.iter_mut().zip(vectors) {
check_vector(&vector, &format!("embedding of chunk `{}`", chunk.id))?;
chunk.vector = vector;
}
}
report.stored = fresh.len();
fresh.append(&mut refreshed);
if !fresh.is_empty() {
self.store.upsert(fresh).await?;
}
if self.replace_existing {
let stale: Vec<String> = existing.keys().filter(|id| !seen.contains(*id)).cloned().collect();
if !stale.is_empty() {
report.removed = self.store.delete_chunks(&doc.dataset_id, &stale).await?;
}
}
Ok(report)
}
pub async fn ingest_bytes(
&self,
doc: Document,
bytes: &[u8],
mime_type: Option<&str>,
file_name: &str,
extractor: &dyn Extractor,
) -> Result<IngestReport, RagError> {
let text = extractor.extract(bytes, mime_type, file_name)?;
self.ingest(&Document { text, ..doc }).await
}
}