1use std::collections::{HashMap, HashSet};
4use std::sync::Arc;
5
6use crate::chunker::{ChunkerConfig, chunk_text};
7use crate::error::RagError;
8use crate::types::{Embedder, Extractor, Metadata, StoredChunk, VectorStore, check_vector, content_hash};
9
10#[derive(Clone, Debug, Default, PartialEq)]
12pub struct Document {
13 pub dataset_id: String,
14 pub document_id: String,
15 pub title: String,
16 pub text: String,
17 pub metadata: Metadata,
19}
20
21impl Document {
22 pub fn new(dataset_id: impl Into<String>, document_id: impl Into<String>, text: impl Into<String>) -> Self {
23 Self {
24 dataset_id: dataset_id.into(),
25 document_id: document_id.into(),
26 title: String::new(),
27 text: text.into(),
28 metadata: Metadata::new(),
29 }
30 }
31
32 pub fn title(mut self, title: impl Into<String>) -> Self {
33 self.title = title.into();
34 self
35 }
36
37 pub fn metadata(mut self, metadata: Metadata) -> Self {
38 self.metadata = metadata;
39 self
40 }
41}
42
43#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
45pub struct IngestReport {
46 pub stored: usize,
48 pub skipped: usize,
51 pub removed: usize,
53}
54
55#[derive(Clone, Debug)]
66pub struct Ingestor {
67 store: Arc<dyn VectorStore>,
68 embedder: Arc<dyn Embedder>,
69 chunker: ChunkerConfig,
70 batch_size: usize,
71 replace_existing: bool,
72}
73
74impl Ingestor {
75 pub fn new(store: Arc<dyn VectorStore>, embedder: Arc<dyn Embedder>) -> Self {
76 Self { store, embedder, chunker: ChunkerConfig::default(), batch_size: 32, replace_existing: true }
77 }
78
79 pub fn chunker(mut self, config: ChunkerConfig) -> Self {
80 self.chunker = config;
81 self
82 }
83
84 pub fn batch_size(mut self, size: usize) -> Self {
86 self.batch_size = size.max(1);
87 self
88 }
89
90 pub fn replace_existing(mut self, replace: bool) -> Self {
93 self.replace_existing = replace;
94 self
95 }
96
97 pub async fn ingest(&self, doc: &Document) -> Result<IngestReport, RagError> {
99 if doc.dataset_id.is_empty() || doc.document_id.is_empty() {
100 return Err(RagError::Input("document needs a dataset_id and a document_id".into()));
101 }
102 let existing: HashMap<String, StoredChunk> = self
103 .store
104 .document_chunks(&doc.dataset_id, &doc.document_id)
105 .await?
106 .into_iter()
107 .map(|c| (c.id.clone(), c))
108 .collect();
109
110 let mut report = IngestReport::default();
111 let mut seen: HashSet<String> = HashSet::new();
112 let mut fresh: Vec<StoredChunk> = Vec::new();
113 let mut refreshed: Vec<StoredChunk> = Vec::new();
114 for chunk in chunk_text(&doc.text, &self.chunker) {
115 let hash = content_hash(&doc.dataset_id, &chunk.text);
116 let id = format!("{}:{}:{hash}", doc.dataset_id, doc.document_id);
117 if !seen.insert(id.clone()) {
118 report.skipped += 1;
119 continue;
120 }
121 let mut metadata = doc.metadata.clone();
122 metadata.insert("chunk_index".into(), serde_json::Value::from(chunk.index));
123 if let Some(h) = &chunk.heading {
124 metadata.insert("heading".into(), serde_json::Value::String(h.clone()));
125 }
126 let mut stored = StoredChunk {
127 id,
128 dataset_id: doc.dataset_id.clone(),
129 document_id: doc.document_id.clone(),
130 title: doc.title.clone(),
131 text: chunk.text,
132 metadata,
133 content_hash: hash,
134 vector: Vec::new(),
135 };
136 match existing.get(&stored.id).filter(|old| !old.vector.is_empty() && old.text == stored.text) {
137 Some(old) => {
138 report.skipped += 1;
139 if old.title != stored.title || old.metadata != stored.metadata {
140 stored.vector.clone_from(&old.vector);
141 refreshed.push(stored);
142 }
143 }
144 None => fresh.push(stored),
145 }
146 }
147
148 for batch in fresh.chunks_mut(self.batch_size) {
150 let texts: Vec<String> = batch.iter().map(|c| c.text.clone()).collect();
151 let vectors = self.embedder.embed_documents(&texts).await?;
152 if vectors.len() != batch.len() {
153 return Err(RagError::Decode(format!(
154 "embedder returned {} vectors for {} texts",
155 vectors.len(),
156 batch.len()
157 )));
158 }
159 for (chunk, vector) in batch.iter_mut().zip(vectors) {
160 check_vector(&vector, &format!("embedding of chunk `{}`", chunk.id))?;
161 chunk.vector = vector;
162 }
163 }
164 report.stored = fresh.len();
165
166 fresh.append(&mut refreshed);
168 if !fresh.is_empty() {
169 self.store.upsert(fresh).await?;
170 }
171
172 if self.replace_existing {
174 let stale: Vec<String> = existing.keys().filter(|id| !seen.contains(*id)).cloned().collect();
175 if !stale.is_empty() {
176 report.removed = self.store.delete_chunks(&doc.dataset_id, &stale).await?;
177 }
178 }
179 Ok(report)
180 }
181
182 pub async fn ingest_bytes(
184 &self,
185 doc: Document,
186 bytes: &[u8],
187 mime_type: Option<&str>,
188 file_name: &str,
189 extractor: &dyn Extractor,
190 ) -> Result<IngestReport, RagError> {
191 let text = extractor.extract(bytes, mime_type, file_name)?;
192 self.ingest(&Document { text, ..doc }).await
193 }
194}