1use std::collections::{HashMap, HashSet};
4use std::sync::Arc;
5
6use crate::chunker::{ChunkerConfig, chunk_text};
7use crate::error::RagError;
8use crate::types::{
9 DISABLED_KEY, Embedder, Extractor, Metadata, PARENT_KEY, StoredChunk, VectorStore, check_vector, content_hash,
10 is_disabled,
11};
12
13#[derive(Clone, Debug, Default, PartialEq)]
15pub struct Document {
16 pub dataset_id: String,
17 pub document_id: String,
18 pub title: String,
19 pub text: String,
20 pub metadata: Metadata,
22}
23
24impl Document {
25 pub fn new(dataset_id: impl Into<String>, document_id: impl Into<String>, text: impl Into<String>) -> Self {
26 Self {
27 dataset_id: dataset_id.into(),
28 document_id: document_id.into(),
29 title: String::new(),
30 text: text.into(),
31 metadata: Metadata::new(),
32 }
33 }
34
35 pub fn title(mut self, title: impl Into<String>) -> Self {
36 self.title = title.into();
37 self
38 }
39
40 pub fn metadata(mut self, metadata: Metadata) -> Self {
41 self.metadata = metadata;
42 self
43 }
44}
45
46#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
48pub struct IngestReport {
49 pub stored: usize,
51 pub skipped: usize,
54 pub removed: usize,
56}
57
58#[derive(Clone, Debug)]
69pub struct Ingestor {
70 store: Arc<dyn VectorStore>,
71 embedder: Arc<dyn Embedder>,
72 chunker: ChunkerConfig,
73 batch_size: usize,
74 replace_existing: bool,
75}
76
77impl Ingestor {
78 pub fn new(store: Arc<dyn VectorStore>, embedder: Arc<dyn Embedder>) -> Self {
79 Self { store, embedder, chunker: ChunkerConfig::default(), batch_size: 32, replace_existing: true }
80 }
81
82 pub fn chunker(mut self, config: ChunkerConfig) -> Self {
83 self.chunker = config;
84 self
85 }
86
87 pub fn batch_size(mut self, size: usize) -> Self {
89 self.batch_size = size.max(1);
90 self
91 }
92
93 pub fn replace_existing(mut self, replace: bool) -> Self {
96 self.replace_existing = replace;
97 self
98 }
99
100 pub async fn ingest(&self, doc: &Document) -> Result<IngestReport, RagError> {
102 if doc.dataset_id.is_empty() || doc.document_id.is_empty() {
103 return Err(RagError::Input("document needs a dataset_id and a document_id".into()));
104 }
105 let existing: HashMap<String, StoredChunk> = self
106 .store
107 .document_chunks(&doc.dataset_id, &doc.document_id)
108 .await?
109 .into_iter()
110 .map(|c| (c.id.clone(), c))
111 .collect();
112
113 let disabled = existing.values().any(|c| is_disabled(&c.metadata));
115 let mut report = IngestReport::default();
116 let mut seen: HashSet<String> = HashSet::new();
117 let mut fresh: Vec<StoredChunk> = Vec::new();
118 let mut refreshed: Vec<StoredChunk> = Vec::new();
119 for chunk in chunk_text(&doc.text, &self.chunker) {
120 let hash = content_hash(&doc.dataset_id, &chunk.text);
121 let id = format!("{}:{}:{hash}", doc.dataset_id, doc.document_id);
122 if !seen.insert(id.clone()) {
123 report.skipped += 1;
124 continue;
125 }
126 let mut metadata = doc.metadata.clone();
127 metadata.insert("chunk_index".into(), serde_json::Value::from(chunk.index));
128 if let Some(h) = &chunk.heading {
129 metadata.insert("heading".into(), serde_json::Value::String(h.clone()));
130 }
131 if let Some(parent) = chunk.parent {
132 metadata.insert(PARENT_KEY.into(), serde_json::Value::from(parent));
133 }
134 if disabled {
135 metadata.insert(DISABLED_KEY.into(), serde_json::Value::Bool(true));
136 }
137 let mut stored = StoredChunk {
138 id,
139 dataset_id: doc.dataset_id.clone(),
140 document_id: doc.document_id.clone(),
141 title: doc.title.clone(),
142 text: chunk.text,
143 metadata,
144 content_hash: hash,
145 vector: Vec::new(),
146 };
147 match existing.get(&stored.id).filter(|old| !old.vector.is_empty() && old.text == stored.text) {
148 Some(old) => {
149 report.skipped += 1;
150 if old.title != stored.title || old.metadata != stored.metadata {
151 stored.vector.clone_from(&old.vector);
152 refreshed.push(stored);
153 }
154 }
155 None => fresh.push(stored),
156 }
157 }
158
159 for batch in fresh.chunks_mut(self.batch_size) {
161 let texts: Vec<String> = batch.iter().map(|c| c.text.clone()).collect();
162 let vectors = self.embedder.embed_documents(&texts).await?;
163 if vectors.len() != batch.len() {
164 return Err(RagError::Decode(format!(
165 "embedder returned {} vectors for {} texts",
166 vectors.len(),
167 batch.len()
168 )));
169 }
170 for (chunk, vector) in batch.iter_mut().zip(vectors) {
171 check_vector(&vector, &format!("embedding of chunk `{}`", chunk.id))?;
172 chunk.vector = vector;
173 }
174 }
175 report.stored = fresh.len();
176
177 fresh.append(&mut refreshed);
179 if !fresh.is_empty() {
180 self.store.upsert(fresh).await?;
181 }
182
183 if self.replace_existing {
185 let stale: Vec<String> = existing.keys().filter(|id| !seen.contains(*id)).cloned().collect();
186 if !stale.is_empty() {
187 report.removed = self.store.delete_chunks(&doc.dataset_id, &stale).await?;
188 }
189 }
190 Ok(report)
191 }
192
193 pub async fn ingest_bytes(
195 &self,
196 doc: Document,
197 bytes: &[u8],
198 mime_type: Option<&str>,
199 file_name: &str,
200 extractor: &dyn Extractor,
201 ) -> Result<IngestReport, RagError> {
202 let text = extractor.extract(bytes, mime_type, file_name)?;
203 self.ingest(&Document { text, ..doc }).await
204 }
205}