Skip to main content

graphrt_rag/
ingest.rs

1//! Ingestion: document -> chunks -> embeddings -> store.
2
3use std::collections::{HashMap, HashSet};
4use std::sync::Arc;
5
6use crate::chunker::{ChunkerConfig, chunk_text};
7use crate::error::RagError;
8use crate::types::{Embedder, Extractor, Metadata, StoredChunk, VectorStore, check_vector, content_hash};
9
10/// A document to ingest.
11#[derive(Clone, Debug, Default, PartialEq)]
12pub struct Document {
13    pub dataset_id: String,
14    pub document_id: String,
15    pub title: String,
16    pub text: String,
17    /// Copied onto every chunk (plus `heading` and `chunk_index`).
18    pub metadata: Metadata,
19}
20
21impl Document {
22    pub fn new(dataset_id: impl Into<String>, document_id: impl Into<String>, text: impl Into<String>) -> Self {
23        Self {
24            dataset_id: dataset_id.into(),
25            document_id: document_id.into(),
26            title: String::new(),
27            text: text.into(),
28            metadata: Metadata::new(),
29        }
30    }
31
32    pub fn title(mut self, title: impl Into<String>) -> Self {
33        self.title = title.into();
34        self
35    }
36
37    pub fn metadata(mut self, metadata: Metadata) -> Self {
38        self.metadata = metadata;
39        self
40    }
41}
42
43/// Outcome of an ingestion.
44#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
45pub struct IngestReport {
46    /// New or changed chunks that were embedded and stored.
47    pub stored: usize,
48    /// Chunks already present with the same content (their vectors are reused,
49    /// metadata is refreshed), plus exact duplicates within the document.
50    pub skipped: usize,
51    /// Chunks of an earlier version of the document that were removed.
52    pub removed: usize,
53}
54
55/// Chunks, embeds (in batches) and stores documents.
56///
57/// Chunk ids are content-addressed per document (`dataset:document:hash`), so
58/// re-ingesting a changed document embeds only new text, keeps unchanged chunks
59/// (refreshing their position and metadata), and — with
60/// [`Ingestor::replace_existing`] — removes chunks the new version no longer
61/// contains. The swap is ordered so a failure never loses the previous version:
62/// everything is embedded first, then upserted, and only then are stale chunks
63/// deleted. Identical text in two different documents is stored once per
64/// document, so deleting one document never removes content from another.
65#[derive(Clone, Debug)]
66pub struct Ingestor {
67    store: Arc<dyn VectorStore>,
68    embedder: Arc<dyn Embedder>,
69    chunker: ChunkerConfig,
70    batch_size: usize,
71    replace_existing: bool,
72}
73
74impl Ingestor {
75    pub fn new(store: Arc<dyn VectorStore>, embedder: Arc<dyn Embedder>) -> Self {
76        Self { store, embedder, chunker: ChunkerConfig::default(), batch_size: 32, replace_existing: true }
77    }
78
79    pub fn chunker(mut self, config: ChunkerConfig) -> Self {
80        self.chunker = config;
81        self
82    }
83
84    /// Texts per embedding request (default 32).
85    pub fn batch_size(mut self, size: usize) -> Self {
86        self.batch_size = size.max(1);
87        self
88    }
89
90    /// Remove chunks of the previous version that the new version no longer
91    /// contains (default `true`). With `false`, ingestion only adds chunks.
92    pub fn replace_existing(mut self, replace: bool) -> Self {
93        self.replace_existing = replace;
94        self
95    }
96
97    /// Ingest one text document.
98    pub async fn ingest(&self, doc: &Document) -> Result<IngestReport, RagError> {
99        if doc.dataset_id.is_empty() || doc.document_id.is_empty() {
100            return Err(RagError::Input("document needs a dataset_id and a document_id".into()));
101        }
102        let existing: HashMap<String, StoredChunk> = self
103            .store
104            .document_chunks(&doc.dataset_id, &doc.document_id)
105            .await?
106            .into_iter()
107            .map(|c| (c.id.clone(), c))
108            .collect();
109
110        let mut report = IngestReport::default();
111        let mut seen: HashSet<String> = HashSet::new();
112        let mut fresh: Vec<StoredChunk> = Vec::new();
113        let mut refreshed: Vec<StoredChunk> = Vec::new();
114        for chunk in chunk_text(&doc.text, &self.chunker) {
115            let hash = content_hash(&doc.dataset_id, &chunk.text);
116            let id = format!("{}:{}:{hash}", doc.dataset_id, doc.document_id);
117            if !seen.insert(id.clone()) {
118                report.skipped += 1;
119                continue;
120            }
121            let mut metadata = doc.metadata.clone();
122            metadata.insert("chunk_index".into(), serde_json::Value::from(chunk.index));
123            if let Some(h) = &chunk.heading {
124                metadata.insert("heading".into(), serde_json::Value::String(h.clone()));
125            }
126            let mut stored = StoredChunk {
127                id,
128                dataset_id: doc.dataset_id.clone(),
129                document_id: doc.document_id.clone(),
130                title: doc.title.clone(),
131                text: chunk.text,
132                metadata,
133                content_hash: hash,
134                vector: Vec::new(),
135            };
136            match existing.get(&stored.id).filter(|old| !old.vector.is_empty() && old.text == stored.text) {
137                Some(old) => {
138                    report.skipped += 1;
139                    if old.title != stored.title || old.metadata != stored.metadata {
140                        stored.vector.clone_from(&old.vector);
141                        refreshed.push(stored);
142                    }
143                }
144                None => fresh.push(stored),
145            }
146        }
147
148        // 1. Embed everything new. A failure here changes nothing in the store.
149        for batch in fresh.chunks_mut(self.batch_size) {
150            let texts: Vec<String> = batch.iter().map(|c| c.text.clone()).collect();
151            let vectors = self.embedder.embed_documents(&texts).await?;
152            if vectors.len() != batch.len() {
153                return Err(RagError::Decode(format!(
154                    "embedder returned {} vectors for {} texts",
155                    vectors.len(),
156                    batch.len()
157                )));
158            }
159            for (chunk, vector) in batch.iter_mut().zip(vectors) {
160                check_vector(&vector, &format!("embedding of chunk `{}`", chunk.id))?;
161                chunk.vector = vector;
162            }
163        }
164        report.stored = fresh.len();
165
166        // 2. Store new and refreshed chunks next to the old version.
167        fresh.append(&mut refreshed);
168        if !fresh.is_empty() {
169            self.store.upsert(fresh).await?;
170        }
171
172        // 3. Only now drop what the new version no longer contains.
173        if self.replace_existing {
174            let stale: Vec<String> = existing.keys().filter(|id| !seen.contains(*id)).cloned().collect();
175            if !stale.is_empty() {
176                report.removed = self.store.delete_chunks(&doc.dataset_id, &stale).await?;
177            }
178        }
179        Ok(report)
180    }
181
182    /// Extract text from raw bytes with `extractor`, then ingest it.
183    pub async fn ingest_bytes(
184        &self,
185        doc: Document,
186        bytes: &[u8],
187        mime_type: Option<&str>,
188        file_name: &str,
189        extractor: &dyn Extractor,
190    ) -> Result<IngestReport, RagError> {
191        let text = extractor.extract(bytes, mime_type, file_name)?;
192        self.ingest(&Document { text, ..doc }).await
193    }
194}