Skip to main content

graphrt_rag/
ingest.rs

1//! Ingestion: document -> chunks -> embeddings -> store.
2
3use std::collections::{HashMap, HashSet};
4use std::sync::Arc;
5
6use crate::chunker::{ChunkerConfig, chunk_text};
7use crate::error::RagError;
8use crate::types::{
9    DISABLED_KEY, Embedder, Extractor, Metadata, PARENT_KEY, StoredChunk, VectorStore, check_vector, content_hash,
10    is_disabled,
11};
12
13/// A document to ingest.
14#[derive(Clone, Debug, Default, PartialEq)]
15pub struct Document {
16    pub dataset_id: String,
17    pub document_id: String,
18    pub title: String,
19    pub text: String,
20    /// Copied onto every chunk (plus `heading` and `chunk_index`).
21    pub metadata: Metadata,
22}
23
24impl Document {
25    pub fn new(dataset_id: impl Into<String>, document_id: impl Into<String>, text: impl Into<String>) -> Self {
26        Self {
27            dataset_id: dataset_id.into(),
28            document_id: document_id.into(),
29            title: String::new(),
30            text: text.into(),
31            metadata: Metadata::new(),
32        }
33    }
34
35    pub fn title(mut self, title: impl Into<String>) -> Self {
36        self.title = title.into();
37        self
38    }
39
40    pub fn metadata(mut self, metadata: Metadata) -> Self {
41        self.metadata = metadata;
42        self
43    }
44}
45
46/// Outcome of an ingestion.
47#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
48pub struct IngestReport {
49    /// New or changed chunks that were embedded and stored.
50    pub stored: usize,
51    /// Chunks already present with the same content (their vectors are reused,
52    /// metadata is refreshed), plus exact duplicates within the document.
53    pub skipped: usize,
54    /// Chunks of an earlier version of the document that were removed.
55    pub removed: usize,
56}
57
58/// Chunks, embeds (in batches) and stores documents.
59///
60/// Chunk ids are content-addressed per document (`dataset:document:hash`), so
61/// re-ingesting a changed document embeds only new text, keeps unchanged chunks
62/// (refreshing their position and metadata), and — with
63/// [`Ingestor::replace_existing`] — removes chunks the new version no longer
64/// contains. The swap is ordered so a failure never loses the previous version:
65/// everything is embedded first, then upserted, and only then are stale chunks
66/// deleted. Identical text in two different documents is stored once per
67/// document, so deleting one document never removes content from another.
68#[derive(Clone, Debug)]
69pub struct Ingestor {
70    store: Arc<dyn VectorStore>,
71    embedder: Arc<dyn Embedder>,
72    chunker: ChunkerConfig,
73    batch_size: usize,
74    replace_existing: bool,
75}
76
77impl Ingestor {
78    pub fn new(store: Arc<dyn VectorStore>, embedder: Arc<dyn Embedder>) -> Self {
79        Self { store, embedder, chunker: ChunkerConfig::default(), batch_size: 32, replace_existing: true }
80    }
81
82    pub fn chunker(mut self, config: ChunkerConfig) -> Self {
83        self.chunker = config;
84        self
85    }
86
87    /// Texts per embedding request (default 32).
88    pub fn batch_size(mut self, size: usize) -> Self {
89        self.batch_size = size.max(1);
90        self
91    }
92
93    /// Remove chunks of the previous version that the new version no longer
94    /// contains (default `true`). With `false`, ingestion only adds chunks.
95    pub fn replace_existing(mut self, replace: bool) -> Self {
96        self.replace_existing = replace;
97        self
98    }
99
100    /// Ingest one text document.
101    pub async fn ingest(&self, doc: &Document) -> Result<IngestReport, RagError> {
102        if doc.dataset_id.is_empty() || doc.document_id.is_empty() {
103            return Err(RagError::Input("document needs a dataset_id and a document_id".into()));
104        }
105        let existing: HashMap<String, StoredChunk> = self
106            .store
107            .document_chunks(&doc.dataset_id, &doc.document_id)
108            .await?
109            .into_iter()
110            .map(|c| (c.id.clone(), c))
111            .collect();
112
113        // A disabled document stays disabled when a new version is ingested.
114        let disabled = existing.values().any(|c| is_disabled(&c.metadata));
115        let mut report = IngestReport::default();
116        let mut seen: HashSet<String> = HashSet::new();
117        let mut fresh: Vec<StoredChunk> = Vec::new();
118        let mut refreshed: Vec<StoredChunk> = Vec::new();
119        for chunk in chunk_text(&doc.text, &self.chunker) {
120            let hash = content_hash(&doc.dataset_id, &chunk.text);
121            let id = format!("{}:{}:{hash}", doc.dataset_id, doc.document_id);
122            if !seen.insert(id.clone()) {
123                report.skipped += 1;
124                continue;
125            }
126            let mut metadata = doc.metadata.clone();
127            metadata.insert("chunk_index".into(), serde_json::Value::from(chunk.index));
128            if let Some(h) = &chunk.heading {
129                metadata.insert("heading".into(), serde_json::Value::String(h.clone()));
130            }
131            if let Some(parent) = chunk.parent {
132                metadata.insert(PARENT_KEY.into(), serde_json::Value::from(parent));
133            }
134            if disabled {
135                metadata.insert(DISABLED_KEY.into(), serde_json::Value::Bool(true));
136            }
137            let mut stored = StoredChunk {
138                id,
139                dataset_id: doc.dataset_id.clone(),
140                document_id: doc.document_id.clone(),
141                title: doc.title.clone(),
142                text: chunk.text,
143                metadata,
144                content_hash: hash,
145                vector: Vec::new(),
146            };
147            match existing.get(&stored.id).filter(|old| !old.vector.is_empty() && old.text == stored.text) {
148                Some(old) => {
149                    report.skipped += 1;
150                    if old.title != stored.title || old.metadata != stored.metadata {
151                        stored.vector.clone_from(&old.vector);
152                        refreshed.push(stored);
153                    }
154                }
155                None => fresh.push(stored),
156            }
157        }
158
159        // 1. Embed everything new. A failure here changes nothing in the store.
160        for batch in fresh.chunks_mut(self.batch_size) {
161            let texts: Vec<String> = batch.iter().map(|c| c.text.clone()).collect();
162            let vectors = self.embedder.embed_documents(&texts).await?;
163            if vectors.len() != batch.len() {
164                return Err(RagError::Decode(format!(
165                    "embedder returned {} vectors for {} texts",
166                    vectors.len(),
167                    batch.len()
168                )));
169            }
170            for (chunk, vector) in batch.iter_mut().zip(vectors) {
171                check_vector(&vector, &format!("embedding of chunk `{}`", chunk.id))?;
172                chunk.vector = vector;
173            }
174        }
175        report.stored = fresh.len();
176
177        // 2. Store new and refreshed chunks next to the old version.
178        fresh.append(&mut refreshed);
179        if !fresh.is_empty() {
180            self.store.upsert(fresh).await?;
181        }
182
183        // 3. Only now drop what the new version no longer contains.
184        if self.replace_existing {
185            let stale: Vec<String> = existing.keys().filter(|id| !seen.contains(*id)).cloned().collect();
186            if !stale.is_empty() {
187                report.removed = self.store.delete_chunks(&doc.dataset_id, &stale).await?;
188            }
189        }
190        Ok(report)
191    }
192
193    /// Extract text from raw bytes with `extractor`, then ingest it.
194    pub async fn ingest_bytes(
195        &self,
196        doc: Document,
197        bytes: &[u8],
198        mime_type: Option<&str>,
199        file_name: &str,
200        extractor: &dyn Extractor,
201    ) -> Result<IngestReport, RagError> {
202        let text = extractor.extract(bytes, mime_type, file_name)?;
203        self.ingest(&Document { text, ..doc }).await
204    }
205}