Skip to main content

graphrt_rag/
doc_extractor.rs

1//! [`Extractor`] backed by `graphrt-doc`: ingest PDF, Office, spreadsheet,
2//! EPUB, HTML, email and subtitle files, not only plain text.
3
4use crate::error::RagError;
5use crate::types::Extractor;
6
7/// Extracts text from any format supported by `graphrt-doc`.
8#[derive(Clone, Debug, Default)]
9pub struct DocExtractor {
10    options: graphrt_doc::Options,
11}
12
13impl DocExtractor {
14    /// Extractor with default limits, emitting Markdown so headings and tables
15    /// survive chunking.
16    pub fn new() -> Self {
17        let options =
18            graphrt_doc::Options { format: graphrt_doc::OutputFormat::Markdown, ..graphrt_doc::Options::default() };
19        Self { options }
20    }
21
22    /// Extractor with explicit options.
23    pub fn with_options(options: graphrt_doc::Options) -> Self {
24        Self { options }
25    }
26}
27
28impl Extractor for DocExtractor {
29    fn extract(&self, bytes: &[u8], mime_type: Option<&str>, file_name: &str) -> Result<String, RagError> {
30        let extraction = graphrt_doc::extract_detailed(bytes, mime_type, Some(file_name), &self.options)
31            .map_err(|e| RagError::Input(format!("cannot extract `{file_name}`: {e}")))?;
32        // Indexing only part of a document would silently hide its tail from search.
33        if extraction.truncated {
34            return Err(RagError::Input(format!(
35                "`{file_name}` has more than {} characters of text; raise the extraction limit to index it fully",
36                self.options.max_output_chars
37            )));
38        }
39        Ok(extraction.text)
40    }
41}
42
43#[cfg(test)]
44mod tests {
45    use super::*;
46
47    #[test]
48    fn extracts_html_as_text() {
49        let html = b"<html><head><style>p{}</style></head><body><h1>Title</h1><p>Body text</p></body></html>";
50        let text = DocExtractor::new().extract(html, Some("text/html"), "page.html").unwrap();
51        assert!(text.contains("Title"));
52        assert!(text.contains("Body text"));
53        assert!(!text.contains("p{}"));
54    }
55
56    #[test]
57    fn truncated_extraction_is_an_error() {
58        let options = graphrt_doc::Options { max_output_chars: 5, ..graphrt_doc::Options::default() };
59        let err = DocExtractor::with_options(options).extract(b"a long plain text", Some("text/plain"), "a.txt");
60        assert!(matches!(err, Err(RagError::Input(m)) if m.contains("raise the extraction limit")));
61    }
62
63    #[test]
64    fn unsupported_format_is_an_error() {
65        let err = DocExtractor::new().extract(&[0, 1, 2, 3], Some("application/x-unknown"), "blob.bin");
66        assert!(err.is_err());
67    }
68}