Skip to main content

graphrt_doc/
node.rs

1//! The `document-extractor` node.
2
3use std::sync::Arc;
4
5use futures::future::BoxFuture;
6use serde::Deserialize;
7
8use graphrt_core::catalog::{NodeCategory, NodeDescriptor, OutputDescriptor};
9use graphrt_core::dsl::NodeDoc;
10use graphrt_core::file::{DataUrlResolver, FileError, FileResolver};
11use graphrt_core::node::{ConfigError, Node, NodeContext, NodeError, NodeOutput, NodeResult, Registry, parse_config};
12use graphrt_core::value::File;
13use graphrt_core::{Selector, Value, ValueType};
14
15use crate::{DocError, Extraction, Options, OutputFormat};
16
17/// Node kind handled by this crate.
18pub const KIND: &str = "document-extractor";
19
20/// Host-level limits for document extraction. Register in
21/// [`graphrt_core::Services`] to override the defaults; workflow config can only
22/// lower them.
23#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub struct DocLimits {
25    /// Maximum size of one input file in bytes.
26    pub max_file_bytes: u64,
27    /// Maximum characters of extracted text per file (longer text is truncated).
28    pub max_output_chars: usize,
29    /// Maximum PDF pages / slides / EPUB chapters.
30    pub max_pages: usize,
31    /// Maximum spreadsheet sheets.
32    pub max_sheets: usize,
33    /// Maximum rows per sheet or CSV file.
34    pub max_rows: usize,
35    /// Maximum columns of any table.
36    pub max_cols: usize,
37    /// Maximum non-empty cells per spreadsheet sheet.
38    pub max_cells: usize,
39    /// Maximum zip entries.
40    pub max_zip_entries: usize,
41    /// Maximum total decompressed bytes from a zip container.
42    pub max_decompressed_bytes: u64,
43}
44
45impl Default for DocLimits {
46    fn default() -> Self {
47        let opts = Options::default();
48        Self {
49            max_file_bytes: 50 * 1024 * 1024,
50            max_output_chars: opts.max_output_chars,
51            max_pages: opts.max_pages,
52            max_sheets: opts.max_sheets,
53            max_rows: opts.max_rows,
54            max_cols: opts.max_cols,
55            max_cells: opts.max_cells,
56            max_zip_entries: opts.max_zip_entries,
57            max_decompressed_bytes: opts.max_decompressed_bytes,
58        }
59    }
60}
61
62impl DocLimits {
63    fn options(&self, format: OutputFormat) -> Options {
64        Options {
65            format,
66            max_output_chars: self.max_output_chars,
67            max_pages: self.max_pages,
68            max_sheets: self.max_sheets,
69            max_rows: self.max_rows,
70            max_cols: self.max_cols,
71            max_cells: self.max_cells,
72            max_zip_entries: self.max_zip_entries,
73            max_decompressed_bytes: self.max_decompressed_bytes,
74        }
75    }
76}
77
78#[derive(Debug, Clone, Deserialize)]
79struct Config {
80    variable_selector: Selector,
81    #[serde(default)]
82    max_bytes: Option<u64>,
83    #[serde(default)]
84    output_format: OutputFormat,
85}
86
87/// Extracts text from a file or a list of files.
88///
89/// Output `text` is a string for a single file and an array of strings (input
90/// order) for a file list. Per-file details (format, characters, truncation) are
91/// reported in the `files` metadata entry.
92#[derive(Debug, Clone)]
93pub struct DocumentExtractorNode {
94    selector: Selector,
95    max_bytes: Option<u64>,
96    format: OutputFormat,
97}
98
99impl DocumentExtractorNode {
100    /// Build from a node document.
101    pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
102        let cfg: Config = parse_config(doc)?;
103        if cfg.max_bytes == Some(0) {
104            return Err(ConfigError::new(doc, "`max_bytes` must be greater than zero"));
105        }
106        Ok(Self { selector: cfg.variable_selector, max_bytes: cfg.max_bytes, format: cfg.output_format })
107    }
108}
109
110impl Node for DocumentExtractorNode {
111    fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
112        Box::pin(async move {
113            let value = ctx
114                .get(&self.selector)
115                .filter(|v| !v.is_null())
116                .ok_or_else(|| NodeError::invalid_input(format!("variable `{}` is not set", self.selector)))?;
117            let (files, many) = match &value {
118                Value::File(f) => (vec![Arc::clone(f)], false),
119                Value::Array(items) => {
120                    let mut files = Vec::with_capacity(items.len());
121                    for (i, item) in items.iter().enumerate() {
122                        match item {
123                            Value::File(f) => files.push(Arc::clone(f)),
124                            other => {
125                                return Err(NodeError::invalid_input(format!(
126                                    "`{}` item {i} is a {}, expected a file",
127                                    self.selector,
128                                    other.type_name()
129                                )));
130                            }
131                        }
132                    }
133                    (files, true)
134                }
135                other => {
136                    return Err(NodeError::invalid_input(format!(
137                        "`{}` is a {}, expected a file or a list of files",
138                        self.selector,
139                        other.type_name()
140                    )));
141                }
142            };
143
144            let limits = ctx.service::<DocLimits>().copied().unwrap_or_default();
145            let max_bytes = self.max_bytes.map_or(limits.max_file_bytes, |m| m.min(limits.max_file_bytes));
146            let opts = limits.options(self.format);
147            let resolver = ctx.service::<Arc<dyn FileResolver>>().cloned();
148
149            let mut texts = Vec::with_capacity(files.len());
150            let mut details = Vec::with_capacity(files.len());
151            for file in &files {
152                let extraction = extract_file(resolver.as_ref(), file, max_bytes, &opts).await?;
153                details.push(Value::object([
154                    ("name", Value::from(file.name.as_str())),
155                    ("format", Value::from(extraction.format.name())),
156                    ("chars", Value::Int(i64::try_from(extraction.text.chars().count()).unwrap_or(i64::MAX))),
157                    ("truncated", Value::Bool(extraction.truncated)),
158                ]));
159                texts.push(Value::from(extraction.text));
160            }
161
162            let text = if many { Value::array(texts) } else { texts.into_iter().next().unwrap_or_default() };
163            Ok(NodeOutput::new().with("text", text).meta("files", Value::array(details)).into())
164        })
165    }
166
167    fn inputs(&self) -> Vec<Selector> {
168        vec![self.selector.clone()]
169    }
170}
171
172/// Read and extract one file.
173async fn extract_file(
174    resolver: Option<&Arc<dyn FileResolver>>,
175    file: &File,
176    max_bytes: u64,
177    opts: &Options,
178) -> Result<Extraction, NodeError> {
179    let read = match resolver {
180        Some(r) => r.read(file, max_bytes).await,
181        None => DataUrlResolver.read(file, max_bytes).await,
182    };
183    let content = read.map_err(|e| file_error(file, e, resolver.is_none()))?;
184    let mime = content.mime_type.or_else(|| file.mime_type.clone());
185    let filename = match &file.extension {
186        Some(ext) if crate::detect::extension(&file.name).is_none() => {
187            format!("{}.{}", file.name, ext.trim_start_matches('.'))
188        }
189        _ => file.name.clone(),
190    };
191    let name = file.name.clone();
192    run_extraction(content.bytes, mime, filename, opts.clone())
193        .await
194        .map_err(|e| NodeError::fatal(e.kind(), format!("file `{name}`: {e}")))
195}
196
197/// Extraction is CPU-bound: run it on Tokio's blocking pool when available.
198async fn run_extraction(
199    bytes: Vec<u8>,
200    mime: Option<String>,
201    filename: String,
202    opts: Options,
203) -> Result<Extraction, DocError> {
204    let work = move || crate::extract_detailed(&bytes, mime.as_deref(), Some(&filename), &opts);
205    #[cfg(feature = "tokio")]
206    if let Ok(handle) = tokio::runtime::Handle::try_current() {
207        return match handle.spawn_blocking(work).await {
208            Ok(result) => result,
209            Err(e) => Err(DocError::malformed("document", format!("extraction task failed: {e}"))),
210        };
211    }
212    work()
213}
214
215fn file_error(file: &File, err: FileError, no_resolver: bool) -> NodeError {
216    match err {
217        FileError::Unsupported(_) if no_resolver => NodeError::fatal(
218            "missing_service",
219            format!("no file resolver configured to read file `{}` (only data URLs work without one)", file.name),
220        ),
221        FileError::Unsupported(_) => {
222            NodeError::fatal("unsupported_file", format!("no resolver can read file `{}`", file.name))
223        }
224        FileError::TooLarge { name, limit } => {
225            NodeError::fatal("file_too_large", format!("file `{name}` exceeds the {limit}-byte limit"))
226        }
227        e @ (FileError::NotFound { .. } | FileError::Denied { .. }) => {
228            NodeError::fatal("file_unavailable", e.to_string())
229        }
230        e @ FileError::Io { .. } => NodeError::new("file_io", e.to_string()),
231        e => NodeError::invalid_input(e.to_string()),
232    }
233}
234
235/// Catalog descriptor for the node.
236pub fn descriptor() -> NodeDescriptor {
237    NodeDescriptor::new(KIND, "Document Extractor", NodeCategory::Knowledge)
238        .describe(
239            "Extracts text from PDF, Word, PowerPoint, Excel/ODS, EPUB, HTML, e-mail, subtitle, \
240             CSV and plain-text files.",
241        )
242        .schema(serde_json::json!({
243            "type": "object",
244            "additionalProperties": false,
245            "required": ["variable_selector"],
246            "properties": {
247                "variable_selector": {
248                    "description": "Selector of a file or a list of files.",
249                    "type": "array",
250                    "items": { "type": "string" },
251                    "minItems": 2
252                },
253                "max_bytes": {
254                    "description": "Per-file size limit in bytes (cannot exceed the host limit).",
255                    "type": "integer",
256                    "minimum": 1
257                },
258                "output_format": {
259                    "description": "`markdown` renders tables and headings as Markdown.",
260                    "type": "string",
261                    "enum": ["text", "markdown"],
262                    "default": "text"
263                }
264            }
265        }))
266        .output(
267            OutputDescriptor::new("text", ValueType::Any)
268                .describe("Extracted text: a string for one file, an array of strings for a file list."),
269        )
270        .icon("file-text")
271        .source("graphrt-doc")
272}
273
274/// Register the `document-extractor` node.
275pub fn register(registry: &mut Registry) {
276    registry.register_lazily_described(KIND, descriptor, |doc| {
277        Ok(Arc::new(DocumentExtractorNode::from_doc(doc)?) as Arc<dyn Node>)
278    });
279}