1use std::sync::Arc;
4
5use futures::future::BoxFuture;
6use serde::Deserialize;
7
8use graphrt_core::catalog::{NodeCategory, NodeDescriptor, OutputDescriptor};
9use graphrt_core::dsl::NodeDoc;
10use graphrt_core::file::{DataUrlResolver, FileError, FileResolver};
11use graphrt_core::node::{ConfigError, Node, NodeContext, NodeError, NodeOutput, NodeResult, Registry, parse_config};
12use graphrt_core::value::File;
13use graphrt_core::{Selector, Value, ValueType};
14
15use crate::{DocError, Extraction, Options, OutputFormat};
16
17pub const KIND: &str = "document-extractor";
19
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub struct DocLimits {
25 pub max_file_bytes: u64,
27 pub max_output_chars: usize,
29 pub max_pages: usize,
31 pub max_sheets: usize,
33 pub max_rows: usize,
35 pub max_cols: usize,
37 pub max_cells: usize,
39 pub max_zip_entries: usize,
41 pub max_decompressed_bytes: u64,
43}
44
45impl Default for DocLimits {
46 fn default() -> Self {
47 let opts = Options::default();
48 Self {
49 max_file_bytes: 50 * 1024 * 1024,
50 max_output_chars: opts.max_output_chars,
51 max_pages: opts.max_pages,
52 max_sheets: opts.max_sheets,
53 max_rows: opts.max_rows,
54 max_cols: opts.max_cols,
55 max_cells: opts.max_cells,
56 max_zip_entries: opts.max_zip_entries,
57 max_decompressed_bytes: opts.max_decompressed_bytes,
58 }
59 }
60}
61
62impl DocLimits {
63 fn options(&self, format: OutputFormat) -> Options {
64 Options {
65 format,
66 max_output_chars: self.max_output_chars,
67 max_pages: self.max_pages,
68 max_sheets: self.max_sheets,
69 max_rows: self.max_rows,
70 max_cols: self.max_cols,
71 max_cells: self.max_cells,
72 max_zip_entries: self.max_zip_entries,
73 max_decompressed_bytes: self.max_decompressed_bytes,
74 }
75 }
76}
77
78#[derive(Debug, Clone, Deserialize)]
79struct Config {
80 variable_selector: Selector,
81 #[serde(default)]
82 max_bytes: Option<u64>,
83 #[serde(default)]
84 output_format: OutputFormat,
85}
86
87#[derive(Debug, Clone)]
93pub struct DocumentExtractorNode {
94 selector: Selector,
95 max_bytes: Option<u64>,
96 format: OutputFormat,
97}
98
99impl DocumentExtractorNode {
100 pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
102 let cfg: Config = parse_config(doc)?;
103 if cfg.max_bytes == Some(0) {
104 return Err(ConfigError::new(doc, "`max_bytes` must be greater than zero"));
105 }
106 Ok(Self { selector: cfg.variable_selector, max_bytes: cfg.max_bytes, format: cfg.output_format })
107 }
108}
109
110impl Node for DocumentExtractorNode {
111 fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
112 Box::pin(async move {
113 let value = ctx
114 .get(&self.selector)
115 .filter(|v| !v.is_null())
116 .ok_or_else(|| NodeError::invalid_input(format!("variable `{}` is not set", self.selector)))?;
117 let (files, many) = match &value {
118 Value::File(f) => (vec![Arc::clone(f)], false),
119 Value::Array(items) => {
120 let mut files = Vec::with_capacity(items.len());
121 for (i, item) in items.iter().enumerate() {
122 match item {
123 Value::File(f) => files.push(Arc::clone(f)),
124 other => {
125 return Err(NodeError::invalid_input(format!(
126 "`{}` item {i} is a {}, expected a file",
127 self.selector,
128 other.type_name()
129 )));
130 }
131 }
132 }
133 (files, true)
134 }
135 other => {
136 return Err(NodeError::invalid_input(format!(
137 "`{}` is a {}, expected a file or a list of files",
138 self.selector,
139 other.type_name()
140 )));
141 }
142 };
143
144 let limits = ctx.service::<DocLimits>().copied().unwrap_or_default();
145 let max_bytes = self.max_bytes.map_or(limits.max_file_bytes, |m| m.min(limits.max_file_bytes));
146 let opts = limits.options(self.format);
147 let resolver = ctx.service::<Arc<dyn FileResolver>>().cloned();
148
149 let mut texts = Vec::with_capacity(files.len());
150 let mut details = Vec::with_capacity(files.len());
151 for file in &files {
152 let extraction = extract_file(resolver.as_ref(), file, max_bytes, &opts).await?;
153 details.push(Value::object([
154 ("name", Value::from(file.name.as_str())),
155 ("format", Value::from(extraction.format.name())),
156 ("chars", Value::Int(i64::try_from(extraction.text.chars().count()).unwrap_or(i64::MAX))),
157 ("truncated", Value::Bool(extraction.truncated)),
158 ]));
159 texts.push(Value::from(extraction.text));
160 }
161
162 let text = if many { Value::array(texts) } else { texts.into_iter().next().unwrap_or_default() };
163 Ok(NodeOutput::new().with("text", text).meta("files", Value::array(details)).into())
164 })
165 }
166
167 fn inputs(&self) -> Vec<Selector> {
168 vec![self.selector.clone()]
169 }
170}
171
172async fn extract_file(
174 resolver: Option<&Arc<dyn FileResolver>>,
175 file: &File,
176 max_bytes: u64,
177 opts: &Options,
178) -> Result<Extraction, NodeError> {
179 let read = match resolver {
180 Some(r) => r.read(file, max_bytes).await,
181 None => DataUrlResolver.read(file, max_bytes).await,
182 };
183 let content = read.map_err(|e| file_error(file, e, resolver.is_none()))?;
184 let mime = content.mime_type.or_else(|| file.mime_type.clone());
185 let filename = match &file.extension {
186 Some(ext) if crate::detect::extension(&file.name).is_none() => {
187 format!("{}.{}", file.name, ext.trim_start_matches('.'))
188 }
189 _ => file.name.clone(),
190 };
191 let name = file.name.clone();
192 run_extraction(content.bytes, mime, filename, opts.clone())
193 .await
194 .map_err(|e| NodeError::fatal(e.kind(), format!("file `{name}`: {e}")))
195}
196
197async fn run_extraction(
199 bytes: Vec<u8>,
200 mime: Option<String>,
201 filename: String,
202 opts: Options,
203) -> Result<Extraction, DocError> {
204 let work = move || crate::extract_detailed(&bytes, mime.as_deref(), Some(&filename), &opts);
205 #[cfg(feature = "tokio")]
206 if let Ok(handle) = tokio::runtime::Handle::try_current() {
207 return match handle.spawn_blocking(work).await {
208 Ok(result) => result,
209 Err(e) => Err(DocError::malformed("document", format!("extraction task failed: {e}"))),
210 };
211 }
212 work()
213}
214
215fn file_error(file: &File, err: FileError, no_resolver: bool) -> NodeError {
216 match err {
217 FileError::Unsupported(_) if no_resolver => NodeError::fatal(
218 "missing_service",
219 format!("no file resolver configured to read file `{}` (only data URLs work without one)", file.name),
220 ),
221 FileError::Unsupported(_) => {
222 NodeError::fatal("unsupported_file", format!("no resolver can read file `{}`", file.name))
223 }
224 FileError::TooLarge { name, limit } => {
225 NodeError::fatal("file_too_large", format!("file `{name}` exceeds the {limit}-byte limit"))
226 }
227 e @ (FileError::NotFound { .. } | FileError::Denied { .. }) => {
228 NodeError::fatal("file_unavailable", e.to_string())
229 }
230 e @ FileError::Io { .. } => NodeError::new("file_io", e.to_string()),
231 e => NodeError::invalid_input(e.to_string()),
232 }
233}
234
235pub fn descriptor() -> NodeDescriptor {
237 NodeDescriptor::new(KIND, "Document Extractor", NodeCategory::Knowledge)
238 .describe(
239 "Extracts text from PDF, Word, PowerPoint, Excel/ODS, EPUB, HTML, e-mail, subtitle, \
240 CSV and plain-text files.",
241 )
242 .schema(serde_json::json!({
243 "type": "object",
244 "required": ["variable_selector"],
245 "properties": {
246 "variable_selector": {
247 "description": "Selector of a file or a list of files.",
248 "type": "array",
249 "items": { "type": "string" },
250 "minItems": 2
251 },
252 "max_bytes": {
253 "description": "Per-file size limit in bytes (cannot exceed the host limit).",
254 "type": "integer",
255 "minimum": 1
256 },
257 "output_format": {
258 "description": "`markdown` renders tables and headings as Markdown.",
259 "type": "string",
260 "enum": ["text", "markdown"],
261 "default": "text"
262 }
263 }
264 }))
265 .output(
266 OutputDescriptor::new("text", ValueType::Any)
267 .describe("Extracted text: a string for one file, an array of strings for a file list."),
268 )
269 .icon("file-text")
270 .source("graphrt-doc")
271}
272
273pub fn register(registry: &mut Registry) {
275 registry
276 .register_described(descriptor(), |doc| Ok(Arc::new(DocumentExtractorNode::from_doc(doc)?) as Arc<dyn Node>));
277}