runmat_runtime/builtins/strings/text_analytics/
file_text.rs

1//! File-backed text extraction helpers for Text Analytics workflows.
2
3use std::io::{Cursor, Read};
4use std::path::{Path, PathBuf};
5
6use encoding_rs::Encoding;
7use runmat_builtins::{
8    BuiltinCompletionPolicy, BuiltinDescriptor, BuiltinErrorDescriptor, BuiltinOutputMode,
9    BuiltinParamArity, BuiltinParamDescriptor, BuiltinParamType, BuiltinSignatureDescriptor,
10    ResolveContext, Type, Value,
11};
12use runmat_macros::runtime_builtin;
13
14use crate::builtins::strings::core::compat::scalar_text;
15use crate::builtins::strings::text_analytics::html::{extract_html_text_value, ExtractionMethod};
16use crate::{build_runtime_error, gather_if_needed_async, BuiltinResult};
17
18const NAME: &str = "extractFileText";
19const MAX_FILE_BYTES: usize = 512 * 1024 * 1024;
20const MAX_DOCX_XML_BYTES: usize = 128 * 1024 * 1024;
21
22const OUT_TEXT: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
23    name: "str",
24    ty: BuiltinParamType::StringScalar,
25    arity: BuiltinParamArity::Required,
26    default: None,
27    description: "Extracted text.",
28}];
29
30const IN_FILE: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
31    name: "filename",
32    ty: BuiltinParamType::Any,
33    arity: BuiltinParamArity::Required,
34    default: None,
35    description: "Path to a text, HTML, or DOCX file.",
36}];
37
38const IN_FILE_REST: [BuiltinParamDescriptor; 2] = [
39    BuiltinParamDescriptor {
40        name: "filename",
41        ty: BuiltinParamType::Any,
42        arity: BuiltinParamArity::Required,
43        default: None,
44        description: "Path to a text, HTML, or DOCX file.",
45    },
46    BuiltinParamDescriptor {
47        name: "NameValue",
48        ty: BuiltinParamType::Any,
49        arity: BuiltinParamArity::Variadic,
50        default: None,
51        description: "Name-value options: Encoding and ExtractionMethod in this slice.",
52    },
53];
54
55const ERROR_INVALID_INPUT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
56    code: "RM.EXTRACT_FILE_TEXT.INVALID_INPUT",
57    identifier: Some("RunMat:extractFileText:InvalidInput"),
58    when: "Inputs do not match a supported extractFileText form.",
59    message: "extractFileText: invalid input",
60};
61
62const ERROR_IO: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
63    code: "RM.EXTRACT_FILE_TEXT.IO",
64    identifier: Some("RunMat:extractFileText:IOError"),
65    when: "The requested file cannot be read.",
66    message: "extractFileText: file read failed",
67};
68
69const ERROR_UNSUPPORTED: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
70    code: "RM.EXTRACT_FILE_TEXT.UNSUPPORTED",
71    identifier: Some("RunMat:extractFileText:UnsupportedFormat"),
72    when: "The requested file type or option requires unsupported extraction infrastructure.",
73    message: "extractFileText: unsupported file type or option",
74};
75
76const ERRORS: [BuiltinErrorDescriptor; 3] = [ERROR_INVALID_INPUT, ERROR_IO, ERROR_UNSUPPORTED];
77
78pub const EXTRACT_FILE_TEXT_DESCRIPTOR: BuiltinDescriptor = BuiltinDescriptor {
79    signatures: &[
80        BuiltinSignatureDescriptor {
81            label: "str = extractFileText(filename)",
82            inputs: &IN_FILE,
83            outputs: &OUT_TEXT,
84        },
85        BuiltinSignatureDescriptor {
86            label: "str = extractFileText(filename,Name,Value)",
87            inputs: &IN_FILE_REST,
88            outputs: &OUT_TEXT,
89        },
90    ],
91    output_mode: BuiltinOutputMode::Fixed,
92    completion_policy: BuiltinCompletionPolicy::Public,
93    errors: &ERRORS,
94};
95
96fn string_type(_args: &[Type], _ctx: &ResolveContext) -> Type {
97    Type::String
98}
99
100fn extract_error(
101    error: &'static BuiltinErrorDescriptor,
102    message: impl Into<String>,
103) -> crate::RuntimeError {
104    let mut builder = build_runtime_error(message).with_builtin(NAME);
105    if let Some(identifier) = error.identifier {
106        builder = builder.with_identifier(identifier);
107    }
108    builder.build()
109}
110
111#[runtime_builtin(
112    name = "extractFileText",
113    category = "strings/text_analytics",
114    summary = "Read text from text, HTML, and DOCX files.",
115    keywords = "extractFileText,text analytics,file,text,HTML,DOCX",
116    accel = "sink",
117    type_resolver(string_type),
118    descriptor(crate::builtins::strings::text_analytics::file_text::EXTRACT_FILE_TEXT_DESCRIPTOR),
119    builtin_path = "crate::builtins::strings::text_analytics::file_text"
120)]
121async fn extract_file_text_builtin(args: Vec<Value>) -> BuiltinResult<Value> {
122    let args = gather_args(args).await?;
123    let (source, options) = parse_args(args)?;
124    if is_url(&source) {
125        return Err(extract_error(
126            &ERROR_UNSUPPORTED,
127            "extractFileText: URL extraction requires website fetching support and remains tracked",
128        ));
129    }
130    let path = PathBuf::from(&source);
131    let kind = FileKind::from_path(&path);
132    validate_options_for_kind(kind, &options)?;
133    let bytes = read_file(&path).await?;
134    let text = match kind {
135        FileKind::Html => {
136            let html = decode_bytes(&bytes, options.encoding.as_deref())?;
137            let method = options.extraction_method.unwrap_or(ExtractionMethod::Tree);
138            match extract_html_text_value(Value::String(html), method)? {
139                Value::String(text) => text,
140                other => {
141                    return Err(extract_error(
142                        &ERROR_INVALID_INPUT,
143                        format!("extractFileText: unexpected HTML extraction output {other:?}"),
144                    ))
145                }
146            }
147        }
148        FileKind::Docx => extract_docx_text(&bytes)?,
149        FileKind::Pdf => unreachable!("PDF files are rejected before file IO"),
150        FileKind::PlainText => decode_bytes(&bytes, options.encoding.as_deref())?,
151    };
152    Ok(Value::String(text))
153}
154
155async fn gather_args(args: Vec<Value>) -> BuiltinResult<Vec<Value>> {
156    let mut out = Vec::with_capacity(args.len());
157    for arg in args {
158        out.push(gather_if_needed_async(&arg).await.map_err(|err| {
159            extract_error(
160                &ERROR_INVALID_INPUT,
161                format!("extractFileText: failed to gather input: {err}"),
162            )
163        })?);
164    }
165    Ok(out)
166}
167
168#[derive(Clone, Debug, Default)]
169struct ExtractFileTextOptions {
170    encoding: Option<String>,
171    extraction_method: Option<ExtractionMethod>,
172    password: Option<String>,
173    pages: Option<Vec<usize>>,
174}
175
176fn parse_args(args: Vec<Value>) -> BuiltinResult<(String, ExtractFileTextOptions)> {
177    if args.is_empty() {
178        return Err(extract_error(
179            &ERROR_INVALID_INPUT,
180            "extractFileText: expected filename input",
181        ));
182    }
183    if !(args.len() - 1).is_multiple_of(2) {
184        return Err(extract_error(
185            &ERROR_INVALID_INPUT,
186            "extractFileText: name-value options must appear in pairs",
187        ));
188    }
189    let source = scalar_file_input(&args[0])?;
190    let mut options = ExtractFileTextOptions::default();
191    let mut idx = 1usize;
192    while idx < args.len() {
193        let name = scalar_text(&args[idx], NAME)
194            .map_err(|err| extract_error(&ERROR_INVALID_INPUT, err.to_string()))?;
195        match name.to_ascii_lowercase().as_str() {
196            "encoding" => {
197                options.encoding = Some(
198                    scalar_text(&args[idx + 1], NAME)
199                        .map_err(|err| extract_error(&ERROR_INVALID_INPUT, err.to_string()))?,
200                );
201            }
202            "extractionmethod" => {
203                let method = scalar_text(&args[idx + 1], NAME)
204                    .map_err(|err| extract_error(&ERROR_INVALID_INPUT, err.to_string()))?;
205                options.extraction_method = Some(
206                    ExtractionMethod::parse(&method)
207                        .map_err(|err| extract_error(&ERROR_INVALID_INPUT, err.to_string()))?,
208                );
209            }
210            "password" => {
211                options.password = Some(
212                    scalar_text(&args[idx + 1], NAME)
213                        .map_err(|err| extract_error(&ERROR_INVALID_INPUT, err.to_string()))?,
214                );
215            }
216            "pages" => {
217                options.pages = Some(parse_pages(&args[idx + 1])?);
218            }
219            other => {
220                return Err(extract_error(
221                    &ERROR_INVALID_INPUT,
222                    format!("extractFileText: unsupported option '{other}'"),
223                ))
224            }
225        }
226        idx += 2;
227    }
228    Ok((source, options))
229}
230
231fn scalar_file_input(value: &Value) -> BuiltinResult<String> {
232    match value {
233        Value::String(value) => nonempty_source(value),
234        Value::StringArray(array) if array.data.len() == 1 => nonempty_source(&array.data[0]),
235        Value::CharArray(array) if array.rows <= 1 => {
236            nonempty_source(&array.data.iter().collect::<String>())
237        }
238        Value::Cell(cell) if cell.data.len() == 1 => scalar_file_input(&cell.data[0]),
239        other => Err(extract_error(
240            &ERROR_INVALID_INPUT,
241            format!("extractFileText: expected scalar filename or URL, got {other:?}"),
242        )),
243    }
244}
245
246fn nonempty_source(value: &str) -> BuiltinResult<String> {
247    let trimmed = value.trim();
248    if trimmed.is_empty() {
249        Err(extract_error(
250            &ERROR_INVALID_INPUT,
251            "extractFileText: filename must not be empty",
252        ))
253    } else {
254        Ok(trimmed.to_string())
255    }
256}
257
258fn parse_pages(value: &Value) -> BuiltinResult<Vec<usize>> {
259    let raw = match value {
260        Value::Num(value) => vec![*value],
261        Value::Tensor(tensor) => tensor.data.clone(),
262        other => {
263            return Err(extract_error(
264                &ERROR_INVALID_INPUT,
265                format!("extractFileText: Pages must be a positive integer vector, got {other:?}"),
266            ))
267        }
268    };
269    if raw.is_empty() {
270        return Err(extract_error(
271            &ERROR_INVALID_INPUT,
272            "extractFileText: Pages must not be empty",
273        ));
274    }
275    raw.into_iter()
276        .map(|value| {
277            if !value.is_finite() || value < 1.0 || value.fract() != 0.0 {
278                return Err(extract_error(
279                    &ERROR_INVALID_INPUT,
280                    format!("extractFileText: Pages values must be positive integers, got {value}"),
281                ));
282            }
283            Ok(value as usize)
284        })
285        .collect()
286}
287
288#[derive(Clone, Copy, Debug, PartialEq, Eq)]
289enum FileKind {
290    PlainText,
291    Html,
292    Docx,
293    Pdf,
294}
295
296impl FileKind {
297    fn from_path(path: &Path) -> Self {
298        match path
299            .extension()
300            .and_then(|ext| ext.to_str())
301            .map(|ext| ext.to_ascii_lowercase())
302            .as_deref()
303        {
304            Some("html" | "htm" | "xhtml") => Self::Html,
305            Some("docx") => Self::Docx,
306            Some("pdf") => Self::Pdf,
307            _ => Self::PlainText,
308        }
309    }
310}
311
312fn validate_options_for_kind(
313    kind: FileKind,
314    options: &ExtractFileTextOptions,
315) -> BuiltinResult<()> {
316    if kind == FileKind::Pdf {
317        return Err(extract_error(
318            &ERROR_UNSUPPORTED,
319            "extractFileText: PDF extraction requires PDF text extraction support and remains tracked",
320        ));
321    }
322    if options.password.is_some() || options.pages.is_some() {
323        return Err(extract_error(
324            &ERROR_UNSUPPORTED,
325            "extractFileText: Password and Pages options require PDF extraction support and remain tracked",
326        ));
327    }
328    if options.encoding.is_some() && kind == FileKind::Docx {
329        return Err(extract_error(
330            &ERROR_INVALID_INPUT,
331            "extractFileText: Encoding is supported for text and HTML files only",
332        ));
333    }
334    if options.extraction_method.is_some() && kind != FileKind::Html {
335        return Err(extract_error(
336            &ERROR_INVALID_INPUT,
337            "extractFileText: ExtractionMethod is supported for HTML files only",
338        ));
339    }
340    Ok(())
341}
342
343fn is_url(value: &str) -> bool {
344    value.starts_with("http://") || value.starts_with("https://")
345}
346
347async fn read_file(path: &Path) -> BuiltinResult<Vec<u8>> {
348    let bytes = runmat_filesystem::read_async(path).await.map_err(|err| {
349        extract_error(
350            &ERROR_IO,
351            format!(
352                "extractFileText: unable to read '{}': {err}",
353                path.display()
354            ),
355        )
356    })?;
357    if bytes.len() > MAX_FILE_BYTES {
358        return Err(extract_error(
359            &ERROR_UNSUPPORTED,
360            format!(
361                "extractFileText: file '{}' exceeds the {} byte extraction limit",
362                path.display(),
363                MAX_FILE_BYTES
364            ),
365        ));
366    }
367    Ok(bytes)
368}
369
370fn decode_bytes(bytes: &[u8], encoding: Option<&str>) -> BuiltinResult<String> {
371    let encoding = encoding.unwrap_or("utf-8");
372    let Some(encoding) = Encoding::for_label(encoding.trim().as_bytes()) else {
373        return Err(extract_error(
374            &ERROR_INVALID_INPUT,
375            format!("extractFileText: unsupported Encoding '{encoding}'"),
376        ));
377    };
378    let (text, _, had_errors) = encoding.decode(bytes);
379    if had_errors && encoding.name().eq_ignore_ascii_case("UTF-8") {
380        return Err(extract_error(
381            &ERROR_INVALID_INPUT,
382            "extractFileText: unable to decode file as UTF-8",
383        ));
384    }
385    Ok(text.into_owned())
386}
387
388fn extract_docx_text(bytes: &[u8]) -> BuiltinResult<String> {
389    let mut archive = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|err| {
390        extract_error(
391            &ERROR_UNSUPPORTED,
392            format!("extractFileText: unable to open DOCX archive: {err}"),
393        )
394    })?;
395    let mut document = archive.by_name("word/document.xml").map_err(|err| {
396        extract_error(
397            &ERROR_UNSUPPORTED,
398            format!("extractFileText: DOCX archive is missing word/document.xml: {err}"),
399        )
400    })?;
401    if document.size() > MAX_DOCX_XML_BYTES as u64 {
402        return Err(extract_error(
403            &ERROR_UNSUPPORTED,
404            "extractFileText: DOCX document.xml is too large for this slice",
405        ));
406    }
407    let mut xml = String::new();
408    document.read_to_string(&mut xml).map_err(|err| {
409        extract_error(
410            &ERROR_INVALID_INPUT,
411            format!("extractFileText: unable to decode DOCX XML as UTF-8: {err}"),
412        )
413    })?;
414    Ok(text_from_docx_document_xml(&xml))
415}
416
417fn text_from_docx_document_xml(xml: &str) -> String {
418    let mut out = String::new();
419    let mut cursor = 0usize;
420    while cursor < xml.len() {
421        let next_text = xml[cursor..].find("<w:t").map(|idx| cursor + idx);
422        let next_paragraph = xml[cursor..].find("</w:p>").map(|idx| cursor + idx);
423        let next_break = xml[cursor..].find("<w:br").map(|idx| cursor + idx);
424        let next_tab = xml[cursor..].find("<w:tab").map(|idx| cursor + idx);
425        let next = [next_text, next_paragraph, next_break, next_tab]
426            .into_iter()
427            .flatten()
428            .min();
429        let Some(pos) = next else {
430            break;
431        };
432        if Some(pos) == next_text {
433            let Some(start_rel) = xml[pos..].find('>') else {
434                break;
435            };
436            let text_start = pos + start_rel + 1;
437            let Some(end_rel) = xml[text_start..].find("</w:t>") else {
438                break;
439            };
440            let text_end = text_start + end_rel;
441            out.push_str(&xml_unescape(&xml[text_start..text_end]));
442            cursor = text_end + "</w:t>".len();
443        } else if Some(pos) == next_break {
444            if !out.ends_with('\n') {
445                out.push('\n');
446            }
447            cursor = xml[pos..]
448                .find('>')
449                .map(|idx| pos + idx + 1)
450                .unwrap_or(xml.len());
451        } else if Some(pos) == next_tab {
452            out.push('\t');
453            cursor = xml[pos..]
454                .find('>')
455                .map(|idx| pos + idx + 1)
456                .unwrap_or(xml.len());
457        } else {
458            if !out.is_empty() && !out.ends_with('\n') {
459                out.push('\n');
460            }
461            cursor = pos + "</w:p>".len();
462        }
463    }
464    out.trim_end_matches('\n').to_string()
465}
466
467fn xml_unescape(value: &str) -> String {
468    value
469        .replace("&lt;", "<")
470        .replace("&gt;", ">")
471        .replace("&quot;", "\"")
472        .replace("&apos;", "'")
473        .replace("&amp;", "&")
474}
475
476#[cfg(test)]
477mod tests {
478    use super::*;
479    use crate::builtins::common::test_support;
480    use runmat_builtins::{CellArray, Tensor};
481    use runmat_time::unix_timestamp_ms;
482    use std::io::Write;
483
484    fn run(args: Vec<Value>) -> BuiltinResult<Value> {
485        futures::executor::block_on(extract_file_text_builtin(args))
486    }
487
488    fn unique_path(name: &str, ext: &str) -> PathBuf {
489        let mut path = std::env::temp_dir();
490        path.push(format!(
491            "runmat_extract_file_text_{}_{}_{}.{}",
492            name,
493            std::process::id(),
494            unix_timestamp_ms(),
495            ext
496        ));
497        path
498    }
499
500    fn string_value(value: Value) -> String {
501        match value {
502            Value::String(text) => text,
503            other => panic!("expected string, got {other:?}"),
504        }
505    }
506
507    #[cfg(not(target_arch = "wasm32"))]
508    #[test]
509    fn reads_plain_text_file_with_encoding_option() {
510        let path = unique_path("plain", "txt");
511        test_support::fs::write(&path, b"caf\xe9").expect("write sample");
512        let value = run(vec![
513            Value::String(path.to_string_lossy().to_string()),
514            Value::String("Encoding".to_string()),
515            Value::String("windows-1252".to_string()),
516        ])
517        .expect("extract");
518        assert_eq!(string_value(value), "café");
519        let _ = test_support::fs::remove_file(&path);
520    }
521
522    #[cfg(not(target_arch = "wasm32"))]
523    #[test]
524    fn extracts_visible_text_from_html_file() {
525        let path = unique_path("html", "html");
526        test_support::fs::write(
527            &path,
528            "<html><body><h1>Title</h1><script>hidden()</script><p>Body</p></body></html>",
529        )
530        .expect("write sample");
531        let value = run(vec![
532            Value::String(path.to_string_lossy().to_string()),
533            Value::String("ExtractionMethod".to_string()),
534            Value::String("all-text".to_string()),
535        ])
536        .expect("extract");
537        let text = string_value(value);
538        assert!(text.contains("Title"));
539        assert!(text.contains("Body"));
540        assert!(!text.contains("hidden"));
541        let _ = test_support::fs::remove_file(&path);
542    }
543
544    #[cfg(not(target_arch = "wasm32"))]
545    #[test]
546    fn extracts_text_from_docx_document_xml() {
547        let path = unique_path("docx", "docx");
548        let file = std::fs::File::create(&path).expect("create docx");
549        let mut zip = zip::ZipWriter::new(file);
550        let options = zip::write::SimpleFileOptions::default()
551            .compression_method(zip::CompressionMethod::Stored);
552        zip.start_file("word/document.xml", options)
553            .expect("start document");
554        zip.write_all(
555            br#"<w:document><w:body><w:p><w:r><w:t>Hello &amp; welcome</w:t></w:r></w:p><w:p><w:r><w:t>Second</w:t></w:r></w:p></w:body></w:document>"#,
556        )
557        .expect("write document");
558        zip.finish().expect("finish docx");
559
560        let value = run(vec![Value::String(path.to_string_lossy().to_string())]).expect("extract");
561        assert_eq!(string_value(value), "Hello & welcome\nSecond");
562        let _ = test_support::fs::remove_file(&path);
563    }
564
565    #[cfg(not(target_arch = "wasm32"))]
566    #[test]
567    fn rejects_pdf_options_and_urls_as_tracked_gaps() {
568        let err = run(vec![
569            Value::String("https://example.com".to_string()),
570            Value::String("ExtractionMethod".to_string()),
571            Value::String("tree".to_string()),
572        ])
573        .expect_err("expected URL rejection");
574        assert!(err.to_string().contains("URL extraction"));
575
576        let err = run(vec![
577            Value::String("paper.pdf".to_string()),
578            Value::String("Pages".to_string()),
579            Value::Tensor(Tensor::new(vec![1.0], vec![1, 1]).unwrap()),
580        ])
581        .expect_err("expected PDF option rejection");
582        assert!(err.to_string().contains("PDF extraction"));
583
584        let err =
585            run(vec![Value::String("paper.pdf".to_string())]).expect_err("expected PDF rejection");
586        assert!(err.to_string().contains("PDF extraction"));
587
588        let err = run(vec![
589            Value::String("report.docx".to_string()),
590            Value::String("Encoding".to_string()),
591            Value::String("UTF-8".to_string()),
592        ])
593        .expect_err("expected DOCX encoding rejection");
594        assert!(err.to_string().contains("Encoding is supported"));
595    }
596
597    #[cfg(not(target_arch = "wasm32"))]
598    #[test]
599    fn accepts_single_cell_filename_input() {
600        let path = unique_path("cell", "txt");
601        test_support::fs::write(&path, "cell path").expect("write sample");
602        let cell = CellArray::new(
603            vec![Value::String(path.to_string_lossy().to_string())],
604            1,
605            1,
606        )
607        .unwrap();
608        let value = run(vec![Value::Cell(cell)]).expect("extract");
609        assert_eq!(string_value(value), "cell path");
610        let _ = test_support::fs::remove_file(&path);
611    }
612}
runmat_runtime/builtins/strings/text_analytics/file_text.rs

runmat_runtime/builtins/strings/text_analytics/
file_text.rs
