doc_extractor 0.1.0

Extract text chunks from common document formats.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

use office_oxide::{Document as OfficeDocument, DocumentFormat};

use super::*;

pub struct OfficeLoader {
  document: OfficeDocument,
}

impl OfficeLoader {
  pub fn new<R: Read + Seek + Send + 'static>(reader: R, format: DocumentFormat) -> Result<Self, LoaderError> {
    Ok(Self {
      document: OfficeDocument::from_reader(reader, format)?,
    })
  }

  fn extract_text(&self) -> String {
    self.document.to_markdown()
  }

  fn extract_text_to_doc(&self) -> Document {
    Document::new(self.extract_text())
  }
}

impl Loader for OfficeLoader {
  fn load(self) -> LoaderResult<Vec<Document>> {
    let doc = self.extract_text_to_doc();
    Ok(vec![doc])
  }
}

#[cfg(test)]
mod tests {
  use std::{fs::read, io::Cursor, path::PathBuf};

  use office_oxide::{DocumentFormat, create::create_from_markdown_to_writer};

  use crate::Doc;

  use super::*;

  fn get_fixtures_path() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../fixtures")
  }

  #[test]
  fn test_parse_docx() {
    let docx_buffer = include_bytes!("../../../../fixtures/demo.docx");
    let parsed_buffer = include_str!("../../../../fixtures/demo.docx.md");

    {
      let loader = OfficeLoader::new(Cursor::new(docx_buffer), DocumentFormat::Docx).unwrap();

      let documents = loader.load().unwrap();

      assert_eq!(documents.len(), 1);
      assert_eq!(documents[0].page_content, parsed_buffer);
    }

    {
      let loader = OfficeLoader::new(Cursor::new(docx_buffer), DocumentFormat::Docx).unwrap();
      let documents = loader.load_and_split(TokenSplitter::default()).unwrap();

      for (idx, doc) in documents.into_iter().enumerate() {
        assert_eq!(
          doc.page_content,
          String::from_utf8_lossy(&read(get_fixtures_path().join(format!("demo.docx.{}.md", idx))).unwrap())
        );
      }
    }
  }

  #[test]
  fn test_parse_generated_ooxml_formats() {
    for format in [DocumentFormat::Docx, DocumentFormat::Xlsx, DocumentFormat::Pptx] {
      let mut buffer = Cursor::new(Vec::new());
      create_from_markdown_to_writer("# Office loader smoke\n\nShared parser text.", format, &mut buffer).unwrap();

      let loader = OfficeLoader::new(Cursor::new(buffer.into_inner()), format).unwrap();
      let documents = loader.load().unwrap();

      assert_eq!(documents.len(), 1);
      assert!(
        documents[0].page_content.contains("Office loader smoke"),
        "{format:?} output was: {}",
        documents[0].page_content
      );
      assert!(
        documents[0].page_content.contains("Shared parser text"),
        "{format:?} output was: {}",
        documents[0].page_content
      );
    }
  }

  #[test]
  fn test_auto_load_generated_ooxml_formats() {
    for (file_name, format) in [
      ("generated.docx", DocumentFormat::Docx),
      ("generated.xlsx", DocumentFormat::Xlsx),
      ("generated.pptx", DocumentFormat::Pptx),
    ] {
      let mut buffer = Cursor::new(Vec::new());
      create_from_markdown_to_writer("# Auto Office loader\n\nLoaded through Doc.", format, &mut buffer).unwrap();

      let doc = Doc::new(file_name, &buffer.into_inner()).unwrap();

      assert_eq!(doc.chunks.len(), 1);
      assert!(
        doc.chunks[0].content.contains("Auto Office loader"),
        "{format:?} output was: {}",
        doc.chunks[0].content
      );
      assert!(
        doc.chunks[0].content.contains("Loaded through Doc"),
        "{format:?} output was: {}",
        doc.chunks[0].content
      );
    }
  }
}