acorn/io/document/
mod.rs

1//! Document conversion utilities.
2use crate::io::http::get;
3use crate::io::{read_file, uri_to_path, ApiResult};
4use crate::prelude::PathBuf;
5use crate::schema::pid::Identifier;
6use crate::util::MimeType;
7use crate::{Location, Scheme};
8use anydoc::Format;
9use bon::Builder;
10use color_eyre::eyre::eyre;
11use derive_more::Display;
12use serde::{Deserialize, Serialize};
13
14mod index;
15mod matching;
16
17pub(crate) use index::{DocumentEntry, DocumentParser};
18pub use index::{DocumentExcerpt, DocumentIndex, DocumentMatch, DocumentPath, DocumentPathSegment, DocumentPosition, DocumentQuery, DocumentSpan};
19
20/// Document formats supported by Anydoc.
21#[derive(Clone, Copy, Debug, Deserialize, Display, Eq, PartialEq, Serialize)]
22pub enum DocumentFormat {
23    /// Comma-separated values.
24    #[display("csv")]
25    Csv,
26    /// Binary Word document.
27    #[display("doc")]
28    Doc,
29    /// OOXML Word document.
30    #[display("docx")]
31    Docx,
32    /// EPUB publication.
33    #[display("epub")]
34    Epub,
35    /// Excel workbook.
36    #[display("excel")]
37    Excel,
38    /// OpenDocument presentation.
39    #[display("odp")]
40    Odp,
41    /// OpenDocument spreadsheet.
42    #[display("ods")]
43    Ods,
44    /// OpenDocument text.
45    #[display("odt")]
46    Odt,
47    /// Portable Document Format.
48    #[display("pdf")]
49    Pdf,
50    /// Binary PowerPoint presentation.
51    #[display("ppt")]
52    Ppt,
53    /// OOXML PowerPoint presentation.
54    #[display("pptx")]
55    Pptx,
56    /// Rich Text Format.
57    #[display("rtf")]
58    Rtf,
59}
60/// Loaded document content and its source metadata.
61#[derive(Builder, Clone, Debug)]
62#[builder(start_fn = init, on(String, into))]
63pub struct SourceDocument {
64    /// Extracted or plain-text document content.
65    pub content: String,
66    /// Source file type.
67    pub format: String,
68    /// Original path, URI, persistent identifier, or input label.
69    pub source: String,
70}
71impl From<DocumentFormat> for Format {
72    fn from(value: DocumentFormat) -> Self {
73        match value {
74            | DocumentFormat::Csv => Self::Csv,
75            | DocumentFormat::Doc => Self::Doc,
76            | DocumentFormat::Docx => Self::Docx,
77            | DocumentFormat::Epub => Self::Epub,
78            | DocumentFormat::Excel => Self::Excel,
79            | DocumentFormat::Odp => Self::Odp,
80            | DocumentFormat::Ods => Self::Ods,
81            | DocumentFormat::Odt => Self::Odt,
82            | DocumentFormat::Pdf => Self::Pdf,
83            | DocumentFormat::Ppt => Self::Ppt,
84            | DocumentFormat::Pptx => Self::Pptx,
85            | DocumentFormat::Rtf => Self::Rtf,
86        }
87    }
88}
89impl From<Format> for DocumentFormat {
90    fn from(value: Format) -> Self {
91        match value {
92            | Format::Csv => Self::Csv,
93            | Format::Doc => Self::Doc,
94            | Format::Docx => Self::Docx,
95            | Format::Epub => Self::Epub,
96            | Format::Excel => Self::Excel,
97            | Format::Odp => Self::Odp,
98            | Format::Ods => Self::Ods,
99            | Format::Odt => Self::Odt,
100            | Format::Pdf => Self::Pdf,
101            | Format::Ppt => Self::Ppt,
102            | Format::Pptx => Self::Pptx,
103            | Format::Rtf => Self::Rtf,
104        }
105    }
106}
107impl From<DocumentFormat> for MimeType {
108    fn from(value: DocumentFormat) -> Self {
109        match value {
110            | DocumentFormat::Csv => Self::Csv,
111            | DocumentFormat::Doc => Self::Doc,
112            | DocumentFormat::Docx => Self::Docx,
113            | DocumentFormat::Epub => Self::Epub,
114            | DocumentFormat::Excel => Self::Excel,
115            | DocumentFormat::Odp => Self::Odp,
116            | DocumentFormat::Ods => Self::Ods,
117            | DocumentFormat::Odt => Self::Odt,
118            | DocumentFormat::Pdf => Self::Pdf,
119            | DocumentFormat::Ppt => Self::Ppt,
120            | DocumentFormat::Pptx => Self::Powerpoint,
121            | DocumentFormat::Rtf => Self::Rtf,
122        }
123    }
124}
125impl TryFrom<&MimeType> for DocumentFormat {
126    type Error = color_eyre::Report;
127    fn try_from(value: &MimeType) -> Result<Self, Self::Error> {
128        match value {
129            | MimeType::Csv => Ok(Self::Csv),
130            | MimeType::Doc => Ok(Self::Doc),
131            | MimeType::Docx => Ok(Self::Docx),
132            | MimeType::Epub => Ok(Self::Epub),
133            | MimeType::Excel => Ok(Self::Excel),
134            | MimeType::Odp => Ok(Self::Odp),
135            | MimeType::Ods => Ok(Self::Ods),
136            | MimeType::Odt => Ok(Self::Odt),
137            | MimeType::Pdf => Ok(Self::Pdf),
138            | MimeType::Ppt => Ok(Self::Ppt),
139            | MimeType::Powerpoint => Ok(Self::Pptx),
140            | MimeType::Rtf => Ok(Self::Rtf),
141            | _ => Err(eyre!("Unsupported Anydoc MIME type: {value}")),
142        }
143    }
144}
145impl SourceDocument {
146    pub(crate) fn is_markdown(&self) -> bool {
147        MimeType::from(self.source.as_str()) == MimeType::Markdown || MimeType::from(self.format.as_str()) == MimeType::Markdown
148    }
149    fn is_yaml(&self) -> bool {
150        let mime = MimeType::from(self.source.as_str());
151        mime.is_yaml() || mime.is_cff()
152    }
153    /// Create an unloaded document source at a local location.
154    pub fn at(location: impl Into<PathBuf>) -> Self {
155        let source = location.into().display().to_string();
156        let format = MimeType::from(source.as_str()).file_type();
157        Self {
158            content: String::new(),
159            format,
160            source,
161        }
162    }
163    /// Load a local path, remote URI, or persistent identifier.
164    pub async fn load(value: impl Into<Location>, offline: bool) -> ApiResult<Self> {
165        let location = value.into();
166        let value: &str = (&location).into();
167        let persistent = !Identifier::find_all(value).is_empty();
168        let scheme = location.scheme();
169        let source = location.uri().unwrap_or_default();
170        match (persistent, scheme, offline) {
171            | (true, _, _) => Ok(Self::init().content(value).format("pid").source(value).build()),
172            | (false, Scheme::HTTP | Scheme::HTTPS, true) => Err(eyre!("Remote input is unavailable in offline mode: {value}")),
173            | (false, Scheme::HTTP | Scheme::HTTPS, false) => get(source.clone())
174                .send()
175                .await
176                .map(|response| response.body)
177                .and_then(|bytes| Self::from_bytes(&bytes, &source)),
178            | (false, Scheme::File | Scheme::Unsupported, _) => Self::from_path(uri_to_path(&source)),
179        }
180    }
181    /// Convert document bytes to Markdown when supported, otherwise decode them as UTF-8.
182    pub fn from_bytes(bytes: &[u8], source: &str) -> ApiResult<Self> {
183        let mime = MimeType::from(source);
184        match DocumentFormat::try_from(&mime) {
185            | Ok(format) => to_markdown_bytes(bytes, Some(format)),
186            | Err(_) => String::from_utf8(bytes.to_vec()).map_err(color_eyre::Report::from),
187        }
188        .map(|content| Self::init().content(content).format(mime.file_type()).source(source).build())
189    }
190    /// Read a local document, converting supported formats to Markdown.
191    pub fn from_path(path: impl Into<PathBuf>) -> ApiResult<Self> {
192        let path = path.into();
193        let source = path.display().to_string();
194        let mime = MimeType::from(source.as_str());
195        let content = match DocumentFormat::try_from(&mime) {
196            | Ok(_) => to_markdown(path),
197            | Err(_) => read_file(path),
198        };
199        content.map(|content| Self::init().content(content).format(mime.file_type()).source(source).build())
200    }
201    /// Read and convert this document to GitHub-Flavored Markdown.
202    pub fn extract(&self) -> ApiResult<String> {
203        match self.content.is_empty() {
204            | true => Self::from_path(self.source.as_str()).map(|document| document.content),
205            | false => Ok(self.content.clone()),
206        }
207    }
208}
209/// Convert a supported document file to GitHub-Flavored Markdown.
210pub fn to_markdown(path: impl Into<PathBuf>) -> ApiResult<String> {
211    let path = path.into();
212    anydoc::to_markdown(&path).map_err(|why| eyre!("Failed to convert document {} - {why}", path.display()))
213}
214/// Convert supported document bytes to GitHub-Flavored Markdown.
215pub fn to_markdown_bytes(bytes: &[u8], format: Option<DocumentFormat>) -> ApiResult<String> {
216    anydoc::to_markdown_bytes(bytes, format.map(Format::from)).map_err(|why| eyre!("Failed to convert document - {why}"))
217}
218
219#[cfg(test)]
220mod tests;
acorn/io/document/mod.rs

acorn/io/document/
mod.rs
