1use serde::{Deserialize, Serialize};
24
25mod binary;
26pub mod detect;
27mod node;
28pub mod text;
29#[cfg(feature = "zip-formats")]
30mod zipfmt;
31
32pub use detect::Format;
33pub use node::{DocLimits, DocumentExtractorNode, KIND, descriptor, register};
34
35#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
37#[serde(rename_all = "lowercase")]
38pub enum OutputFormat {
39 #[default]
41 Text,
42 Markdown,
44}
45
46#[derive(Clone, Debug, PartialEq, Eq)]
48pub struct Options {
49 pub format: OutputFormat,
50 pub max_output_chars: usize,
52 pub max_pages: usize,
54 pub max_sheets: usize,
56 pub max_rows: usize,
58 pub max_cols: usize,
60 pub max_cells: usize,
63 pub max_zip_entries: usize,
65 pub max_decompressed_bytes: u64,
67}
68
69impl Default for Options {
70 fn default() -> Self {
71 Self {
72 format: OutputFormat::Text,
73 max_output_chars: 5_000_000,
74 max_pages: 2_000,
75 max_sheets: 100,
76 max_rows: 200_000,
77 max_cols: 2_048,
78 max_cells: 2_000_000,
79 max_zip_entries: 10_000,
80 max_decompressed_bytes: 256 * 1024 * 1024,
81 }
82 }
83}
84
85impl Options {
86 pub fn with_format(mut self, format: OutputFormat) -> Self {
87 self.format = format;
88 self
89 }
90
91 pub fn with_max_output_chars(mut self, max: usize) -> Self {
92 self.max_output_chars = max;
93 self
94 }
95}
96
97#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
99#[non_exhaustive]
100pub enum DocError {
101 #[error("unsupported document format: {detected}")]
103 Unsupported { detected: String },
104 #[error("{format} documents are not supported by this build (enable the `{feature}` feature)")]
106 FeatureDisabled { format: Format, feature: &'static str },
107 #[error("malformed {format} document: {message}")]
109 Malformed { format: String, message: String },
110 #[error("document exceeds limits: {0}")]
112 LimitExceeded(String),
113}
114
115impl DocError {
116 #[cfg_attr(
117 not(any(
118 feature = "pdf",
119 feature = "spreadsheet",
120 feature = "zip-formats",
121 feature = "email",
122 feature = "tokio"
123 )),
124 allow(dead_code)
125 )]
126 pub(crate) fn malformed(format: &str, err: impl std::fmt::Display) -> Self {
127 DocError::Malformed { format: format.to_owned(), message: err.to_string() }
128 }
129
130 pub fn kind(&self) -> &'static str {
132 match self {
133 DocError::Unsupported { .. } | DocError::FeatureDisabled { .. } => "unsupported_format",
134 DocError::Malformed { .. } => "malformed_document",
135 DocError::LimitExceeded(_) => "limit_exceeded",
136 }
137 }
138}
139
140#[derive(Clone, Debug, PartialEq, Eq)]
142pub struct Extraction {
143 pub text: String,
144 pub format: Format,
145 pub truncated: bool,
147}
148
149pub fn extract(bytes: &[u8], mime: Option<&str>, filename: Option<&str>, opts: &Options) -> Result<String, DocError> {
155 extract_detailed(bytes, mime, filename, opts).map(|e| e.text)
156}
157
158pub fn extract_detailed(
160 bytes: &[u8],
161 mime: Option<&str>,
162 filename: Option<&str>,
163 opts: &Options,
164) -> Result<Extraction, DocError> {
165 let format = resolve_format(bytes, mime, filename, opts)?;
166 let text = extract_as(format, bytes, opts)?;
167 let (text, truncated) = truncate_chars(text, opts.max_output_chars);
168 Ok(Extraction { text, format, truncated })
169}
170
171pub fn resolve_format(
173 bytes: &[u8],
174 mime: Option<&str>,
175 filename: Option<&str>,
176 opts: &Options,
177) -> Result<Format, DocError> {
178 let claimed = mime
179 .and_then(detect::from_mime)
180 .or_else(|| filename.and_then(detect::extension).and_then(detect::from_extension));
181 if let Some(f) = claimed {
182 if !f.is_textual() {
183 return Ok(f);
184 }
185 if detect::sniff(bytes).is_some_and(|s| !s.is_textual()) || detect::is_zip(bytes) {
188 return sniffed_binary(bytes, opts);
189 }
190 if !bytes.is_empty() && !detect::looks_textual(bytes) {
191 return Err(DocError::Unsupported { detected: format!("binary data labelled as {f}") });
192 }
193 return Ok(f);
194 }
195 if let Some(f) = detect::sniff(bytes) {
196 if f == Format::Xls && !detect::ole_is_workbook(bytes) {
197 return Err(DocError::Unsupported {
198 detected: "legacy OLE document (.doc / .ppt / .msg) — only spreadsheets are supported".into(),
199 });
200 }
201 return Ok(f);
202 }
203 if detect::is_zip(bytes) {
204 #[cfg(feature = "zip-formats")]
205 if let Some(f) = zipfmt::identify(bytes, opts)? {
206 return Ok(f);
207 }
208 return Err(DocError::Unsupported { detected: "zip archive".into() });
209 }
210 let _ = opts;
211 if !bytes.is_empty() && detect::looks_textual(bytes) {
212 return Ok(Format::PlainText);
213 }
214 let detected = match (mime, filename.and_then(detect::extension)) {
215 (Some(m), _) => m.to_owned(),
216 (None, Some(ext)) => format!(".{ext}"),
217 (None, None) if bytes.is_empty() => "empty file".to_owned(),
218 (None, None) => "binary data".to_owned(),
219 };
220 Err(DocError::Unsupported { detected })
221}
222
223fn sniffed_binary(bytes: &[u8], opts: &Options) -> Result<Format, DocError> {
225 resolve_format(bytes, None, None, opts)
226}
227
228pub fn extract_as(format: Format, bytes: &[u8], opts: &Options) -> Result<String, DocError> {
230 #[allow(unused_variables)]
231 let disabled = |feature: &'static str| DocError::FeatureDisabled { format, feature };
232 match format {
233 Format::PlainText | Format::Markdown | Format::Json | Format::Yaml | Format::Xml => {
234 Ok(text::decode(bytes).trim_end().to_owned())
235 }
236 Format::Csv | Format::Tsv => {
237 let delimiter = if format == Format::Csv { ',' } else { '\t' };
238 let rows = text::parse_delimited(&text::decode(bytes), delimiter);
239 if rows.len() > opts.max_rows {
240 return Err(DocError::LimitExceeded(format!("{} rows (limit {})", rows.len(), opts.max_rows)));
241 }
242 text::render_table(&rows, opts)
243 }
244 Format::Html => Ok(text::html_to_text(&text::decode(bytes), opts.format)),
245 Format::Vtt | Format::Srt => Ok(text::subtitles(&text::decode(bytes))),
246 Format::Pdf => {
247 #[cfg(feature = "pdf")]
248 return binary::pdf(bytes, opts);
249 #[cfg(not(feature = "pdf"))]
250 return Err(disabled("pdf"));
251 }
252 Format::Docx => {
253 #[cfg(feature = "office")]
254 return zipfmt::docx(bytes, opts);
255 #[cfg(not(feature = "office"))]
256 return Err(disabled("office"));
257 }
258 Format::Pptx => {
259 #[cfg(feature = "office")]
260 return zipfmt::pptx(bytes, opts);
261 #[cfg(not(feature = "office"))]
262 return Err(disabled("office"));
263 }
264 Format::Epub => {
265 #[cfg(feature = "epub")]
266 return zipfmt::epub(bytes, opts);
267 #[cfg(not(feature = "epub"))]
268 return Err(disabled("epub"));
269 }
270 Format::Xlsx | Format::Xls | Format::Ods => {
271 #[cfg(feature = "spreadsheet")]
272 return binary::spreadsheet(bytes, format, opts);
273 #[cfg(not(feature = "spreadsheet"))]
274 return Err(disabled("spreadsheet"));
275 }
276 Format::Eml => {
277 #[cfg(feature = "email")]
278 return binary::email(bytes, opts);
279 #[cfg(not(feature = "email"))]
280 return Err(disabled("email"));
281 }
282 }
283}
284
285fn truncate_chars(mut text: String, max: usize) -> (String, bool) {
287 match text.char_indices().nth(max) {
288 Some((byte_index, _)) => {
289 text.truncate(byte_index);
290 (text, true)
291 }
292 None => (text, false),
293 }
294}
295
296#[cfg(test)]
297mod tests {
298 use super::*;
299
300 #[test]
301 fn truncation_respects_char_boundaries() {
302 let (t, cut) = truncate_chars("héllo".to_owned(), 2);
303 assert_eq!((t.as_str(), cut), ("hé", true));
304 let (t, cut) = truncate_chars("hi".to_owned(), 2);
305 assert_eq!((t.as_str(), cut), ("hi", false));
306 }
307
308 #[test]
309 fn format_resolution_order() {
310 let opts = Options::default();
311 assert_eq!(resolve_format(b"x", Some("text/csv"), Some("a.json"), &opts), Ok(Format::Csv));
313 assert_eq!(resolve_format(b"x", Some("application/octet-stream"), Some("a.md"), &opts), Ok(Format::Markdown));
315 assert_eq!(resolve_format(b"%PDF-1.7", None, None, &opts), Ok(Format::Pdf));
317 assert_eq!(resolve_format(b"just words", None, None, &opts), Ok(Format::PlainText));
318 assert!(matches!(resolve_format(&[0, 1, 2, 3], None, Some("x.bin"), &opts), Err(DocError::Unsupported { .. })));
319 }
320
321 #[test]
322 fn csv_markdown_and_text() {
323 let csv = b"name,qty\napple,3\n";
324 let md = extract(csv, Some("text/csv"), None, &Options::default().with_format(OutputFormat::Markdown));
325 assert_eq!(md.as_deref(), Ok("| name | qty |\n| --- | --- |\n| apple | 3 |"));
326 let txt = extract(csv, None, Some("f.csv"), &Options::default());
327 assert_eq!(txt.as_deref(), Ok("name\tqty\napple\t3"));
328 }
329}