1#[cfg(feature = "shell-accel")]
25pub mod accel;
26mod archive;
27mod email;
28#[cfg(feature = "ocr")]
29pub mod ocr;
30mod ooxml;
31mod pdf;
32mod sheet;
33mod textfmt;
34
35use anyhow::Result;
36use std::path::Path;
37use std::sync::{Arc, Mutex, OnceLock};
38
39pub const MAX_EXTRACTED_BYTES: usize = 32 * 1024 * 1024;
41pub const MAX_ARCHIVE_ENTRIES: usize = 2_000;
43pub const MAX_PDF_PAGES: u32 = 5_000;
44
45const MAX_STUCK_PARSERS: usize = 4;
51static STUCK_PARSERS: Mutex<Vec<std::thread::JoinHandle<()>>> = Mutex::new(Vec::new());
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub struct Capabilities {
60 pub max_file_bytes: u64,
63 pub max_pages: u32,
65 pub max_entries: usize,
67 pub streaming: bool,
70 pub thread_safe: bool,
72 pub requires_external: bool,
74}
75
76impl Default for Capabilities {
77 fn default() -> Self {
78 Self {
79 max_file_bytes: 64 * 1024 * 1024, max_pages: 5_000,
81 max_entries: 2_000,
82 streaming: false,
83 thread_safe: true,
84 requires_external: false,
85 }
86 }
87}
88
89#[derive(Debug, Clone)]
92pub struct ExtractedDocument {
93 pub path: String,
94 pub format: &'static str,
95 pub blocks: Vec<Block>,
96 pub warnings: Vec<String>,
99}
100
101#[derive(Debug, Clone, PartialEq, Eq)]
102pub struct Block {
103 pub text: String,
104 pub page: i64,
106 pub anchor: Option<String>,
109 pub section: Option<String>,
112 pub line_start: i64,
115 pub line_end: i64,
116}
117
118impl Block {
119 pub fn text(text: impl Into<String>) -> Self {
120 let text = text.into();
121 let lines = text.lines().count().max(1) as i64;
122 Self {
123 text,
124 page: 0,
125 anchor: None,
126 section: None,
127 line_start: 1,
128 line_end: lines,
129 }
130 }
131
132 pub fn anchored(text: impl Into<String>, anchor: impl Into<String>) -> Self {
133 let mut block = Self::text(text);
134 block.anchor = Some(anchor.into());
135 block
136 }
137}
138
139pub trait Extractor: Send + Sync {
144 fn name(&self) -> &'static str;
146
147 fn extensions(&self) -> &'static [&'static str];
149
150 fn sniff(&self, _bytes: &[u8]) -> bool {
152 false
153 }
154
155 fn capabilities(&self) -> Capabilities {
161 Capabilities::default()
162 }
163
164 fn can_handle(&self, name: &str, bytes: &[u8]) -> bool {
165 self.extensions().contains(&extension_of(name).as_str()) || self.sniff(bytes)
166 }
167
168 fn extract(&self, name: &str, bytes: &[u8]) -> Result<Option<Vec<ExtractedDocument>>>;
169}
170
171fn builtin() -> Vec<Arc<dyn Extractor>> {
172 vec![
173 Arc::new(pdf::Pdf),
174 Arc::new(ooxml::Docx),
175 Arc::new(ooxml::Pptx),
176 Arc::new(ooxml::Epub),
177 Arc::new(sheet::Sheet),
178 Arc::new(archive::Archive),
179 Arc::new(email::Email),
180 Arc::new(textfmt::Html),
181 Arc::new(textfmt::Plain),
183 ]
184}
185
186fn custom() -> &'static Mutex<Vec<Arc<dyn Extractor>>> {
187 static CUSTOM: OnceLock<Mutex<Vec<Arc<dyn Extractor>>>> = OnceLock::new();
188 CUSTOM.get_or_init(|| Mutex::new(Vec::new()))
189}
190
191pub fn register(extractor: Arc<dyn Extractor>) {
194 if let Ok(mut list) = custom().lock() {
195 list.push(extractor);
196 }
197}
198
199pub fn extractors() -> Vec<Arc<dyn Extractor>> {
201 let mut all = builtin();
202 if let Ok(list) = custom().lock() {
203 all.extend(list.iter().cloned());
204 }
205 all
206}
207
208pub fn supported_extensions() -> Vec<&'static str> {
210 let mut out: Vec<&'static str> = Vec::new();
211 for extractor in extractors() {
212 out.extend_from_slice(extractor.extensions());
213 }
214 out.sort_unstable();
215 out.dedup();
216 out
217}
218
219pub fn extension_of(path: &str) -> String {
220 Path::new(path)
221 .extension()
222 .and_then(|e| e.to_str())
223 .unwrap_or("")
224 .to_ascii_lowercase()
225}
226
227pub fn is_supported(path: &str) -> bool {
229 let ext = extension_of(path);
230 extractors()
231 .iter()
232 .any(|e| e.extensions().contains(&ext.as_str()))
233}
234
235pub fn extract_path(path: &Path) -> Result<Option<Vec<ExtractedDocument>>> {
238 let name = path.to_string_lossy().to_string();
239 if !is_supported(&name) {
240 return Ok(None);
241 }
242 let metadata = std::fs::symlink_metadata(path)?;
245 let mut max_allowed: u64 = 0;
246 for extractor in extractors() {
247 if extractor
248 .extensions()
249 .contains(&extension_of(&name).as_str())
250 {
251 max_allowed = max_allowed.max(extractor.capabilities().max_file_bytes);
252 }
253 }
254 if max_allowed == 0 || metadata.len() > max_allowed {
255 return Ok(None);
256 }
257 let bytes = std::fs::read(path)?;
258 extract_named(&name, &bytes, false)
259}
260
261pub fn extract_named(
264 name: &str,
265 bytes: &[u8],
266 _ocr_enabled: bool,
267) -> Result<Option<Vec<ExtractedDocument>>> {
268 #[cfg(feature = "shell-accel")]
270 {
271 if let Some(path) = std::path::Path::new(name).extension() {
272 if let Some(ext) = path.to_str() {
273 if let Ok(Some(docs)) =
274 crate::core::extract::accel::try_accelerators(std::path::Path::new(name), ext)
275 {
276 if !docs.is_empty() {
277 return Ok(Some(docs));
278 }
279 }
280 }
281 }
282 }
283
284 for extractor in extractors() {
285 if !extractor.can_handle(name, bytes) {
286 continue;
287 }
288 let caps = extractor.capabilities();
289
290 if bytes.len() as u64 > caps.max_file_bytes {
292 return Ok(Some(single(
293 name,
294 extractor.name(),
295 Vec::new(),
296 vec![format!(
297 "file size {} exceeds maximum {} bytes",
298 bytes.len(),
299 caps.max_file_bytes
300 )],
301 )));
302 }
303
304 {
312 let mut stuck = STUCK_PARSERS.lock().unwrap_or_else(|e| e.into_inner());
313 stuck.retain(|h| !h.is_finished());
314 if stuck.len() >= MAX_STUCK_PARSERS {
315 return Ok(Some(single(
316 name,
317 extractor.name(),
318 Vec::new(),
319 vec![format!(
320 "skipped: {MAX_STUCK_PARSERS} parsers are stalled; restart the indexer"
321 )],
322 )));
323 }
324 }
325
326 let extractor_clone = extractor.clone();
327 let name_owned = name.to_string();
328 let bytes_owned = bytes.to_vec();
329 let (tx, rx) = std::sync::mpsc::channel();
330 let handle = std::thread::spawn(move || {
331 let _ = tx.send(extractor_clone.extract(&name_owned, &bytes_owned));
332 });
333 let result = rx.recv_timeout(std::time::Duration::from_secs(5));
334
335 let mut docs = match result {
336 Ok(Ok(Some(docs))) => {
337 let _ = handle.join();
338 docs
339 }
340 Ok(Ok(None)) => {
341 let _ = handle.join();
342 continue;
343 }
344 Ok(Err(err)) => {
345 let _ = handle.join();
346 return Ok(Some(single(
347 name,
348 extractor.name(),
349 Vec::new(),
350 vec![format!("extraction error: {err}")],
351 )));
352 }
353 Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
354 tracing::warn!("{}: extraction timed out after 5 seconds", name);
357 STUCK_PARSERS
358 .lock()
359 .unwrap_or_else(|e| e.into_inner())
360 .push(handle);
361 return Ok(Some(single(
362 name,
363 extractor.name(),
364 Vec::new(),
365 vec!["extraction timed out after 5 seconds".to_string()],
366 )));
367 }
368 Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
369 let _ = handle.join();
370 return Ok(Some(single(
371 name,
372 extractor.name(),
373 Vec::new(),
374 vec!["extraction task panicked".to_string()],
375 )));
376 }
377 };
378
379 for doc in &mut docs {
380 doc.format = extractor.name();
381 if doc.path.is_empty() {
382 doc.path = name.to_string();
383 }
384 doc.blocks = finish_blocks(std::mem::take(&mut doc.blocks));
385 }
386 docs.retain(|doc| !doc.blocks.is_empty());
387 if docs.is_empty() {
388 return Ok(None);
390 }
391 return Ok(Some(docs));
392 }
393 Ok(None)
394}
395
396pub(crate) fn finish_blocks(mut blocks: Vec<Block>) -> Vec<Block> {
398 let mut total = 0usize;
399 blocks.retain(|b| !b.text.trim().is_empty());
400 for block in &mut blocks {
401 if block.text.len() > MAX_EXTRACTED_BYTES {
402 let mut cut = MAX_EXTRACTED_BYTES;
403 while cut > 0 && !block.text.is_char_boundary(cut) {
404 cut -= 1;
405 }
406 block.text.truncate(cut);
407 }
408 total += block.text.len();
409 if total > MAX_EXTRACTED_BYTES {
410 break;
411 }
412 }
413 blocks
414}
415
416pub(crate) fn single(
418 name: &str,
419 format: &'static str,
420 blocks: Vec<Block>,
421 warnings: Vec<String>,
422) -> Vec<ExtractedDocument> {
423 vec![ExtractedDocument {
424 path: name.to_string(),
425 format,
426 blocks,
427 warnings,
428 }]
429}