1#[cfg(feature = "shell-accel")]
25pub mod accel;
26mod archive;
27mod email;
28#[cfg(feature = "ocr")]
29pub mod ocr;
30mod ooxml;
31mod pdf;
32mod sheet;
33mod textfmt;
34
35use anyhow::Result;
36use std::path::Path;
37use std::sync::{Arc, Mutex, OnceLock};
38
39pub const MAX_EXTRACTED_BYTES: usize = 32 * 1024 * 1024;
41pub const MAX_ARCHIVE_ENTRIES: usize = 2_000;
43pub const MAX_PDF_PAGES: u32 = 5_000;
44
45const MAX_STUCK_PARSERS: usize = 4;
51static STUCK_PARSERS: Mutex<Vec<std::thread::JoinHandle<()>>> = Mutex::new(Vec::new());
52
53fn reap_stuck_parsers() {
56 let mut stuck = STUCK_PARSERS.lock().unwrap_or_else(|e| e.into_inner());
57 let mut i = 0;
58 while i < stuck.len() {
59 if stuck[i].is_finished() {
60 let handle = stuck.remove(i);
61 let _ = handle.join();
62 } else {
63 i += 1;
64 }
65 }
66 drop(stuck);
67}
68
69fn ensure_reaper() {
72 static REAPER: std::sync::OnceLock<std::thread::JoinHandle<()>> = std::sync::OnceLock::new();
73 let _ = REAPER.get_or_init(|| {
74 std::thread::spawn(move || loop {
75 std::thread::sleep(std::time::Duration::from_secs(5));
76 reap_stuck_parsers();
77 })
78 });
79}
80
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
87pub struct Capabilities {
88 pub max_file_bytes: u64,
91 pub max_pages: u32,
93 pub max_entries: usize,
95 pub streaming: bool,
98 pub thread_safe: bool,
100 pub requires_external: bool,
102}
103
104impl Default for Capabilities {
105 fn default() -> Self {
106 Self {
107 max_file_bytes: 64 * 1024 * 1024, max_pages: 5_000,
109 max_entries: 2_000,
110 streaming: false,
111 thread_safe: true,
112 requires_external: false,
113 }
114 }
115}
116
117#[derive(Debug, Clone)]
120pub struct ExtractedDocument {
121 pub path: String,
122 pub format: &'static str,
123 pub blocks: Vec<Block>,
124 pub warnings: Vec<String>,
127}
128
129#[derive(Debug, Clone, PartialEq, Eq)]
130pub struct Block {
131 pub text: String,
132 pub page: i64,
134 pub anchor: Option<String>,
137 pub section: Option<String>,
140 pub line_start: i64,
143 pub line_end: i64,
144}
145
146impl Block {
147 pub fn text(text: impl Into<String>) -> Self {
148 let text = text.into();
149 let lines = text.lines().count().max(1) as i64;
150 Self {
151 text,
152 page: 0,
153 anchor: None,
154 section: None,
155 line_start: 1,
156 line_end: lines,
157 }
158 }
159
160 pub fn anchored(text: impl Into<String>, anchor: impl Into<String>) -> Self {
161 let mut block = Self::text(text);
162 block.anchor = Some(anchor.into());
163 block
164 }
165}
166
167pub trait Extractor: Send + Sync {
172 fn name(&self) -> &'static str;
174
175 fn extensions(&self) -> &'static [&'static str];
177
178 fn sniff(&self, _bytes: &[u8]) -> bool {
180 false
181 }
182
183 fn capabilities(&self) -> Capabilities {
189 Capabilities::default()
190 }
191
192 fn can_handle(&self, name: &str, bytes: &[u8]) -> bool {
193 self.extensions().contains(&extension_of(name).as_str()) || self.sniff(bytes)
194 }
195
196 fn extract(&self, name: &str, bytes: &[u8]) -> Result<Option<Vec<ExtractedDocument>>>;
197}
198
199fn builtin() -> Vec<Arc<dyn Extractor>> {
200 vec![
201 Arc::new(pdf::Pdf),
202 Arc::new(ooxml::Docx),
203 Arc::new(ooxml::Pptx),
204 Arc::new(ooxml::Epub),
205 Arc::new(sheet::Sheet),
206 Arc::new(archive::Archive),
207 Arc::new(email::Email),
208 Arc::new(textfmt::Html),
209 Arc::new(textfmt::Plain),
211 ]
212}
213
214fn custom() -> &'static Mutex<Vec<Arc<dyn Extractor>>> {
215 static CUSTOM: OnceLock<Mutex<Vec<Arc<dyn Extractor>>>> = OnceLock::new();
216 CUSTOM.get_or_init(|| Mutex::new(Vec::new()))
217}
218
219pub fn register(extractor: Arc<dyn Extractor>) {
222 if let Ok(mut list) = custom().lock() {
223 list.push(extractor);
224 }
225}
226
227pub fn extractors() -> Vec<Arc<dyn Extractor>> {
229 let mut all = builtin();
230 if let Ok(list) = custom().lock() {
231 all.extend(list.iter().cloned());
232 }
233 all
234}
235
236pub fn supported_extensions() -> Vec<&'static str> {
238 let mut out: Vec<&'static str> = Vec::new();
239 for extractor in extractors() {
240 out.extend_from_slice(extractor.extensions());
241 }
242 out.sort_unstable();
243 out.dedup();
244 out
245}
246
247pub fn extension_of(path: &str) -> String {
248 Path::new(path)
249 .extension()
250 .and_then(|e| e.to_str())
251 .unwrap_or("")
252 .to_ascii_lowercase()
253}
254
255pub fn is_supported(path: &str) -> bool {
257 let ext = extension_of(path);
258 extractors()
259 .iter()
260 .any(|e| e.extensions().contains(&ext.as_str()))
261}
262
263pub fn extract_path(path: &Path) -> Result<Option<Vec<ExtractedDocument>>> {
266 let name = path.to_string_lossy().to_string();
267 if !is_supported(&name) {
268 return Ok(None);
269 }
270 let metadata = std::fs::symlink_metadata(path)?;
273 let mut max_allowed: u64 = 0;
274 for extractor in extractors() {
275 if extractor
276 .extensions()
277 .contains(&extension_of(&name).as_str())
278 {
279 max_allowed = max_allowed.max(extractor.capabilities().max_file_bytes);
280 }
281 }
282 if max_allowed == 0 || metadata.len() > max_allowed {
283 return Ok(None);
284 }
285 let bytes = std::fs::read(path)?;
286 extract_named(&name, &bytes, false)
287}
288
289pub fn extract_named(
292 name: &str,
293 bytes: &[u8],
294 _ocr_enabled: bool,
295) -> Result<Option<Vec<ExtractedDocument>>> {
296 #[cfg(feature = "shell-accel")]
298 {
299 if let Some(path) = std::path::Path::new(name).extension() {
300 if let Some(ext) = path.to_str() {
301 if let Ok(Some(docs)) =
302 crate::core::extract::accel::try_accelerators(std::path::Path::new(name), ext)
303 {
304 if !docs.is_empty() {
305 return Ok(Some(docs));
306 }
307 }
308 }
309 }
310 }
311
312 for extractor in extractors() {
313 if !extractor.can_handle(name, bytes) {
314 continue;
315 }
316 let caps = extractor.capabilities();
317
318 if bytes.len() as u64 > caps.max_file_bytes {
320 return Ok(Some(single(
321 name,
322 extractor.name(),
323 Vec::new(),
324 vec![format!(
325 "file size {} exceeds maximum {} bytes",
326 bytes.len(),
327 caps.max_file_bytes
328 )],
329 )));
330 }
331
332 {
340 reap_stuck_parsers();
341 ensure_reaper();
342 let stuck = STUCK_PARSERS.lock().unwrap_or_else(|e| e.into_inner());
343 if stuck.len() >= MAX_STUCK_PARSERS {
344 return Ok(Some(single(
345 name,
346 extractor.name(),
347 Vec::new(),
348 vec![format!(
349 "skipped: {MAX_STUCK_PARSERS} parsers are stalled; restart the indexer"
350 )],
351 )));
352 }
353 drop(stuck);
354 }
355
356 let extractor_clone = extractor.clone();
357 let name_owned = name.to_string();
358 let bytes_owned = bytes.to_vec();
359 let (tx, rx) = std::sync::mpsc::channel();
360 let handle = std::thread::spawn(move || {
361 let _ = tx.send(extractor_clone.extract(&name_owned, &bytes_owned));
362 });
363 let result = rx.recv_timeout(std::time::Duration::from_secs(5));
364
365 let mut docs = match result {
366 Ok(Ok(Some(docs))) => {
367 let _ = handle.join();
368 docs
369 }
370 Ok(Ok(None)) => {
371 let _ = handle.join();
372 continue;
373 }
374 Ok(Err(err)) => {
375 let _ = handle.join();
376 return Ok(Some(single(
377 name,
378 extractor.name(),
379 Vec::new(),
380 vec![format!("extraction error: {err}")],
381 )));
382 }
383 Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
384 tracing::warn!("{}: extraction timed out after 5 seconds", name);
388 ensure_reaper();
389 STUCK_PARSERS
390 .lock()
391 .unwrap_or_else(|e| e.into_inner())
392 .push(handle);
393 return Ok(Some(single(
394 name,
395 extractor.name(),
396 Vec::new(),
397 vec!["extraction timed out after 5 seconds".to_string()],
398 )));
399 }
400 Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
401 let _ = handle.join();
402 return Ok(Some(single(
403 name,
404 extractor.name(),
405 Vec::new(),
406 vec!["extraction task panicked".to_string()],
407 )));
408 }
409 };
410
411 for doc in &mut docs {
412 doc.format = extractor.name();
413 if doc.path.is_empty() {
414 doc.path = name.to_string();
415 }
416 doc.blocks = finish_blocks(std::mem::take(&mut doc.blocks));
417 }
418 docs.retain(|doc| !doc.blocks.is_empty());
419 if docs.is_empty() {
420 return Ok(None);
422 }
423 return Ok(Some(docs));
424 }
425 Ok(None)
426}
427
428pub(crate) fn finish_blocks(mut blocks: Vec<Block>) -> Vec<Block> {
430 let mut total = 0usize;
431 blocks.retain(|b| !b.text.trim().is_empty());
432 for block in &mut blocks {
433 if block.text.len() > MAX_EXTRACTED_BYTES {
434 let mut cut = MAX_EXTRACTED_BYTES;
435 while cut > 0 && !block.text.is_char_boundary(cut) {
436 cut -= 1;
437 }
438 block.text.truncate(cut);
439 }
440 total += block.text.len();
441 if total > MAX_EXTRACTED_BYTES {
442 break;
443 }
444 }
445 blocks
446}
447
448pub(crate) fn single(
450 name: &str,
451 format: &'static str,
452 blocks: Vec<Block>,
453 warnings: Vec<String>,
454) -> Vec<ExtractedDocument> {
455 vec![ExtractedDocument {
456 path: name.to_string(),
457 format,
458 blocks,
459 warnings,
460 }]
461}