#[cfg(feature = "shell-accel")]
pub mod accel;
mod archive;
mod email;
#[cfg(feature = "ocr")]
pub mod ocr;
mod ooxml;
mod pdf;
mod sheet;
mod textfmt;
use anyhow::Result;
use std::path::Path;
use std::sync::{Arc, Mutex, OnceLock};
pub const MAX_EXTRACTED_BYTES: usize = 32 * 1024 * 1024;
pub const MAX_ARCHIVE_ENTRIES: usize = 2_000;
pub const MAX_PDF_PAGES: u32 = 5_000;
const MAX_STUCK_PARSERS: usize = 4;
static STUCK_PARSERS: Mutex<Vec<std::thread::JoinHandle<()>>> = Mutex::new(Vec::new());
fn reap_stuck_parsers() {
let mut stuck = STUCK_PARSERS.lock().unwrap_or_else(|e| e.into_inner());
let mut i = 0;
while i < stuck.len() {
if stuck[i].is_finished() {
let handle = stuck.remove(i);
let _ = handle.join();
} else {
i += 1;
}
}
drop(stuck);
}
fn ensure_reaper() {
static REAPER: std::sync::OnceLock<std::thread::JoinHandle<()>> = std::sync::OnceLock::new();
let _ = REAPER.get_or_init(|| {
std::thread::spawn(move || loop {
std::thread::sleep(std::time::Duration::from_secs(5));
reap_stuck_parsers();
})
});
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Capabilities {
pub max_file_bytes: u64,
pub max_pages: u32,
pub max_entries: usize,
pub streaming: bool,
pub thread_safe: bool,
pub requires_external: bool,
}
impl Default for Capabilities {
fn default() -> Self {
Self {
max_file_bytes: 64 * 1024 * 1024, max_pages: 5_000,
max_entries: 2_000,
streaming: false,
thread_safe: true,
requires_external: false,
}
}
}
#[derive(Debug, Clone)]
pub struct ExtractedDocument {
pub path: String,
pub format: &'static str,
pub blocks: Vec<Block>,
pub warnings: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Block {
pub text: String,
pub page: i64,
pub anchor: Option<String>,
pub section: Option<String>,
pub line_start: i64,
pub line_end: i64,
}
impl Block {
pub fn text(text: impl Into<String>) -> Self {
let text = text.into();
let lines = text.lines().count().max(1) as i64;
Self {
text,
page: 0,
anchor: None,
section: None,
line_start: 1,
line_end: lines,
}
}
pub fn anchored(text: impl Into<String>, anchor: impl Into<String>) -> Self {
let mut block = Self::text(text);
block.anchor = Some(anchor.into());
block
}
}
pub trait Extractor: Send + Sync {
fn name(&self) -> &'static str;
fn extensions(&self) -> &'static [&'static str];
fn sniff(&self, _bytes: &[u8]) -> bool {
false
}
fn capabilities(&self) -> Capabilities {
Capabilities::default()
}
fn can_handle(&self, name: &str, bytes: &[u8]) -> bool {
self.extensions().contains(&extension_of(name).as_str()) || self.sniff(bytes)
}
fn extract(&self, name: &str, bytes: &[u8]) -> Result<Option<Vec<ExtractedDocument>>>;
}
fn builtin() -> Vec<Arc<dyn Extractor>> {
vec![
Arc::new(pdf::Pdf),
Arc::new(ooxml::Docx),
Arc::new(ooxml::Pptx),
Arc::new(ooxml::Epub),
Arc::new(sheet::Sheet),
Arc::new(archive::Archive),
Arc::new(email::Email),
Arc::new(textfmt::Html),
Arc::new(textfmt::Plain),
]
}
fn custom() -> &'static Mutex<Vec<Arc<dyn Extractor>>> {
static CUSTOM: OnceLock<Mutex<Vec<Arc<dyn Extractor>>>> = OnceLock::new();
CUSTOM.get_or_init(|| Mutex::new(Vec::new()))
}
pub fn register(extractor: Arc<dyn Extractor>) {
if let Ok(mut list) = custom().lock() {
list.push(extractor);
}
}
pub fn extractors() -> Vec<Arc<dyn Extractor>> {
let mut all = builtin();
if let Ok(list) = custom().lock() {
all.extend(list.iter().cloned());
}
all
}
pub fn supported_extensions() -> Vec<&'static str> {
let mut out: Vec<&'static str> = Vec::new();
for extractor in extractors() {
out.extend_from_slice(extractor.extensions());
}
out.sort_unstable();
out.dedup();
out
}
pub fn extension_of(path: &str) -> String {
Path::new(path)
.extension()
.and_then(|e| e.to_str())
.unwrap_or("")
.to_ascii_lowercase()
}
pub fn is_supported(path: &str) -> bool {
let ext = extension_of(path);
extractors()
.iter()
.any(|e| e.extensions().contains(&ext.as_str()))
}
pub fn extract_path(path: &Path) -> Result<Option<Vec<ExtractedDocument>>> {
let name = path.to_string_lossy().to_string();
if !is_supported(&name) {
return Ok(None);
}
let metadata = std::fs::symlink_metadata(path)?;
let mut max_allowed: u64 = 0;
for extractor in extractors() {
if extractor
.extensions()
.contains(&extension_of(&name).as_str())
{
max_allowed = max_allowed.max(extractor.capabilities().max_file_bytes);
}
}
if max_allowed == 0 || metadata.len() > max_allowed {
return Ok(None);
}
let bytes = std::fs::read(path)?;
extract_named(&name, &bytes, false)
}
pub fn extract_named(
name: &str,
bytes: &[u8],
_ocr_enabled: bool,
) -> Result<Option<Vec<ExtractedDocument>>> {
#[cfg(feature = "shell-accel")]
{
if let Some(path) = std::path::Path::new(name).extension() {
if let Some(ext) = path.to_str() {
if let Ok(Some(docs)) =
crate::core::extract::accel::try_accelerators(std::path::Path::new(name), ext)
{
if !docs.is_empty() {
return Ok(Some(docs));
}
}
}
}
}
for extractor in extractors() {
if !extractor.can_handle(name, bytes) {
continue;
}
let caps = extractor.capabilities();
if bytes.len() as u64 > caps.max_file_bytes {
return Ok(Some(single(
name,
extractor.name(),
Vec::new(),
vec![format!(
"file size {} exceeds maximum {} bytes",
bytes.len(),
caps.max_file_bytes
)],
)));
}
{
reap_stuck_parsers();
ensure_reaper();
let stuck = STUCK_PARSERS.lock().unwrap_or_else(|e| e.into_inner());
if stuck.len() >= MAX_STUCK_PARSERS {
return Ok(Some(single(
name,
extractor.name(),
Vec::new(),
vec![format!(
"skipped: {MAX_STUCK_PARSERS} parsers are stalled; restart the indexer"
)],
)));
}
drop(stuck);
}
let extractor_clone = extractor.clone();
let name_owned = name.to_string();
let bytes_owned = bytes.to_vec();
let (tx, rx) = std::sync::mpsc::channel();
let handle = std::thread::spawn(move || {
let _ = tx.send(extractor_clone.extract(&name_owned, &bytes_owned));
});
let result = rx.recv_timeout(std::time::Duration::from_secs(5));
let mut docs = match result {
Ok(Ok(Some(docs))) => {
let _ = handle.join();
docs
}
Ok(Ok(None)) => {
let _ = handle.join();
continue;
}
Ok(Err(err)) => {
let _ = handle.join();
return Ok(Some(single(
name,
extractor.name(),
Vec::new(),
vec![format!("extraction error: {err}")],
)));
}
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
tracing::warn!("{}: extraction timed out after 5 seconds", name);
ensure_reaper();
STUCK_PARSERS
.lock()
.unwrap_or_else(|e| e.into_inner())
.push(handle);
return Ok(Some(single(
name,
extractor.name(),
Vec::new(),
vec!["extraction timed out after 5 seconds".to_string()],
)));
}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
let _ = handle.join();
return Ok(Some(single(
name,
extractor.name(),
Vec::new(),
vec!["extraction task panicked".to_string()],
)));
}
};
for doc in &mut docs {
doc.format = extractor.name();
if doc.path.is_empty() {
doc.path = name.to_string();
}
doc.blocks = finish_blocks(std::mem::take(&mut doc.blocks));
}
docs.retain(|doc| !doc.blocks.is_empty());
if docs.is_empty() {
return Ok(None);
}
return Ok(Some(docs));
}
Ok(None)
}
pub(crate) fn finish_blocks(mut blocks: Vec<Block>) -> Vec<Block> {
let mut total = 0usize;
blocks.retain(|b| !b.text.trim().is_empty());
for block in &mut blocks {
if block.text.len() > MAX_EXTRACTED_BYTES {
let mut cut = MAX_EXTRACTED_BYTES;
while cut > 0 && !block.text.is_char_boundary(cut) {
cut -= 1;
}
block.text.truncate(cut);
}
total += block.text.len();
if total > MAX_EXTRACTED_BYTES {
break;
}
}
blocks
}
pub(crate) fn single(
name: &str,
format: &'static str,
blocks: Vec<Block>,
warnings: Vec<String>,
) -> Vec<ExtractedDocument> {
vec![ExtractedDocument {
path: name.to_string(),
format,
blocks,
warnings,
}]
}