kreuzberg 4.5.4

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 88+ formats with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

//! Helper functions and utilities for extraction operations.
//!
//! This module provides shared utilities used across extraction modules.

use crate::plugins::DocumentExtractor;
use crate::types::{ErrorMetadata, ExtractionResult, Metadata};
use crate::utils::{PoolSizeHint, estimate_pool_size};
use crate::{KreuzbergError, Result};
use std::borrow::Cow;
use std::sync::Arc;

/// Get an extractor from the registry.
///
/// This function acquires the registry read lock and retrieves the appropriate
/// extractor for the given MIME type.
///
/// # Performance
///
/// RwLock read + HashMap lookup is ~100ns, fast enough without caching.
/// Removed thread-local cache to avoid Tokio work-stealing scheduler issues.
pub(in crate::core::extractor) fn get_extractor(mime_type: &str) -> Result<Arc<dyn DocumentExtractor>> {
    let registry = crate::plugins::registry::get_document_extractor_registry();
    let registry_read = registry
        .read()
        .map_err(|e| KreuzbergError::Other(format!("Document extractor registry lock poisoned: {}", e)))?;
    registry_read.get(mime_type)
}

/// Get optimal pool sizing hint for a document.
///
/// This function calculates recommended pool sizes based on the document's
/// file size and MIME type. The hint can be used to create appropriately
/// sized thread pools for extraction, reducing memory waste from over-allocation.
///
/// # Arguments
///
/// * `file_size` - The size of the file in bytes
/// * `mime_type` - The MIME type of the document
///
/// # Returns
///
/// A `PoolSizeHint` with recommended pool configurations
///
/// # Example
///
/// ```rust,ignore
/// use kreuzberg::core::extractor::get_pool_sizing_hint;
///
/// let hint = get_pool_sizing_hint(5_000_000, "application/pdf");
/// println!("Recommended string buffers: {}", hint.string_buffer_count);
/// ```
/// Build an error `ExtractionResult` for failed batch items.
///
/// Used by both tokio-based batch functions and WASM synchronous fallbacks
/// to construct a uniform error result.
pub(crate) fn error_extraction_result(e: &KreuzbergError, elapsed_ms: Option<u64>) -> ExtractionResult {
    let metadata = Metadata {
        error: Some(ErrorMetadata {
            error_type: format!("{:?}", e),
            message: e.to_string(),
        }),
        extraction_duration_ms: elapsed_ms,
        ..Default::default()
    };

    ExtractionResult {
        content: format!("Error: {}", e),
        mime_type: Cow::Borrowed("text/plain"),
        metadata,
        tables: vec![],
        detected_languages: None,
        chunks: None,
        images: None,
        djot_content: None,
        pages: None,
        elements: None,
        ocr_elements: None,
        document: None,
        #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
        extracted_keywords: None,
        quality_score: None,
        processing_warnings: Vec::new(),
        annotations: None,
    }
}

#[inline]
pub fn get_pool_sizing_hint(file_size: u64, mime_type: &str) -> PoolSizeHint {
    estimate_pool_size(file_size, mime_type)
}