xberg 1.0.7

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 101 formats and 306 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

//! Helper functions and utilities for extraction operations.
//!
//! This module provides shared utilities used across extraction modules.

use crate::Result;
#[cfg(all(feature = "tokio-runtime", not(target_arch = "wasm32")))]
use crate::XbergError;
use crate::plugins::InternalDocumentExtractor;
#[cfg(all(feature = "tokio-runtime", not(target_arch = "wasm32")))]
use crate::types::{ErrorMetadata, ExtractedDocument, Metadata};
#[cfg(all(feature = "tokio-runtime", not(target_arch = "wasm32")))]
use std::borrow::Cow;
use std::sync::Arc;

/// Get an extractor from the registry.
///
/// This function acquires the registry read lock and retrieves the appropriate
/// extractor for the given MIME type.
///
/// When the `otel` feature is enabled, the returned extractor is wrapped in an
/// [`InstrumentedExtractor`](crate::plugins::extractor::instrumented::InstrumentedExtractor)
/// that adds tracing spans and metrics automatically.
///
/// # Performance
///
/// RwLock read + HashMap lookup is ~100ns, fast enough without caching.
/// Removed thread-local cache to avoid Tokio work-stealing scheduler issues.
pub(in crate::core::extractor) fn get_extractor(mime_type: &str) -> Result<Arc<dyn InternalDocumentExtractor>> {
    let registry = crate::plugins::registry::get_document_extractor_registry();
    let registry_read = registry.read();
    let extractor = registry_read.get_registered(mime_type)?;
    let extractor: Arc<dyn InternalDocumentExtractor> = Arc::new(extractor);

    #[cfg(feature = "otel")]
    {
        Ok(Arc::new(
            crate::plugins::extractor::instrumented::InstrumentedExtractor::new(extractor),
        ))
    }

    #[cfg(not(feature = "otel"))]
    {
        Ok(extractor)
    }
}

/// Get optimal pool sizing hint for a document.
///
/// This function calculates recommended pool sizes based on the document's
/// file size and MIME type. The hint can be used to create appropriately
/// sized thread pools for extraction, reducing memory waste from over-allocation.
///
/// # Arguments
///
/// * `file_size` - The size of the file in bytes
/// * `mime_type` - The MIME type of the document
///
/// # Returns
///
/// A `PoolSizeHint` with recommended pool configurations
///
/// # Example
///
/// ```rust,ignore
/// use xberg::core::extractor::get_pool_sizing_hint;
///
/// let hint = get_pool_sizing_hint(5_000_000, "application/pdf");
/// println!("Recommended string buffers: {}", hint.string_buffer_count);
/// ```
/// Build an error `ExtractedDocument` for failed batch items.
///
/// Used by the tokio-based concurrent batch functions to construct a uniform
/// error result for a single failed input. Its sole caller is `collect_batch`
/// (`batch.rs`), which is native-only — JoinSet needs `Send` futures + OS
/// threads — so this carries the exact same gate to stay name-resolvable there.
/// The batch machinery is currently reachable only from tests, so mirror the
/// module-level `#![allow(dead_code)]` on `batch.rs` to keep non-test builds warning-free.
#[cfg(all(feature = "tokio-runtime", not(target_arch = "wasm32")))]
#[allow(dead_code)]
pub(crate) fn error_extraction_result(e: &XbergError, elapsed_ms: Option<u64>) -> ExtractedDocument {
    let metadata = Metadata {
        error: Some(ErrorMetadata {
            error_type: format!("{:?}", e),
            message: e.to_string(),
        }),
        extraction_duration_ms: elapsed_ms,
        ..Default::default()
    };

    ExtractedDocument {
        content: format!("Error: {}", e),
        mime_type: Cow::Borrowed("text/plain"),
        metadata,
        ..Default::default()
    }
}