xberg 1.3.5

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 107 formats and 371 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

//! Unified public extraction API.
//!
//! These functions are the stable, binding-generated public surface. Their
//! signatures must remain byte-identical (they are scanned by the alef binding
//! generator). The implementation delegates to a process-global default
//! [`crate::engine::Engine`]; the extraction internals live in
//! [`crate::engine`] and are a pure refactor of what previously lived here.

use std::sync::LazyLock;

use crate::Result;
#[cfg(feature = "url-ingestion")]
use crate::core::config::UrlExtractionConfig;
use crate::core::config::{ExtractInput, ExtractionConfig, ExtractionResult};
#[cfg(feature = "url-ingestion")]
use crawlberg::{CrawlEngine, MapResult};

/// Process-global default engine backing the free `extract` / `extract_batch`
/// functions. Construction is cheap and side-effect free.
static DEFAULT_ENGINE: LazyLock<crate::engine::Engine> = LazyLock::new(crate::engine::Engine::new_default);

/// Extract content from a single bytes or URI input.
pub async fn extract(input: ExtractInput, config: &ExtractionConfig) -> Result<ExtractionResult> {
    DEFAULT_ENGINE.extract(input, config).await
}

/// Extract one bytes input and redact a JSON array or JSON Lines payload from an external inspection engine.
///
/// The payload is parsed in Rust so vendor aliases and nested fields remain intact across language bindings.
/// `offset_encoding` defaults to `unicode_code_points` and `max_findings` defaults to 10,000 when omitted.
/// Unknown encodings return a validation error.
#[cfg(feature = "redaction")]
#[cfg_attr(feature = "alef-meta", alef(since = "1.3.1"))]
pub async fn extract_with_external_redaction(
    input: ExtractInput,
    config: &ExtractionConfig,
    findings_json: &str,
    offset_encoding: Option<&str>,
    max_findings: Option<u32>,
) -> Result<ExtractionResult> {
    let offset_encoding = offset_encoding.unwrap_or("unicode_code_points").parse()?;
    let requested_limit = max_findings.unwrap_or(crate::text::redaction::external::DEFAULT_MAX_FINDINGS);
    let default_limits = crate::extractors::security::SecurityLimits::default();
    let security_limit = crate::text::redaction::external::security_finding_limit(
        config.security_limits.as_ref().unwrap_or(&default_limits),
    );
    let effective_limit = u32::try_from(security_limit.min(requested_limit as usize)).map_err(|_| {
        crate::XbergError::validation("effective redaction finding limit exceeds the supported u32 range".to_string())
    })?;
    let findings = crate::text::redaction::parse_external_findings_bounded(findings_json, effective_limit)?;
    DEFAULT_ENGINE
        .extract_with_external_redaction(input, config, findings, offset_encoding, max_findings)
        .await
}

/// Extract content from multiple bytes or URI inputs.
pub async fn extract_batch(inputs: Vec<ExtractInput>, config: &ExtractionConfig) -> Result<ExtractionResult> {
    DEFAULT_ENGINE.extract_batch(inputs, config).await
}

/// Discover all pages and sitemaps reachable from `uri` without extracting document content.
///
/// Builds a [`crawlberg::CrawlEngine`] from `config.crawl`, calls
/// [`CrawlEngine::map`], and returns the set of discovered URLs as a
/// [`crawlberg::MapResult`] (re-exported as [`crate::MapResult`]).
///
/// Use this when you need the URL inventory of a site before committing to
/// full document extraction — e.g. to build a crawl queue or validate scope.
///
/// # Errors
///
/// Returns [`crate::XbergError::Validation`] if the crawl configuration fails
/// validation or if the map operation itself fails.
#[cfg(feature = "url-ingestion")]
pub async fn map_url(uri: &str, config: &UrlExtractionConfig) -> Result<MapResult> {
    config.crawl.validate().map_err(map_crawl_err)?;
    let engine = CrawlEngine::builder()
        .config(config.crawl.clone())
        .build()
        .map_err(map_crawl_err)?;
    engine.map(uri).await.map_err(map_crawl_err)
}

/// Convert a [`crawlberg::CrawlError`] into an [`crate::XbergError`].
///
/// Mirrors the conversion used by the URL-ingestion extraction paths.
#[cfg(feature = "url-ingestion")]
fn map_crawl_err(error: crawlberg::CrawlError) -> crate::XbergError {
    crate::XbergError::validation(format!("crawlberg URL extraction failed: {error}"))
}