xberg 1.1.5

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 107 formats and 371 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

//! Page extraction and tracking configuration.
//!
//! Controls how pages are extracted, tracked, and represented in extraction results.
//! When `None`, page tracking is disabled.

use serde::{Deserialize, Serialize};

/// Page extraction and tracking configuration.
///
/// Controls how pages are extracted, tracked, and represented in the extraction results.
/// When `None`, page tracking is disabled.
///
/// Page range tracking in chunk metadata (first_page/last_page) is automatically enabled
/// when page boundaries are available and chunking is configured.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct PageConfig {
    /// Extract pages as separate array (ExtractedDocument.pages)
    #[serde(default)]
    pub extract_pages: bool,

    /// Insert page markers in main content string
    #[serde(default)]
    pub insert_page_markers: bool,

    /// Page marker format (use {page_num} placeholder)
    /// Default: "\n\n<!-- PAGE {page_num} -->\n\n"
    #[serde(default = "default_page_marker_format")]
    pub marker_format: String,
}

impl Default for PageConfig {
    fn default() -> Self {
        Self {
            extract_pages: false,
            insert_page_markers: false,
            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
        }
    }
}

fn default_page_marker_format() -> String {
    "\n\n<!-- PAGE {page_num} -->\n\n".to_string()
}

/// Regex matching one whole line that is a rendered instance of `marker_format`
/// (every `{page_num}` placeholder replaced by digits). Used to recognize page
/// marker lines so renderers pass them through verbatim.
pub(crate) fn marker_line_regex(marker_format: &str) -> regex::Regex {
    let pattern = marker_format
        .trim()
        .split("{page_num}")
        .map(regex::escape)
        .collect::<Vec<_>>()
        .join(r"\d+");
    regex::Regex::new(&format!("^{pattern}$")).expect("escaped marker format is a valid regex")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_page_config_default() {
        let config = PageConfig::default();
        assert!(!config.extract_pages);
        assert!(!config.insert_page_markers);
        assert_eq!(config.marker_format, "\n\n<!-- PAGE {page_num} -->\n\n");
    }
}