xberg 1.1.0

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 107 formats and 371 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

//! Regression tests for #155 (structural rendering for YAML/TOML/JSONL and
//! top-level JSON arrays) and #166 (surfacing the flattened structured-data
//! view instead of discarding it).

#![allow(clippy::print_stdout, clippy::print_stderr, clippy::dbg_macro)] // ~keep: test/bench binaries print by design; org logging policy exempts tests

use xberg::core::config::ExtractionConfig;

mod helpers;
use helpers::extract_bytes_document;

/// #155: a top-level JSON array of objects must render as headings/list items,
/// not as one opaque code block.
#[tokio::test]
async fn should_render_structure_for_top_level_json_array() {
    let config = ExtractionConfig::default();
    let json = br#"[{"name": "Ada"}, {"name": "Grace"}]"#;

    let extraction = extract_bytes_document(json, "application/json", &config)
        .await
        .expect("json extraction should succeed");

    assert!(
        extraction.content.contains("Item 1") && extraction.content.contains("Item 2"),
        "expected per-item headings, got: {}",
        extraction.content
    );
    assert!(
        extraction.content.contains("name: Ada") && extraction.content.contains("name: Grace"),
        "expected rendered field values, got: {}",
        extraction.content
    );
    assert!(
        !extraction.content.trim_start().starts_with('['),
        "content should not be a raw JSON code block: {}",
        extraction.content
    );
}

/// #155: YAML must render structure (headings), not fall through to a raw code block.
#[tokio::test]
async fn should_render_structure_for_yaml() {
    let config = ExtractionConfig::default();
    let yaml = b"name: Alice\nrole: Engineer\n";

    let extraction = extract_bytes_document(yaml, "application/yaml", &config)
        .await
        .expect("yaml extraction should succeed");

    assert!(
        extraction.content.contains("name: Alice"),
        "expected rendered field, got: {}",
        extraction.content
    );
    assert!(
        !extraction.content.contains("name: Alice\nrole: Engineer"),
        "content should not be the verbatim raw YAML code block: {}",
        extraction.content
    );
}

/// #155: TOML must render structure (headings) for a table, not fall through to a raw
/// code block.
#[tokio::test]
async fn should_render_structure_for_toml() {
    let config = ExtractionConfig::default();
    let toml = b"[user]\nname = \"Alice\"\nrole = \"Engineer\"\n";

    let extraction = extract_bytes_document(toml, "application/toml", &config)
        .await
        .expect("toml extraction should succeed");

    assert!(
        extraction.content.contains("user"),
        "expected a heading for the [user] table, got: {}",
        extraction.content
    );
    assert!(
        extraction.content.contains("name: Alice"),
        "expected rendered field, got: {}",
        extraction.content
    );
}

/// #155: JSONL (an array of records) must render per-record structure instead of one
/// opaque code block.
#[tokio::test]
async fn should_render_structure_for_jsonl() {
    let config = ExtractionConfig::default();
    let jsonl = b"{\"name\": \"Alice\"}\n{\"name\": \"Bob\"}";

    let extraction = extract_bytes_document(jsonl, "application/x-ndjson", &config)
        .await
        .expect("jsonl extraction should succeed");

    assert!(
        extraction.content.contains("Item 1") && extraction.content.contains("Item 2"),
        "expected per-record headings, got: {}",
        extraction.content
    );
    assert!(extraction.content.contains("name: Alice"));
    assert!(extraction.content.contains("name: Bob"));
}

/// #166: the flattened `path: value` view must be surfaced in metadata instead of
/// silently discarded.
#[tokio::test]
async fn should_surface_flattened_fields_in_metadata() {
    let config = ExtractionConfig::default();
    let json = br#"{"user": {"name": "Alice", "role": "Engineer"}}"#;

    let extraction = extract_bytes_document(json, "application/json", &config)
        .await
        .expect("json extraction should succeed");

    let flattened = extraction
        .metadata
        .additional
        .get("flattened_fields")
        .expect("flattened_fields metadata must be present")
        .as_array()
        .expect("flattened_fields must be a JSON array");

    let flattened: Vec<&str> = flattened.iter().map(|v| v.as_str().unwrap()).collect();
    assert_eq!(
        flattened,
        vec!["user.name: Alice", "user.role: Engineer"],
        "flattened view should contain every leaf path:value pair"
    );
}