xberg 1.0.9

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 101 formats and 371 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

//! Tests verifying CSV extraction produces embedding-friendly output.
//!
//! Header-value pairs preserve semantic associations between column names
//! and cell values, improving vector search quality over flat space-separated output.

mod helpers;
use helpers::extract_bytes_document;

use xberg::core::config::ExtractionConfig;

/// Header-value pairs should be explicit in the content.
#[tokio::test]
async fn test_csv_preserves_header_value_association() {
    let config = ExtractionConfig::default();
    let csv = b"Name,Age,City\nAlice,30,NYC\nBob,25,LA\n";

    let result = extract_bytes_document(csv, "text/csv", &config).await.unwrap();

    assert!(result.content.contains("Name: Alice"));
    assert!(result.content.contains("Age: 30"));
    assert!(result.content.contains("City: NYC"));
    assert!(result.content.contains("Name: Bob"));
    assert!(result.content.contains("Age: 25"));
    assert!(result.content.contains("City: LA"));
}

/// Rows should be labeled and separated by blank lines for chunker-friendly splitting.
#[tokio::test]
async fn test_csv_row_grouping() {
    let config = ExtractionConfig::default();
    let csv = b"Name,Score\nAlice,95\nBob,88\nCarol,72\n";

    let result = extract_bytes_document(csv, "text/csv", &config).await.unwrap();

    assert!(result.content.contains("Row 1:"));
    assert!(result.content.contains("Row 2:"));
    assert!(result.content.contains("Row 3:"));
    assert!(
        result.content.contains("\n\nRow 2:"),
        "Rows should be separated by blank lines"
    );
}

/// Empty cells should be omitted to avoid noisy `Header:` lines.
#[tokio::test]
async fn test_csv_skips_empty_values() {
    let config = ExtractionConfig::default();
    let csv = b"Name,Age,City\nAlice,,NYC\nBob,25,LA\n";

    let result = extract_bytes_document(csv, "text/csv", &config).await.unwrap();

    assert!(result.content.contains("Name: Alice"));
    assert!(result.content.contains("City: NYC"));
    let row1 = result.content.split("\n\n").next().unwrap_or("");
    assert!(!row1.contains("Age:"), "Empty cells should be skipped");
}

/// The tables field should still contain the full parsed structure.
#[tokio::test]
async fn test_csv_tables_field_unchanged() {
    let config = ExtractionConfig::default();
    let csv = b"Name,Age\nAlice,30\nBob,25\n";

    let result = extract_bytes_document(csv, "text/csv", &config).await.unwrap();

    assert_eq!(result.tables.len(), 1);
    assert_eq!(result.tables[0].cells.len(), 3);
    assert_eq!(result.tables[0].cells[0], vec!["Name", "Age"]);
    assert!(!result.tables[0].markdown.is_empty());
}

/// Rows shorter than the header should not panic; extra headers are skipped.
#[tokio::test]
async fn test_csv_short_row_no_panic() {
    let config = ExtractionConfig::default();
    let csv = b"Name,Age,City\nAlice,30\nBob,25,LA\n";

    let result = extract_bytes_document(csv, "text/csv", &config).await.unwrap();

    assert!(result.content.contains("Name: Alice"));
    assert!(result.content.contains("Age: 30"));
    let row1 = result.content.split("\n\n").next().unwrap_or("");
    assert!(!row1.contains("City:"));
}

/// Rows where all cells are empty should be omitted entirely.
#[tokio::test]
async fn test_csv_all_empty_data_rows() {
    let config = ExtractionConfig::default();
    let csv = b"Name,Age\n,,\nAlice,30\n";

    let result = extract_bytes_document(csv, "text/csv", &config).await.unwrap();

    assert!(result.content.contains("Name: Alice"));
}

/// When no header is detected, should fall back to space-separated output.
#[tokio::test]
async fn test_csv_no_header_fallback() {
    let config = ExtractionConfig::default();
    let csv = b"Alice,NYC,Engineer\nBob,LA,Designer\n";

    let result = extract_bytes_document(csv, "text/csv", &config).await.unwrap();

    assert!(
        !result.content.contains("Row 1:"),
        "No header detected — should not label rows"
    );
    assert!(result.content.contains("Alice"));
    assert!(result.content.contains("Bob"));
}

/// Header-only CSV (no data rows) should still produce output.
#[tokio::test]
async fn test_csv_header_only() {
    let config = ExtractionConfig::default();
    let csv = b"Name,Age,City\n";

    let result = extract_bytes_document(csv, "text/csv", &config).await.unwrap();

    assert!(!result.content.is_empty());
    assert!(result.content.contains("Name"));
}

/// Real CSV file with header-value pairing.
#[tokio::test]
async fn test_csv_real_file_header_value() {
    let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test_documents/csv/data_table.csv");
    if !path.exists() {
        return;
    }
    let content = std::fs::read(&path).unwrap();
    let config = ExtractionConfig::default();

    let result = extract_bytes_document(&content, "text/csv", &config).await.unwrap();

    assert!(result.content.contains("Name: Alice Johnson"));
    assert!(result.content.contains("Department: Engineering"));
    assert!(result.content.contains("Row 1:"));
}