xberg 1.1.0

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 107 formats and 371 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229

//! ZIP archive extraction.
//!
//! Provides functions for extracting metadata and text content from ZIP archives.

use super::{ArchiveEntry, ArchiveMetadata, TEXT_EXTENSIONS};
use crate::error::{Result, XbergError};
use crate::extractors::security::SecurityLimits;
use ahash::AHashMap;
use std::io::{Cursor, Read};
use zip::ZipArchive;

/// Extract metadata from a ZIP archive.
///
/// # Arguments
///
/// * `bytes` - The ZIP archive bytes
/// * `limits` - Security limits for archive extraction
///
/// # Returns
///
/// Returns `ArchiveMetadata` containing:
/// - Format: "ZIP"
/// - File list with paths, sizes, and directory flags
/// - Total file count
/// - Total uncompressed size
///
/// # Errors
///
/// Returns an error if the ZIP archive cannot be read or parsed,
/// or if security limits are exceeded.
pub(crate) fn extract_zip_metadata(bytes: &[u8], limits: &SecurityLimits) -> Result<ArchiveMetadata> {
    let cursor = Cursor::new(bytes);
    let mut archive =
        ZipArchive::new(cursor).map_err(|e| XbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;

    if archive.len() > limits.max_files_in_archive {
        return Err(XbergError::validation(format!(
            "ZIP archive has too many files: {} (max: {})",
            archive.len(),
            limits.max_files_in_archive
        )));
    }

    let mut file_list = Vec::with_capacity(archive.len());
    let mut total_size = 0u64;

    for i in 0..archive.len() {
        let file = archive
            .by_index(i)
            .map_err(|e| XbergError::parsing(format!("Failed to read ZIP entry: {}", e)))?;

        let path = file.name().to_string();
        let size = file.size();
        let is_dir = file.is_dir();

        if !is_dir {
            total_size += size;
        }

        if total_size > limits.max_archive_size as u64 {
            return Err(XbergError::validation(format!(
                "ZIP archive total uncompressed size exceeds limit: {} bytes (max: {} bytes)",
                total_size, limits.max_archive_size
            )));
        }

        file_list.push(ArchiveEntry { path, size, is_dir });
    }

    Ok(ArchiveMetadata {
        format: "ZIP".to_string(),
        file_list,
        file_count: archive.len(),
        total_size,
    })
}

/// Extract text content from files within a ZIP archive.
///
/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log, .yaml, .toml
///
/// # Arguments
///
/// * `bytes` - The ZIP archive bytes
///
/// # Returns
///
/// Returns a `HashMap` mapping file paths to their text content.
/// Binary files and files with non-text extensions are excluded.
///
/// # Errors
///
/// Returns an error if the ZIP archive cannot be read or parsed.
pub(crate) fn extract_zip_text_content(bytes: &[u8], limits: &SecurityLimits) -> Result<AHashMap<String, String>> {
    let cursor = Cursor::new(bytes);
    let mut archive =
        ZipArchive::new(cursor).map_err(|e| XbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;

    if archive.len() > limits.max_files_in_archive {
        return Err(XbergError::validation(format!(
            "ZIP archive has too many files: {} (max: {})",
            archive.len(),
            limits.max_files_in_archive
        )));
    }

    let estimated_text_files = archive.len().saturating_mul(3).saturating_div(10).max(2);
    let mut contents = AHashMap::with_capacity(estimated_text_files);
    let mut total_content_size = 0usize;

    for i in 0..archive.len() {
        let mut file = archive
            .by_index(i)
            .map_err(|e| XbergError::parsing(format!("Failed to read ZIP entry: {}", e)))?;

        let path = file.name().to_string();

        if !file.is_dir() && TEXT_EXTENSIONS.iter().any(|ext| path.to_lowercase().ends_with(ext)) {
            // `zip` 2.4.2 builds the entry's `Read` impl as `Crc32Reader::new(Decompressor::new(..))`
            // with no `Take` on the decompressed side (only the *compressed* side is bounded) --
            // the declared `size()` header field is not enforced by the reader at all, so a member
            // can expand arbitrarily regardless of what it claims. Bound the read ourselves at
            // `max_content_size`: that is the aggregate budget this member's decoded text is about
            // to be checked against a few lines down, so no single member can legitimately need
            // more than the whole document's text budget. `+1` lets us tell "exactly at the limit"
            // apart from "over the limit" instead of silently truncating a hostile member's content.
            let cap = limits.max_content_size as u64;
            let mut raw = Vec::new();
            let mut limited = (&mut file).take(cap.saturating_add(1));
            if let Err(e) = limited.read_to_end(&mut raw) {
                tracing::warn!(member = %path, error = %e, "skipping ZIP member: read failed");
                continue;
            }
            if raw.len() as u64 > cap {
                // Reject rather than silently truncate or skip: the pre-existing code already
                // surfaced a hard error once the *decoded* total crossed `max_content_size`
                // (below), so capping the read must not become a quieter failure mode than the
                // one it replaces.
                return Err(XbergError::validation(format!(
                    "ZIP archive member '{}' exceeds max_content_size while decompressing (limit: {} bytes)",
                    path, limits.max_content_size
                )));
            }
            let content = super::decode_archive_text(&raw, &path);
            total_content_size = total_content_size.saturating_add(content.len());
            if total_content_size > limits.max_content_size {
                return Err(XbergError::validation(format!(
                    "ZIP archive text content exceeds limit: {} bytes (max: {} bytes)",
                    total_content_size, limits.max_content_size
                )));
            }
            contents.insert(path, content);
        }
    }

    Ok(contents)
}

/// Extract raw file bytes for all non-directory entries in a ZIP archive.
///
/// Returns a `HashMap` mapping file paths to their raw byte content.
/// Respects security limits for file count and total archive size.
///
/// # Arguments
///
/// * `bytes` - The ZIP archive bytes
/// * `limits` - Security limits for archive extraction
///
/// # Errors
///
/// Returns an error if the ZIP archive cannot be read or if security limits are exceeded.
pub(crate) fn extract_zip_file_bytes(bytes: &[u8], limits: &SecurityLimits) -> Result<AHashMap<String, Vec<u8>>> {
    let cursor = Cursor::new(bytes);
    let mut archive =
        ZipArchive::new(cursor).map_err(|e| XbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;

    if archive.len() > limits.max_files_in_archive {
        return Err(XbergError::validation(format!(
            "ZIP archive has too many files: {} (max: {})",
            archive.len(),
            limits.max_files_in_archive
        )));
    }

    let mut file_bytes = AHashMap::with_capacity(archive.len());
    let mut total_size = 0usize;

    for i in 0..archive.len() {
        let mut file = archive
            .by_index(i)
            .map_err(|e| XbergError::parsing(format!("Failed to read ZIP entry: {}", e)))?;

        if file.is_dir() {
            continue;
        }

        let path = file.name().to_string();
        // See the comment in `extract_zip_text_content`: the underlying `zip` reader has no
        // `Take` on the decompressed side, so `size()` (the declared header value) bounds
        // nothing about this read. Cap it ourselves at `max_archive_size` -- the same budget
        // `total_size` is checked against below, so no single member can legitimately need
        // more than the whole archive's budget. The `+1` distinguishes "exactly at the limit"
        // from "over" instead of silently truncating a hostile member's bytes.
        let cap = limits.max_archive_size as u64;
        let mut content = Vec::new();
        let mut limited = (&mut file).take(cap.saturating_add(1));
        if limited.read_to_end(&mut content).is_ok() {
            if content.len() as u64 > cap {
                // Reject rather than silently truncate: the pre-existing code already surfaced
                // a hard error once the running `total_size` crossed `max_archive_size` (below),
                // so capping the read must not become a quieter failure mode than that.
                return Err(XbergError::validation(format!(
                    "ZIP archive member '{}' exceeds max_archive_size while decompressing (limit: {} bytes)",
                    path, limits.max_archive_size
                )));
            }
            total_size = total_size.saturating_add(content.len());
            if total_size > limits.max_archive_size {
                return Err(XbergError::validation(format!(
                    "ZIP archive total extracted size exceeds limit: {} bytes (max: {} bytes)",
                    total_size, limits.max_archive_size
                )));
            }
            file_bytes.insert(path, content);
        }
    }

    Ok(file_bytes)
}