xberg 1.1.2

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 107 formats and 371 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

//! Cache utilities for key generation and disk space management.

use crate::error::Result;
use std::io::Read;
use std::path::Path;

#[cfg(unix)]
use crate::error::XbergError;

/// Cache key hash format width (32 hex digits = 128 bits of blake3 output).
const CACHE_KEY_HASH_WIDTH: usize = 32;

/// Generate a deterministic cache key from configuration parameters.
///
/// # Algorithm
///
/// Uses blake3 (cryptographic, SIMD-accelerated) for collision-resistant cache keys.
/// Cache keys are generated by:
/// 1. Sorting key-value pairs by key (for determinism)
/// 2. Concatenating as "key1=val1&key2=val2&..."
/// 3. Hashing with blake3 and formatting as 32-character hex (first 128 bits)
///
/// # Example
///
/// Not run as a doctest: this helper is `#[cfg(test)] pub(crate)`, so it does not
/// exist in any build a downstream crate can link against.
///
/// ```ignore
/// use xberg::cache::generate_cache_key;
///
/// let parts = vec![
///     ("format".to_string(), "pdf".to_string()),
///     ("ocr".to_string(), "true".to_string()),
///     ("lang".to_string(), "en".to_string()),
/// ];
/// let key = generate_cache_key(&parts);
/// assert_eq!(key.len(), 32);
/// ```
#[cfg(test)]
pub(crate) fn generate_cache_key(parts: &[(String, String)]) -> String {
    if parts.is_empty() {
        return "empty".to_string();
    }

    let mut sorted_parts: Vec<(&str, &str)> = parts.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
    sorted_parts.sort_by_key(|(k, _)| *k);

    let estimated_size = sorted_parts.iter().map(|(k, v)| k.len() + v.len() + 2).sum::<usize>();
    let mut cache_str = String::with_capacity(estimated_size);

    for (i, (key, val)) in sorted_parts.iter().enumerate() {
        if i > 0 {
            cache_str.push('&');
        }
        cache_str.push_str(&format!("{}={}", key, val));
    }

    blake3_hash_to_hex(cache_str.as_bytes())
}

/// Hash arbitrary bytes with blake3, returning a 32-char hex string.
#[cfg(feature = "ocr")]
pub(crate) fn blake3_hash_bytes(data: &[u8]) -> String {
    blake3_hash_to_hex(data)
}

/// Hash a file's content with blake3 using streaming 64 KiB reads.
///
/// Returns a 32-char hex string (128 bits of blake3 output).
pub(crate) fn blake3_hash_file(path: &Path) -> Result<String> {
    let file = std::fs::File::open(path)
        .map_err(|e| crate::error::XbergError::cache(format!("Failed to open file for hashing: {e}")))?;
    let mut reader = std::io::BufReader::with_capacity(64 * 1024, file);
    let mut hasher = blake3::Hasher::new();

    let mut buf = [0u8; 64 * 1024];
    loop {
        let n = reader
            .read(&mut buf)
            .map_err(|e| crate::error::XbergError::cache(format!("Failed to read file for hashing: {e}")))?;
        if n == 0 {
            break;
        }
        hasher.update(&buf[..n]);
    }

    let hash = hasher.finalize();
    Ok(hex::encode(&hash.as_bytes()[..CACHE_KEY_HASH_WIDTH / 2]))
}

/// Hash bytes with blake3 and return first 128 bits as 32 hex chars.
#[cfg(any(test, feature = "ocr"))]
fn blake3_hash_to_hex(data: &[u8]) -> String {
    let hash = blake3::hash(data);
    hex::encode(&hash.as_bytes()[..CACHE_KEY_HASH_WIDTH / 2])
}

#[allow(unsafe_code)]
pub(crate) fn get_available_disk_space(path: &str) -> Result<f64> {
    #[cfg(unix)]
    {
        let path = Path::new(path);
        let check_path = if path.exists() {
            path
        } else if let Some(parent) = path.parent() {
            parent
        } else {
            Path::new("/")
        };

        use libc::{statvfs, statvfs as statvfs_struct};
        use std::ffi::CString;

        let path_str = check_path
            .to_str()
            .ok_or_else(|| XbergError::validation("Path contains invalid UTF-8".to_string()))?;
        let c_path = CString::new(path_str).map_err(|e| XbergError::validation(format!("Invalid path: {}", e)))?;

        let mut stat: statvfs_struct = unsafe { std::mem::zeroed() };

        let result = unsafe { statvfs(c_path.as_ptr(), &mut stat) };

        if result == 0 {
            #[allow(clippy::unnecessary_cast)]
            let available_bytes = stat.f_bavail as u64 * stat.f_frsize as u64;
            Ok(available_bytes as f64 / (1024.0 * 1024.0))
        } else {
            tracing::debug!("Failed to get disk stats for {}: errno {}", path_str, result);
            Ok(10000.0)
        }
    }

    #[cfg(not(unix))]
    {
        let _ = path;
        Ok(10000.0)
    }
}

#[cfg(test)]
pub(crate) fn fast_hash(data: &[u8]) -> u64 {
    let hash = blake3::hash(data);
    u64::from_le_bytes(hash.as_bytes()[..8].try_into().unwrap())
}

#[cfg(test)]
pub(crate) fn validate_cache_key(key: &str) -> bool {
    key.len() == 32 && key.chars().all(|c| c.is_ascii_hexdigit())
}