xberg 1.1.0

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 107 formats and 371 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

mod cjk_utils;
mod config;
mod core;
mod filters;
mod semantic;
mod simd_text;

pub use config::{ReductionLevel, TokenReductionConfig};
pub use core::TokenReducer;

/// Reduces token count in text while preserving meaning and structure.
///
/// This function removes stopwords, redundancy, and applies compression techniques
/// based on the specified reduction level. Supports 64 languages with automatic
/// stopword removal and optional semantic clustering.
///
/// # Arguments
///
/// * `text` - The input text to reduce
/// * `config` - Configuration specifying reduction level and options
/// * `language_hint` - Optional ISO 639-3 language code (e.g., "eng", "spa")
///
/// # Returns
///
/// Returns the reduced text with preserved structure (markdown, code blocks).
///
/// # Errors
///
/// Returns an error if the language hint is invalid or stopwords cannot be loaded.
///
/// # Examples
///
/// Not run as a doctest: this function is `pub(crate)`. Downstream crates reach it by
/// setting [`crate::TokenReductionConfig`] on the extraction config; only the config
/// types ([`crate::TokenReductionConfig`], [`crate::ReductionLevel`]) are public.
///
/// ```ignore
/// use xberg::text::token_reduction::{reduce_tokens, TokenReductionConfig, ReductionLevel};
///
/// let text = "This is a simple example text with some stopwords.";
/// let config = TokenReductionConfig::default();
/// let reduced = reduce_tokens(text, &config, Some("eng"))?;
/// println!("Reduced: {}", reduced);
/// # Ok::<(), xberg::error::XbergError>(())
/// ```
pub(crate) fn reduce_tokens(
    text: &str,
    config: &TokenReductionConfig,
    language_hint: Option<&str>,
) -> crate::error::Result<String> {
    let reducer = TokenReducer::new(config, language_hint)?;
    Ok(reducer.reduce(text))
}