xberg 1.1.1

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 107 formats and 371 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

//! Chunk-classification configuration.
//!
//! When `ExtractionConfig::chunk_classification` is `Some`, the chunk-classification
//! post-processor runs at the Middle stage (after chunking has populated
//! `ExtractedDocument::chunks`) and populates
//! [`ChunkMetadata::classifications`](crate::types::ChunkMetadata::classifications)
//! on every chunk.

use serde::{Deserialize, Serialize};

/// Default number of chunks grouped into a single classification request.
///
/// Batching amortizes the fixed cost of the (potentially large) definitions
/// block, which is repeated verbatim in every request, across more chunks.
pub const DEFAULT_BATCH_SIZE: usize = 10;

/// Default number of concurrent in-flight classification batch requests.
///
/// Bounds concurrency against the configured LLM provider to avoid
/// rate-limit failures on large documents.
pub const DEFAULT_MAX_CONCURRENCY: usize = 4;

/// A single labeled definition the chunk classifier may emit.
///
/// Unlike `PageClassificationConfig::labels` (bare label names), chunk
/// classification targets potentially large domain taxonomies where every
/// label carries its own semantic description, letting the LLM disambiguate
/// similarly named labels without relying on the label string alone.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "alef-meta", alef(since = "1.0.0"))]
pub struct ChunkClassificationDefinition {
    /// Label name returned in `ChunkMetadata::classifications`.
    pub label: String,
    /// Semantic description of when this label applies. Injected verbatim into
    /// the classification prompt next to the label name.
    pub description: String,
}

/// Configuration for the chunk-classification post-processor.
///
/// Chunk classification is always multi-label: a chunk may match zero, one, or
/// many of the configured definitions. This is the chunk-level equivalent of
/// [`super::PageClassificationConfig`], but scoped to individual chunks
/// (`ExtractedDocument::chunks`) rather than whole pages, and built for large
/// taxonomies where each label needs its own description rather than a bare name.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "alef-meta", alef(since = "1.0.0"))]
pub struct ChunkClassificationConfig {
    /// Minijinja prompt template. Receives `{{ definitions }}` (rendered label +
    /// description list) and `{{ chunks }}` (a numbered list of chunk texts in
    /// the current batch) variables. `None` lets the backend pick a sensible
    /// default.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub prompt_template: Option<String>,
    /// The set of label definitions the classifier may emit. Must contain at
    /// least one entry.
    pub definitions: Vec<ChunkClassificationDefinition>,
    /// LLM configuration used for classification.
    pub llm: super::llm::LlmConfig,
    /// Number of chunks batched into a single LLM request.
    ///
    /// Larger batches amortize the fixed prompt cost (definitions block) across
    /// more chunks, at the risk of exceeding the model's context window for
    /// very large taxonomies or chunk texts. Defaults to [`DEFAULT_BATCH_SIZE`].
    #[serde(default = "ChunkClassificationConfig::default_batch_size")]
    pub batch_size: usize,
    /// Maximum number of in-flight batch requests.
    ///
    /// Bounds concurrency against the configured LLM provider. Defaults to
    /// [`DEFAULT_MAX_CONCURRENCY`].
    #[serde(default = "ChunkClassificationConfig::default_max_concurrency")]
    pub max_concurrency: usize,
}

impl ChunkClassificationConfig {
    /// Default [`Self::batch_size`]: [`DEFAULT_BATCH_SIZE`].
    ///
    /// Public and on the type rather than a free private `fn` because generated bindings
    /// have to call it to reproduce the default, and a private one is out of their reach.
    pub fn default_batch_size() -> usize {
        DEFAULT_BATCH_SIZE
    }

    /// Default [`Self::max_concurrency`]: [`DEFAULT_MAX_CONCURRENCY`].
    ///
    /// Public and on the type rather than a free private `fn` because generated bindings
    /// have to call it to reproduce the default, and a private one is out of their reach.
    pub fn default_max_concurrency() -> usize {
        DEFAULT_MAX_CONCURRENCY
    }
}