xberg 1.1.0

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 107 formats and 371 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

//! Named-entity recognition (NER).
//!
//! Shared by:
//! - the NER post-processor at `crate::plugins::processor::builtin::ner` (populates
//!   [`ExtractedDocument::entities`](crate::types::ExtractedDocument::entities))
//! - the redaction engine at `crate::text::redaction::engine` (consumes the same
//!   `Entity` stream to redact PERSON / ORGANIZATION / LOCATION mentions that the
//!   pure-Rust pattern engine cannot detect).
//!
//! Backends implement the [`NerBackend`] trait. Two are bundled:
//!
//! - [`gline::GlineBackend`] under `#[cfg(feature = "ner-onnx")]` — local ONNX
//!   inference via `xberg-gliner`. Models download lazily from the
//!   `xberg-io/gliner-models` Hugging Face repository via
//!   `crate::model_download`.
//! - [`llm::LlmBackend`] under `#[cfg(feature = "ner-llm")]` — liter-llm with a
//!   structured-output schema. Used when categories outstrip the ONNX taxonomy.
//!
//! Every backend has a bounded input length, so both bundled backends window
//! long input through [`offsets::split_into_windows`] rather than letting the
//! model silently truncate — an undetected entity is PII that never gets
//! redacted (xberg-io/xberg#262).

#![cfg(feature = "ner")]

pub mod backend;
#[cfg(feature = "ner-candle-backend")]
pub mod candle;
#[cfg(feature = "ner-onnx")]
pub mod gline;
#[cfg(all(
    feature = "ner-llm",
    not(target_arch = "wasm32"),
    not(all(target_os = "android", target_arch = "x86_64"))
))]
pub mod llm;
pub mod offsets;

pub use backend::NerBackend;

use crate::Result;
use crate::types::entity::Entity;

#[cfg(feature = "ner-onnx")]
use std::path::PathBuf;

/// Eagerly download a NER model into the Hugging Face Hub cache.
///
/// `name` is a supported xberg GLiNER alias or catalog id. The CLI flag
/// `xberg cache warm --ner` delegates here. `cache_dir`, when provided, is a
/// custom Hugging Face Hub cache root; otherwise the standard `HF_HUB_CACHE`,
/// `HF_HOME`, and platform defaults apply.
#[cfg(feature = "ner-onnx")]
pub fn download_model(name: &str, cache_dir: Option<PathBuf>) -> crate::Result<PathBuf> {
    gline::download_model(name, cache_dir)
}

/// Pinned default NER model identifier.
#[cfg(feature = "ner-onnx")]
pub fn default_model_name() -> &'static str {
    gline::DEFAULT_MODEL_NAME
}

/// All NER models xberg knows about (used by `--all-ner-models`).
#[cfg(feature = "ner-onnx")]
pub fn known_models() -> &'static [&'static str] {
    gline::KNOWN_MODELS
}

/// Expected GLiNER cache artifacts for manifest tooling.
#[cfg(feature = "ner-onnx")]
#[cfg_attr(alef, alef(skip))]
pub fn manifest() -> Vec<gline::GlinerManifestEntry> {
    gline::manifest()
}

/// Detect named entities in the given text using the provided backend.
///
/// Identifies entities such as persons, organizations, locations, dates, and more
/// based on the backend's capabilities and the categories requested.
///
/// # Arguments
///
/// * `text` - The input text to analyze.
/// * `backend` - The NER backend implementation to use (either ONNX-based GLiNER or LLM-driven).
/// * `categories` - Entity categories to detect. If empty, the backend returns all entities it can identify.
///
/// # Returns
///
/// A vector of detected `Entity` objects in source byte-offset order.
///
/// # Example
///
/// ```rust,no_run
/// use xberg::types::entity::EntityCategory;
/// use xberg::text::ner::detect_entities;
/// use xberg::core::config::LlmConfig;
/// use xberg::LlmBackend;
///
/// # async fn example() -> xberg::Result<()> {
/// let backend = LlmBackend::new(LlmConfig::default());
/// let categories = vec![EntityCategory::Person, EntityCategory::Organization];
/// let entities = detect_entities("Alice works at Acme Corp.", &backend, &categories).await?;
/// # Ok(())
/// # }
/// ```
#[cfg_attr(alef, alef(skip))]
pub async fn detect_entities(
    text: &str,
    backend: &dyn NerBackend,
    categories: &[crate::types::entity::EntityCategory],
) -> Result<Vec<Entity>> {
    backend.detect(text, categories).await
}