xberg-gliner 1.3.10

GLiNER inference used by xberg NER: span-mode ONNX runtime plus an optional Candle GLiNER2 backend.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

#[cfg(feature = "ort-backend")]
use crate::config::{MAX_BATCH_SIZE, MAX_ENTITY_LABEL_CHARS, MAX_ENTITY_LABELS};
#[cfg(feature = "ort-backend")]
use crate::{GlinerError, Result};

#[cfg(feature = "ort-backend")]
const ENTITY_TOKEN: &str = "<<ENT>>";
#[cfg(feature = "ort-backend")]
const SEP_TOKEN: &str = "<<SEP>>";

/// Raw text input and zero-shot entity labels.
///
/// Only consumed by the ORT-backed engines ([`crate::engine::Gliner`],
/// [`crate::v2::engine::Gliner2`]) and their preprocessing; dead weight
/// without `ort-backend`.
#[cfg(feature = "ort-backend")]
#[derive(Debug, Clone)]
pub struct TextInput {
    pub(crate) texts: Vec<String>,
    pub(crate) entities: Vec<String>,
}

#[cfg(feature = "ort-backend")]
impl TextInput {
    /// Construct a text batch.
    pub fn new(texts: Vec<String>, entities: Vec<String>) -> Result<Self> {
        if texts.is_empty() {
            return Err(GlinerError::InvalidInput("texts must not be empty".to_string()));
        }
        if entities.is_empty() {
            return Err(GlinerError::InvalidInput("entity labels must not be empty".to_string()));
        }
        if texts.len() > MAX_BATCH_SIZE {
            return Err(GlinerError::InvalidInput(format!(
                "batch size must be at most {MAX_BATCH_SIZE}, got {}",
                texts.len()
            )));
        }
        if entities.len() > MAX_ENTITY_LABELS {
            return Err(GlinerError::InvalidInput(format!(
                "entity label count must be at most {MAX_ENTITY_LABELS}, got {}",
                entities.len()
            )));
        }
        if let Some(index) = texts.iter().position(|text| text.trim().is_empty()) {
            return Err(GlinerError::InvalidInput(format!("texts[{index}] must not be empty")));
        }
        if let Some(index) = entities.iter().position(|label| label.trim().is_empty()) {
            return Err(GlinerError::InvalidInput(format!(
                "entity labels[{index}] must not be empty"
            )));
        }
        if let Some((index, label)) = entities
            .iter()
            .enumerate()
            .find(|(_, label)| label.chars().count() > MAX_ENTITY_LABEL_CHARS)
        {
            return Err(GlinerError::InvalidInput(format!(
                "entity labels[{index}] must be at most {MAX_ENTITY_LABEL_CHARS} characters, got {}",
                label.chars().count()
            )));
        }
        if let Some(index) = entities
            .iter()
            .position(|label| label.contains(ENTITY_TOKEN) || label.contains(SEP_TOKEN))
        {
            return Err(GlinerError::InvalidInput(format!(
                "entity labels[{index}] must not contain reserved GLiNER prompt markers"
            )));
        }
        Ok(Self { texts, entities })
    }

    /// Construct a text batch from borrowed string slices.
    pub fn from_str(texts: &[&str], entities: &[&str]) -> Result<Self> {
        Self::new(
            texts.iter().map(|text| (*text).to_string()).collect(),
            entities.iter().map(|entity| (*entity).to_string()).collect(),
        )
    }
}

/// A word token with byte offsets in the source text.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
    start: usize,
    end: usize,
    text: String,
}

impl Token {
    pub fn new(start: usize, end: usize, text: &str) -> Self {
        Self {
            start,
            end,
            text: text.to_string(),
        }
    }

    /// Start byte offset.
    pub fn start(&self) -> usize {
        self.start
    }

    /// End byte offset.
    pub fn end(&self) -> usize {
        self.end
    }

    /// Token text.
    pub fn text(&self) -> &str {
        &self.text
    }
}