Skip to main content

docling_core/
confidence.rs

1//! Conversion-confidence report — the Rust counterpart of docling's
2//! `ConfidenceReport` / `PageConfidenceScores` (`docling.datamodel.base_models`,
3//! surfaced per conversion by Python docling-serve v1.25+, #183).
4//!
5//! Semantics mirror docling exactly:
6//!
7//! - Four per-page scores, each in `[0, 1]` or *unset* (docling uses `NaN`;
8//!   here `Option<f64>` so JSON serializes as `null` instead of an invalid
9//!   `NaN` literal): `layout_score` (mean confidence of the kept layout
10//!   clusters), `ocr_score` (mean confidence of OCR-recognized cells),
11//!   `parse_score` (10th-percentile text-layer quality — the quantile
12//!   emphasises problems), `table_score` (unset; docling never assigns it
13//!   either, the field exists for wire compatibility).
14//! - A page's `mean_score`/`low_score` are the NaN-ignoring mean / 5th
15//!   percentile of its four scores; document-level `mean_score`/`low_score`
16//!   are the plain means of the per-page values (docling's
17//!   `ConfidenceReport` overrides — note: *mean*, not quantile, for both).
18//! - Document-level per-field aggregation: mean for layout/table/ocr,
19//!   10th percentile for parse.
20//! - Grades: `< 0.5` poor, `< 0.8` fair, `< 0.9` good, `≥ 0.9` excellent,
21//!   unset → unspecified.
22
23use std::collections::BTreeMap;
24
25/// docling's `QualityGrade`: a score bucketed for human consumption.
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub enum QualityGrade {
28    Poor,
29    Fair,
30    Good,
31    Excellent,
32    /// No score available (e.g. a declarative conversion with no ML stages).
33    Unspecified,
34}
35
36impl QualityGrade {
37    /// docling's `_score_to_grade` thresholds.
38    pub fn from_score(score: Option<f64>) -> Self {
39        match score {
40            Some(s) if s < 0.5 => Self::Poor,
41            Some(s) if s < 0.8 => Self::Fair,
42            Some(s) if s < 0.9 => Self::Good,
43            Some(_) => Self::Excellent,
44            None => Self::Unspecified,
45        }
46    }
47
48    /// The wire spelling (docling's lowercase enum values).
49    pub fn as_str(self) -> &'static str {
50        match self {
51            Self::Poor => "poor",
52            Self::Fair => "fair",
53            Self::Good => "good",
54            Self::Excellent => "excellent",
55            Self::Unspecified => "unspecified",
56        }
57    }
58}
59
60/// NaN-ignoring mean (docling's `np.nanmean`): `None` entries are skipped;
61/// all-unset yields `None` (where numpy would warn and return `NaN`).
62pub fn nanmean(values: &[Option<f64>]) -> Option<f64> {
63    let known: Vec<f64> = values.iter().filter_map(|v| *v).collect();
64    if known.is_empty() {
65        return None;
66    }
67    Some(known.iter().sum::<f64>() / known.len() as f64)
68}
69
70/// NaN-ignoring quantile with numpy's default linear interpolation
71/// (docling's `np.nanquantile(..., q)`).
72pub fn nanquantile(values: &[Option<f64>], q: f64) -> Option<f64> {
73    let mut known: Vec<f64> = values.iter().filter_map(|v| *v).collect();
74    if known.is_empty() {
75        return None;
76    }
77    known.sort_by(|a, b| a.total_cmp(b));
78    let pos = q * (known.len() - 1) as f64;
79    let lo = pos.floor() as usize;
80    let hi = pos.ceil() as usize;
81    if lo == hi {
82        return Some(known[lo]);
83    }
84    let frac = pos - lo as f64;
85    Some(known[lo] * (1.0 - frac) + known[hi] * frac)
86}
87
88/// One page's confidence scores (docling's `PageConfidenceScores`).
89#[derive(Debug, Clone, Copy, Default, PartialEq)]
90pub struct PageConfidence {
91    pub parse_score: Option<f64>,
92    pub layout_score: Option<f64>,
93    pub table_score: Option<f64>,
94    pub ocr_score: Option<f64>,
95}
96
97impl PageConfidence {
98    fn scores(&self) -> [Option<f64>; 4] {
99        // docling's aggregation order: ocr, table, layout, parse.
100        [
101            self.ocr_score,
102            self.table_score,
103            self.layout_score,
104            self.parse_score,
105        ]
106    }
107
108    /// NaN-ignoring mean of the four scores.
109    pub fn mean_score(&self) -> Option<f64> {
110        nanmean(&self.scores())
111    }
112
113    /// 5th percentile of the four scores (docling's `low_score`).
114    pub fn low_score(&self) -> Option<f64> {
115        nanquantile(&self.scores(), 0.05)
116    }
117
118    fn to_json(self) -> serde_json::Value {
119        serde_json::json!({
120            "parse_score": self.parse_score,
121            "layout_score": self.layout_score,
122            "table_score": self.table_score,
123            "ocr_score": self.ocr_score,
124            "mean_grade": QualityGrade::from_score(self.mean_score()).as_str(),
125            "low_grade": QualityGrade::from_score(self.low_score()).as_str(),
126            "mean_score": self.mean_score(),
127            "low_score": self.low_score(),
128        })
129    }
130}
131
132/// One OCR-recognized cell with the recognition confidence the pipeline
133/// computed for it (#915): the mean emitted-character probability of the
134/// line or word crop, the figure the page `ocr_score` averages. A docling
135/// extension — docling's report stops at the page — so it lives beside the
136/// report, never in the document JSON (docling-core's schema has no field for
137/// it). Boxes are page points, top-left origin, like the pipeline's cells.
138#[derive(Debug, Clone, PartialEq)]
139pub struct OcrCellConfidence {
140    pub text: String,
141    pub l: f32,
142    pub t: f32,
143    pub r: f32,
144    pub b: f32,
145    pub confidence: f32,
146}
147
148/// The document-level report (docling's `ConfidenceReport`): the four scores
149/// aggregated across pages, plus the per-page breakdown. Page keys are the
150/// **real 1-based page numbers** — the same numbering as the JSON export's
151/// `pages` map (#171), `--pages` windows included. (docling keys by its
152/// 0-based internal page index; ours is the more useful spelling and the
153/// difference is documented in `docs/MIGRATION.md`.)
154#[derive(Debug, Clone, Default, PartialEq)]
155pub struct ConfidenceReport {
156    pub pages: BTreeMap<usize, PageConfidence>,
157    /// Per-cell OCR confidences by 1-based page number (#915); empty for
158    /// pages read from a text layer. Not part of [`ConfidenceReport::to_json`]
159    /// (docling's wire shape) — read it with [`ConfidenceReport::ocr_cells`].
160    ocr_cells: BTreeMap<usize, Vec<OcrCellConfidence>>,
161}
162
163impl ConfidenceReport {
164    /// Build from per-page scores keyed by 1-based page number.
165    pub fn from_pages(pages: BTreeMap<usize, PageConfidence>) -> Self {
166        Self {
167            pages,
168            ocr_cells: BTreeMap::new(),
169        }
170    }
171
172    /// Attach the per-cell OCR confidences of a page (1-based). Empty lists
173    /// are not stored.
174    pub fn with_ocr_cells(mut self, page_no: usize, cells: Vec<OcrCellConfidence>) -> Self {
175        if !cells.is_empty() {
176            self.ocr_cells.insert(page_no, cells);
177        }
178        self
179    }
180
181    /// The OCR cells of a page with their recognition confidence, in the
182    /// order the pipeline produced them; empty when the page was not OCR'd.
183    pub fn ocr_cells(&self, page_no: usize) -> &[OcrCellConfidence] {
184        self.ocr_cells.get(&page_no).map_or(&[], Vec::as_slice)
185    }
186
187    /// The cells of every page whose confidence is below `threshold`,
188    /// as `(page_no, cell)`.
189    pub fn low_confidence_cells(
190        &self,
191        threshold: f32,
192    ) -> impl Iterator<Item = (usize, &OcrCellConfidence)> {
193        self.ocr_cells.iter().flat_map(move |(n, cells)| {
194            cells
195                .iter()
196                .filter(move |c| c.confidence < threshold)
197                .map(move |c| (*n, c))
198        })
199    }
200
201    fn field(&self, get: impl Fn(&PageConfidence) -> Option<f64>) -> Vec<Option<f64>> {
202        self.pages.values().map(get).collect()
203    }
204
205    /// Document `layout_score`: mean of the per-page values.
206    pub fn layout_score(&self) -> Option<f64> {
207        nanmean(&self.field(|p| p.layout_score))
208    }
209
210    /// Document `parse_score`: 10th percentile of the per-page values
211    /// (docling: quantile here too, to emphasise problem pages).
212    pub fn parse_score(&self) -> Option<f64> {
213        nanquantile(&self.field(|p| p.parse_score), 0.1)
214    }
215
216    /// Document `table_score`: mean of the per-page values.
217    pub fn table_score(&self) -> Option<f64> {
218        nanmean(&self.field(|p| p.table_score))
219    }
220
221    /// Document `ocr_score`: mean of the per-page values.
222    pub fn ocr_score(&self) -> Option<f64> {
223        nanmean(&self.field(|p| p.ocr_score))
224    }
225
226    /// Document `mean_score`: mean of the per-page mean scores.
227    pub fn mean_score(&self) -> Option<f64> {
228        nanmean(&self.field(|p| p.mean_score()))
229    }
230
231    /// Document `low_score`: mean (sic — docling's override uses `nanmean`,
232    /// not a quantile) of the per-page low scores.
233    pub fn low_score(&self) -> Option<f64> {
234        nanmean(&self.field(|p| p.low_score()))
235    }
236
237    pub fn mean_grade(&self) -> QualityGrade {
238        QualityGrade::from_score(self.mean_score())
239    }
240
241    pub fn low_grade(&self) -> QualityGrade {
242        QualityGrade::from_score(self.low_score())
243    }
244
245    /// The full report as JSON — docling's `ConfidenceReport` dump shape
246    /// (unset scores as `null`, grades as lowercase strings, `pages` keyed by
247    /// stringified page number).
248    pub fn to_json(&self) -> serde_json::Value {
249        let mut value = self.summary_json();
250        value["pages"] = serde_json::Value::Object(
251            self.pages
252                .iter()
253                .map(|(n, p)| (n.to_string(), p.to_json()))
254                .collect(),
255        );
256        value
257    }
258
259    /// The document-level scores/grades only (no `pages`) — compact enough
260    /// for a response header.
261    pub fn summary_json(&self) -> serde_json::Value {
262        serde_json::json!({
263            "parse_score": self.parse_score(),
264            "layout_score": self.layout_score(),
265            "table_score": self.table_score(),
266            "ocr_score": self.ocr_score(),
267            "mean_grade": self.mean_grade().as_str(),
268            "low_grade": self.low_grade().as_str(),
269            "mean_score": self.mean_score(),
270            "low_score": self.low_score(),
271        })
272    }
273}
274
275#[cfg(test)]
276mod tests {
277    use super::*;
278
279    #[test]
280    fn ocr_cells_are_kept_beside_the_scores() {
281        let cell = |t: &str, c: f32| OcrCellConfidence {
282            text: t.into(),
283            l: 0.0,
284            t: 0.0,
285            r: 1.0,
286            b: 1.0,
287            confidence: c,
288        };
289        let report = ConfidenceReport::from_pages(BTreeMap::new())
290            .with_ocr_cells(2, vec![cell("ok", 0.99), cell("smudge", 0.31)])
291            .with_ocr_cells(3, Vec::new());
292        assert_eq!(report.ocr_cells(2).len(), 2);
293        assert!(report.ocr_cells(1).is_empty() && report.ocr_cells(3).is_empty());
294        let low: Vec<_> = report.low_confidence_cells(0.5).collect();
295        assert_eq!(low.len(), 1);
296        assert_eq!((low[0].0, low[0].1.text.as_str()), (2, "smudge"));
297        // docling's wire shape is untouched.
298        assert!(!report.to_json().to_string().contains("smudge"));
299    }
300
301    #[test]
302    fn grades_follow_docling_thresholds() {
303        assert_eq!(QualityGrade::from_score(Some(0.49)), QualityGrade::Poor);
304        assert_eq!(QualityGrade::from_score(Some(0.5)), QualityGrade::Fair);
305        assert_eq!(QualityGrade::from_score(Some(0.79)), QualityGrade::Fair);
306        assert_eq!(QualityGrade::from_score(Some(0.8)), QualityGrade::Good);
307        assert_eq!(QualityGrade::from_score(Some(0.89)), QualityGrade::Good);
308        assert_eq!(QualityGrade::from_score(Some(0.9)), QualityGrade::Excellent);
309        assert_eq!(QualityGrade::from_score(None), QualityGrade::Unspecified);
310    }
311
312    #[test]
313    fn nan_handling_matches_numpy() {
314        // nanmean skips unset entries; all-unset is unset.
315        assert_eq!(nanmean(&[Some(0.5), None, Some(1.0)]), Some(0.75));
316        assert_eq!(nanmean(&[None, None]), None);
317        // Linear interpolation: quantile 0.05 over [0.2, 1.0] = 0.2 + 0.05*0.8.
318        let q = nanquantile(&[Some(1.0), Some(0.2)], 0.05).unwrap();
319        assert!((q - 0.24).abs() < 1e-12, "{q}");
320        // Single value: every quantile is that value.
321        assert_eq!(nanquantile(&[None, Some(0.7)], 0.1), Some(0.7));
322    }
323
324    #[test]
325    fn report_aggregates_like_docling() {
326        let mut pages = BTreeMap::new();
327        pages.insert(
328            1,
329            PageConfidence {
330                parse_score: Some(1.0),
331                layout_score: Some(0.9),
332                table_score: None,
333                ocr_score: None,
334            },
335        );
336        pages.insert(
337            2,
338            PageConfidence {
339                parse_score: Some(0.6),
340                layout_score: Some(0.7),
341                table_score: None,
342                ocr_score: Some(0.8),
343            },
344        );
345        let report = ConfidenceReport::from_pages(pages);
346        // layout: mean(0.9, 0.7); ocr: mean over the one page that has it.
347        assert_eq!(report.layout_score(), Some(0.8));
348        assert_eq!(report.ocr_score(), Some(0.8));
349        assert_eq!(report.table_score(), None);
350        // parse: 10th percentile of [0.6, 1.0] = 0.6 + 0.1*0.4.
351        let parse = report.parse_score().unwrap();
352        assert!((parse - 0.64).abs() < 1e-12, "{parse}");
353        // Document mean = mean of the page means: page1 (0.9+1.0)/2 = 0.95,
354        // page2 (0.8+0.7+0.6)/3 = 0.7 → 0.825.
355        let mean = report.mean_score().unwrap();
356        assert!((mean - 0.825).abs() < 1e-12, "{mean}");
357        assert_eq!(report.mean_grade(), QualityGrade::Good);
358
359        let json = report.to_json();
360        assert_eq!(json["mean_grade"], "good");
361        assert_eq!(json["table_score"], serde_json::Value::Null);
362        assert!(json["pages"]["1"]["layout_score"].as_f64().is_some());
363    }
364
365    #[test]
366    fn empty_report_is_unspecified() {
367        let report = ConfidenceReport::default();
368        assert_eq!(report.mean_score(), None);
369        assert_eq!(report.mean_grade(), QualityGrade::Unspecified);
370        assert_eq!(report.to_json()["low_grade"], "unspecified");
371    }
372}