docling_pdf/ocr.rs
1//! OCR for scanned pages, via the PP-OCRv3 recognition model (CRNN/SVTR) run
2//! with `ort`. The layout model locates the text regions on the page image
3//! (it works without a text layer); inside them the recognizer reads the
4//! lines the PP-OCR text detector found (`ocr_det`, #429/#570 — RapidOCR's
5//! own crops, each one text run with the detector's margin), falling back to
6//! a horizontal-projection split of the region crop where the detector saw
7//! nothing or is not installed; each line is recognised and decoded with CTC
8//! — producing [`TextCell`]s the normal layout assembly then consumes. A line
9//! under RapidOCR's `text_score` confidence is dropped (`ocr_prep::text_score`).
10
11use image::RgbImage;
12use ort::session::Session;
13use ort::value::Tensor;
14
15use crate::layout::Region;
16// The ONNX-free half (line prep, batching, CTC decode) lives in `ocr_prep`
17// so the wasm build shares it verbatim (#79 phase 2).
18use crate::ocr_prep::{
19 batch_input, decode_row_scored, dict_chars, prep_region_lines, prep_region_lines_det,
20 prep_table_words, prep_table_words_det, width_batches, PrepLine, REC_HEIGHT,
21};
22use crate::pdfium_backend::TextCell;
23
24pub struct OcrModel {
25 /// Single-threaded recognition sessions, one per parallel lane (see
26 /// [`Self::load_with`]); lines are dealt across them by batch index.
27 recs: Vec<Session>,
28 /// CTC classes: index 0 = blank, then the dictionary, then space.
29 chars: Vec<String>,
30 /// RapidOCR's line orientation handling (#666). See
31 /// [`crate::Pipeline::ocr_textline_orientation`].
32 orient: bool,
33 /// The text-line orientation classifier, loaded on the first oriented
34 /// page.
35 cls: ClsSlot,
36}
37
38/// [`OcrModel::cls`]: `Missing` records a failed load (warned once per
39/// process), so it is not retried per page.
40enum ClsSlot {
41 Unloaded,
42 Ready(crate::ocr_cls::TextlineCls),
43 Missing,
44}
45
46/// OCR recognition language: which PP-OCRv3 model + dictionary pair runs
47/// when the PP-OCRv6 recognizer is not installed.
48///
49/// With `.models/ocr_rec_v6.onnx` + `.models/ocr_rec_v6_dict.txt` on disk
50/// (#570; `download_dependencies.sh` fetches them) both languages run that
51/// one multilingual model — RapidOCR's `PP-OCRv6_rec_small`, the recognizer
52/// docling runs for English and Chinese alike, and the single largest factor
53/// in the FUNSD word-recall gap once the detector's lines are the crops
54/// (0.70 → 0.77 on 30 forms). Without it, the PP-OCRv3 pairs: the default is
55/// **English** (`.models/ocr_rec_en.onnx` + `.models/en_dict.txt`) — the
56/// multilingual `ch_` v3 model reads Latin scripts with badly degraded word
57/// spacing (glued words on ordinary English scans) — and `Ch` selects the
58/// `ch_` pair (`.models/ocr_rec.onnx` + `.models/ppocr_keys_v1.txt`), what
59/// the PDF conformance baselines were pinned against;
60/// `scripts/conformance/pdf_*.sh` pin it explicitly by path, which wins over
61/// this selector and over the v6 preference.
62#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
63pub enum OcrLang {
64 /// en_PP-OCRv3 — English-only, proper Latin word spacing.
65 #[default]
66 En,
67 /// ch_PP-OCRv3 — multilingual; the docling-conformance model.
68 Ch,
69}
70
71impl OcrLang {
72 /// Parse a user-supplied language id (#388): the engine's own codes
73 /// (`en`, `ch`) and BCP-47 tags naming a language one of the two
74 /// recognizers reads, trimmed and case-insensitive. `None` for anything
75 /// else — callers surface their own error/warning.
76 ///
77 /// docling canonicalizes OCR languages across its engines (docling#4075):
78 /// a bare value is the engine's native code, an `iso:`-prefixed value a
79 /// BCP-47 tag reduced to a language-script pair with the region dropped
80 /// (`zh-CN` and `zh-Hans` are the same recognizer, `en-GB` is `en`), and
81 /// the RapidOCR adapter maps `en` → its `en` model and `zh-Hans` → `ch`.
82 /// With only those two PP-OCRv3 pairs on board there is no ambiguity, so
83 /// the prefix is optional here: `en-US`, `eng`, `zh`, `zh-Hans`, `iso:zh-CN`
84 /// all resolve without a warning. Accepted primary subtags: English as
85 /// `en` / ISO 639-2/3 `eng` / docling's legacy `english`; Chinese as the
86 /// engine code `ch` (and RapidOCR's `chinese_cht`), `zh` / `zho` / `chi` /
87 /// `cmn` / legacy `chinese` / EasyOCR's `ch_sim` / `ch_tra`. Script,
88 /// region and variant subtags (`-Hans`, `-Hant`, `-CN`, `-TW`, `_US`) are
89 /// ignored: a traditional-script request (`zh-Hant`, `zh-TW`) gets the
90 /// multilingual `ch` recognizer too, the closest model shipped — upstream
91 /// would pick RapidOCR's separate `chinese_cht`, which this engine does
92 /// not carry. Genuinely unsupported languages (`de`, `fr`, `ja`, …) parse
93 /// to `None` and keep warning.
94 pub fn parse(s: &str) -> Option<Self> {
95 let token = s.trim().to_ascii_lowercase();
96 let tag = token.strip_prefix("iso:").unwrap_or(&token).trim();
97 let primary = tag.split(['-', '_']).next().unwrap_or_default();
98 match primary {
99 "en" | "eng" | "english" => Some(Self::En),
100 "ch" | "chinese_cht" | "zh" | "zho" | "chi" | "cmn" | "chinese" | "ch_sim"
101 | "ch_tra" => Some(Self::Ch),
102 _ => None,
103 }
104 }
105
106 /// The process-level choice from `DOCLING_RS_OCR_LANG` (empty/unset → the
107 /// English default; unknown values warn and use English). Under a
108 /// `DOCLING_RS_OCR_ENGINE=tesseract` default the variable is Tesseract's
109 /// language list (`rus+eng`, [`crate::tesseract::lang_arg`]), not a
110 /// PP-OCR model choice: a value naming no en/ch model then selects the
111 /// English model silently — it is only used if a request switches the
112 /// engine back to PP-OCR — instead of warning on every request (#889).
113 pub fn from_env() -> Self {
114 let raw = docling_core::env::nonempty("DOCLING_RS_OCR_LANG");
115 Self::from_env_value(raw.as_deref(), OcrEngine::from_env)
116 }
117
118 /// [`Self::from_env`] on the variable's value and the default engine
119 /// (asked only when the value names no model).
120 fn from_env_value(raw: Option<&str>, engine: impl Fn() -> OcrEngine) -> Self {
121 let Some(raw) = raw else {
122 return Self::default();
123 };
124 Self::parse(raw).unwrap_or_else(|| {
125 if engine() == OcrEngine::Tesseract {
126 return Self::default();
127 }
128 eprintln!(
129 "docling-pdf: DOCLING_RS_OCR_LANG={raw:?} names no language the en/ch \
130 recognizers read ({}); using en",
131 Self::ACCEPTED
132 );
133 Self::default()
134 })
135 }
136
137 /// The accepted spellings, for error messages and docs.
138 pub const ACCEPTED: &'static str =
139 "en | ch, or a BCP-47 tag for English or Chinese such as en-US, eng, zh, zh-Hans, zh-TW";
140}
141
142/// Which document regions feed the OCR — docling 2.116's `OcrMode` (#254,
143/// upstream docling#3710). Upstream restructured its pipeline so OCR runs
144/// *after* layout, on layout regions filtered by the PDF text layer — the
145/// architecture this port has always had — and named the strategies:
146///
147/// - `PdfAwareLayoutRegions` (upstream's **default**): OCR only layout regions
148/// the embedded text layer can't cover. Exactly the standard path here —
149/// scanned pages OCR their regions, digital pages OCR their bare pictures
150/// and every text/furniture/table region no programmatic text touches
151/// (vector-outlined text beside a native text layer, #931).
152/// - `FullPage` / `LayoutRegions`: ignore the PDF text layer and OCR
153/// everything. Both map onto the [`force_full_page_ocr`] machinery (discard
154/// the text layer, OCR every layout region): the upstream distinction —
155/// whole-page vs per-region *detector* input — has no analogue in this
156/// engine, whose PP-OCR recognizer always consumes per-region line crops.
157///
158/// [`force_full_page_ocr`]: crate::Pipeline::force_full_page_ocr
159#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
160pub enum OcrMode {
161 /// Upstream's `default`: currently wired to `PdfAwareLayoutRegions`.
162 #[default]
163 Default,
164 /// OCR the full page, text layer ignored (docling's `full_page`; the
165 /// mode-shaped spelling of `force_full_page_ocr`).
166 FullPage,
167 /// OCR every layout region, text layer ignored (docling's
168 /// `layout_regions`).
169 LayoutRegions,
170 /// OCR layout regions the text layer can't cover (docling's
171 /// `pdf_aware_layout_regions` — the default behavior).
172 PdfAwareLayoutRegions,
173}
174
175impl OcrMode {
176 /// Parse docling's mode ids. `None` for anything else — callers surface
177 /// their own error/warning.
178 pub fn parse(s: &str) -> Option<Self> {
179 match s.trim().to_ascii_lowercase().as_str() {
180 "default" => Some(Self::Default),
181 "full_page" => Some(Self::FullPage),
182 "layout_regions" => Some(Self::LayoutRegions),
183 "pdf_aware_layout_regions" => Some(Self::PdfAwareLayoutRegions),
184 _ => None,
185 }
186 }
187
188 /// The process-level choice from `DOCLING_RS_OCR_MODE` (empty/unset → the
189 /// default; unknown values warn and use the default).
190 pub fn from_env() -> Self {
191 let Some(raw) = docling_core::env::nonempty("DOCLING_RS_OCR_MODE") else {
192 return Self::default();
193 };
194 Self::parse(&raw).unwrap_or_else(|| {
195 eprintln!(
196 "docling-pdf: DOCLING_RS_OCR_MODE={raw:?} is not \
197 default|full_page|layout_regions|pdf_aware_layout_regions; using default"
198 );
199 Self::default()
200 })
201 }
202
203 /// Whether this mode discards the embedded text layer — the engine truth
204 /// both non-default modes reduce to.
205 pub fn forces_full_page(self) -> bool {
206 matches!(self, Self::FullPage | Self::LayoutRegions)
207 }
208}
209
210/// Which OCR engine recognizes text (#460): the built-in PP-OCRv3 recognizer
211/// (+ the RapidOCR text detector) — the default and the engine every
212/// conformance baseline is pinned against — or the system `tesseract` binary
213/// (see [`crate::tesseract`]), docling's `TesseractCliOcrOptions`
214/// counterpart. Both consume the same layout-region crops and produce the
215/// same cells; everything downstream is engine-agnostic.
216#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
217pub enum OcrEngine {
218 /// PP-OCRv3 recognition via ONNX Runtime (docling's `rapidocr` kind).
219 #[default]
220 PpOcr,
221 /// The `tesseract` CLI (docling's `tesseract` kind).
222 Tesseract,
223}
224
225impl OcrEngine {
226 /// Parse an engine id: `ppocr` (also `pp-ocr`, `rapidocr`, docling's
227 /// kind name) or `tesseract` (also `tesseract_cli`, `tesserocr`),
228 /// trimmed and case-insensitive. `None` for anything else.
229 pub fn parse(s: &str) -> Option<Self> {
230 match s.trim().to_ascii_lowercase().as_str() {
231 "ppocr" | "pp-ocr" | "pp_ocr" | "rapidocr" | "onnx" | "default" => Some(Self::PpOcr),
232 "tesseract" | "tesseract_cli" | "tesseract-cli" | "tesserocr" => Some(Self::Tesseract),
233 _ => None,
234 }
235 }
236
237 /// The process-level choice from `DOCLING_RS_OCR_ENGINE` (empty/unset →
238 /// PP-OCR; unknown values warn and use PP-OCR).
239 pub fn from_env() -> Self {
240 let Some(raw) = docling_core::env::nonempty("DOCLING_RS_OCR_ENGINE") else {
241 return Self::default();
242 };
243 Self::parse(&raw).unwrap_or_else(|| {
244 eprintln!(
245 "docling-pdf: DOCLING_RS_OCR_ENGINE={raw:?} is not ppocr|tesseract; using ppocr"
246 );
247 Self::default()
248 })
249 }
250
251 /// The accepted spellings, for error messages and docs.
252 pub const ACCEPTED: &'static str = "ppocr | tesseract";
253
254 /// Whether `raw` is an `ocr_lang` this engine can act on — what the
255 /// option surfaces validate up front: the en/ch model switch (or a
256 /// BCP-47 tag for either, [`OcrLang::parse`]) under PP-OCR; tessdata
257 /// stems and BCP-47 tags ([`crate::tesseract::lang_arg`]) under
258 /// Tesseract. The `Err` says what is accepted.
259 pub fn validate_lang(self, raw: &str) -> Result<(), String> {
260 match self {
261 Self::PpOcr => OcrLang::parse(raw).map(|_| ()).ok_or_else(|| {
262 format!(
263 "ocr_lang {raw:?} names no language the OCR models read ({})",
264 OcrLang::ACCEPTED
265 )
266 }),
267 Self::Tesseract => crate::tesseract::lang_arg(raw).map(|_| ()),
268 }
269 }
270}
271
272/// The process-level OCR render scale from `DOCLING_RS_OCR_SCALE` (#254,
273/// upstream docling#3877's `OcrOptions.scale`): pixels per PDF point fed to
274/// the recognizer. Unset/empty → `None` (OCR reads the pipeline's own page
275/// render, 2.0 px/pt); non-positive or unparsable values warn and are ignored.
276pub fn scale_from_env() -> Option<f32> {
277 positive_scale_env("DOCLING_RS_OCR_SCALE")
278}
279
280/// The process-level picture OCR scale from `DOCLING_RS_PICTURE_OCR_SCALE`
281/// (#666): pixels per picture pixel for the picture-OCR enrichment, before
282/// RapidOCR's 2000 px cap — see [`crate::Pipeline::picture_ocr_scale`].
283/// Unset/empty → `None`; non-positive or unparsable values warn and are
284/// ignored.
285pub fn picture_scale_from_env() -> Option<f32> {
286 positive_scale_env("DOCLING_RS_PICTURE_OCR_SCALE")
287}
288
289fn positive_scale_env(name: &str) -> Option<f32> {
290 let raw = docling_core::env::nonempty(name)?;
291 match raw.parse::<f32>() {
292 Ok(s) if s > 0.0 && s.is_finite() => Some(s),
293 _ => {
294 eprintln!("docling-pdf: {name}={raw:?} is not a positive number; ignored");
295 None
296 }
297 }
298}
299
300/// The process-level default of [`crate::Pipeline::ocr_textline_orientation`]
301/// from `DOCLING_RS_OCR_TEXTLINE_ORIENTATION` (#666; a `docling_core::env`
302/// flag). Off when unset.
303pub fn textline_orientation_from_env() -> bool {
304 docling_core::env::flag("DOCLING_RS_OCR_TEXTLINE_ORIENTATION")
305}
306
307/// Whether the text detector's boxes are the recognizer's line source inside
308/// layout regions (#570; `DOCLING_RS_OCR_LINES`: `det` default, `projection`
309/// = the ink-projection strips alone, the pre-#570 behavior). Cached.
310pub fn det_lines() -> bool {
311 static MODE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
312 *MODE.get_or_init(|| {
313 let raw = docling_core::env::nonempty("DOCLING_RS_OCR_LINES").unwrap_or_default();
314 match raw.trim().to_ascii_lowercase().as_str() {
315 "" | "det" | "detector" | "auto" => true,
316 "projection" | "proj" | "off" => false,
317 _ => {
318 eprintln!(
319 "docling-pdf: DOCLING_RS_OCR_LINES={raw:?} is not det|projection; using det"
320 );
321 true
322 }
323 }
324 })
325}
326
327/// Resolve the recognition model + dictionary pair for `lang`. An English
328/// default that isn't on disk (older model checkouts) degrades to the `ch_`
329/// pair with a warning rather than failing — the usual missing-optional-asset
330/// convention. Explicit `DOCLING_OCR_REC_ONNX` / `DOCLING_OCR_DICT` paths win
331/// over all of this; they are a pair, so set both together.
332pub(crate) fn resolve_rec_pair(lang: OcrLang) -> (String, String) {
333 const CH: (&str, &str) = (".models/ocr_rec.onnx", ".models/ppocr_keys_v1.txt");
334 const EN: (&str, &str) = (".models/ocr_rec_en.onnx", ".models/en_dict.txt");
335 const V6: (&str, &str) = (".models/ocr_rec_v6.onnx", ".models/ocr_rec_v6_dict.txt");
336 let exists = |p: &str| std::path::Path::new(p).exists();
337 // The multilingual PP-OCRv6 recognizer, when installed, serves both
338 // languages (see `OcrLang`); explicit paths below still win.
339 let (v6_rec, v6_dict) = (crate::resolve_asset(V6.0), crate::resolve_asset(V6.1));
340 let (mut rec, mut dict) = if exists(&v6_rec) && exists(&v6_dict) {
341 (v6_rec, v6_dict)
342 } else {
343 let pick = if lang == OcrLang::Ch { CH } else { EN };
344 (crate::resolve_asset(pick.0), crate::resolve_asset(pick.1))
345 };
346 let want_ch = lang == OcrLang::Ch;
347 if !want_ch && (!exists(&rec) || !exists(&dict)) {
348 let (ch_rec, ch_dict) = (crate::resolve_asset(CH.0), crate::resolve_asset(CH.1));
349 if std::path::Path::new(&ch_rec).exists() && std::path::Path::new(&ch_dict).exists() {
350 eprintln!(
351 "docling-pdf: English OCR model not found ({rec}); falling back to the \
352 multilingual ch_ model — expect weak Latin word spacing. Fetch it with \
353 scripts/install/download_dependencies.sh"
354 );
355 (rec, dict) = (ch_rec, ch_dict);
356 }
357 }
358 (
359 docling_core::env::nonempty("DOCLING_OCR_REC_ONNX").unwrap_or(rec),
360 docling_core::env::nonempty("DOCLING_OCR_DICT").unwrap_or(dict),
361 )
362}
363
364/// One recognised line: its text and mean emitted-character confidence.
365type Recognized = (String, f32);
366
367/// One crop read by [`OcrModel::recognize_crops`] /
368/// [`OcrModel::recognize_det_boxes`] (#939).
369#[derive(Debug, Clone, PartialEq)]
370pub struct RecognizedLine {
371 /// The CTC-decoded text, trimmed of surrounding whitespace (as the page
372 /// OCR trims its cells); empty when nothing decoded.
373 pub text: String,
374 /// The *recognition* confidence: the mean probability of the emitted
375 /// characters (PaddleOCR's per-line score — what the page OCR reports on
376 /// its cells and `ConfidenceReport::ocr_cells`), `0.0` when nothing
377 /// decoded. Not the detector's [`DetBox::score`](crate::ocr_det::DetBox).
378 pub confidence: f32,
379}
380
381impl RecognizedLine {
382 /// Whether the page OCR would keep this line: non-empty text at or above
383 /// RapidOCR's `text_score` ([`crate::ocr_prep::text_score`], 0.5,
384 /// `DOCLING_RS_OCR_TEXT_SCORE`). The crop API itself drops nothing.
385 pub fn passes_text_score(&self) -> bool {
386 !self.text.is_empty() && self.confidence >= crate::ocr_prep::text_score()
387 }
388}
389
390impl OcrModel {
391 /// Load the recognition model and its character dictionary for `lang` —
392 /// see `resolve_rec_pair` for the selection rules (explicit
393 /// `DOCLING_OCR_REC_ONNX`/`DOCLING_OCR_DICT` paths win) — with `lanes`
394 /// recognition sessions.
395 ///
396 /// Each session is pinned to one intra-op thread: ORT's multi-threaded
397 /// float-reduction order varies across runs, which flips the CTC argmax on
398 /// low-confidence characters (e.g. noisy faxes) and makes the snapshot
399 /// output non-deterministic. Recognition is linear in line width (~0.17 ms
400 /// per pixel column on one core) and on a scanned page it, plus the
401 /// orientation probe that reads the six widest lines, is ~35% of the wall
402 /// time while the other cores idle. Lines are independent, so `lanes`
403 /// sessions recognise disjoint same-width batches concurrently — each
404 /// line still sees exactly the single-thread kernel path, results are
405 /// placed by index, and the output is byte-identical to one lane.
406 /// `DOCLING_RS_OCR_SESSIONS` overrides the caller's lane count; under a
407 /// CUDA-class provider the default is one lane (#842) — the lanes exist
408 /// to use idle CPU cores, and on one GPU each is another session's arena.
409 pub fn load_with(lang: OcrLang, lanes: usize) -> Result<Self, String> {
410 let (rec_path, dict_path) = resolve_rec_pair(lang);
411 let lanes = docling_core::env::parse::<usize>("DOCLING_RS_OCR_SESSIONS")
412 .filter(|&n| n > 0)
413 .unwrap_or(if docling_onnx::prefers_single_worker() {
414 1
415 } else {
416 lanes
417 })
418 .clamp(1, 8);
419 let open = || -> Result<Session, String> {
420 let builder = docling_onnx::session_builder()
421 .map_err(|e| format!("ocr: builder: {e}"))?
422 .with_intra_threads(1)
423 .map_err(|e| format!("ocr: intra_threads: {e}"))?;
424 let builder = docling_onnx::apply(builder).map_err(|e| format!("ocr: {e}"))?;
425 docling_onnx::commit(builder, &rec_path, "rec")
426 .map_err(|e| format!("ocr: load {rec_path}: {e}"))
427 };
428 // The lanes are independent sessions over the same file — open them
429 // concurrently so extra lanes cost no extra start-up latency.
430 let recs: Vec<Session> = std::thread::scope(|s| {
431 let handles: Vec<_> = (0..lanes).map(|_| s.spawn(open)).collect();
432 handles
433 .into_iter()
434 .map(|h| {
435 h.join()
436 .map_err(|_| "ocr: session thread panicked".to_string())?
437 })
438 .collect::<Result<Vec<_>, String>>()
439 })?;
440 let dict = std::fs::read_to_string(&dict_path)
441 .map_err(|e| format!("ocr: read dict {dict_path}: {e}"))?;
442 Ok(Self {
443 recs,
444 chars: dict_chars(&dict),
445 orient: false,
446 cls: ClsSlot::Unloaded,
447 })
448 }
449
450 /// Turn RapidOCR's line orientation handling on or off (#666): oriented
451 /// crops, the 90° turn, and the 0/180 classifier on detector lines.
452 pub fn set_textline_orientation(&mut self, enable: bool) {
453 self.orient = enable;
454 }
455
456 /// The orientation classifier, loading it on first use; `None` (warned
457 /// once) when the model is not installed or fails to load.
458 fn cls(&mut self) -> Option<&mut crate::ocr_cls::TextlineCls> {
459 if matches!(self.cls, ClsSlot::Unloaded) {
460 self.cls = match crate::ocr_cls::TextlineCls::load() {
461 Ok(m) => ClsSlot::Ready(m),
462 Err(e) => {
463 static WARNED: std::sync::Once = std::sync::Once::new();
464 WARNED.call_once(|| {
465 eprintln!(
466 "warning: {e}; text lines are cropped and turned upright but not \
467 checked for 180° (run scripts/install/download_dependencies.sh)"
468 );
469 });
470 ClsSlot::Missing
471 }
472 };
473 }
474 match &mut self.cls {
475 ClsSlot::Ready(m) => Some(m),
476 _ => None,
477 }
478 }
479
480 /// Turn the crops the orientation classifier reads as upside down by
481 /// 180°, in place; a no-op without the classifier (warned once).
482 fn upright(&mut self, mut crops: Vec<&mut RgbImage>) -> Result<(), String> {
483 if crops.is_empty() {
484 return Ok(());
485 }
486 let Some(cls) = self.cls() else {
487 return Ok(());
488 };
489 let refs: Vec<&RgbImage> = crops.iter().map(|c| &**c).collect();
490 let flip = crate::timing::timed("ocr.cls", || cls.upside_down(&refs))?;
491 for (c, flip) in crops.iter_mut().zip(flip) {
492 if flip {
493 image::imageops::rotate180_in_place(*c);
494 }
495 }
496 Ok(())
497 }
498
499 /// The oriented line prep (#666): RapidOCR's crops, the upside-down ones
500 /// turned 180° by the classifier when it is installed, then prepared
501 /// for recognition like any other line.
502 fn oriented_lines(
503 &mut self,
504 img: &RgbImage,
505 regions: &[Region],
506 scale: f32,
507 detected: &[crate::ocr_det::DetBox],
508 ) -> Result<(Vec<crate::ocr_prep::LineBox>, Vec<PrepLine>), String> {
509 use crate::ocr_prep::LineSrc;
510 let (bboxes, mut srcs) =
511 crate::ocr_prep::region_line_crops_det(img, regions, scale, detected);
512 let crops: Vec<&mut RgbImage> = srcs
513 .iter_mut()
514 .filter_map(|s| match s {
515 LineSrc::Crop(c) => Some(c),
516 LineSrc::Prepared(_) => None,
517 })
518 .collect();
519 self.upright(crops)?;
520 let mut boxes = Vec::with_capacity(bboxes.len());
521 let mut lines = Vec::with_capacity(bboxes.len());
522 for (bb, src) in bboxes.into_iter().zip(srcs) {
523 let line = match src {
524 LineSrc::Crop(c) => crate::ocr_prep::prep_line(&c),
525 LineSrc::Prepared(p) => Some(p),
526 };
527 if let Some(line) = line {
528 boxes.push(bb);
529 lines.push(line);
530 }
531 }
532 Ok((boxes, lines))
533 }
534
535 /// Recognize standalone line crops (#939) — no layout model, no page
536 /// assembly, no detector: each image is taken to be one upright text
537 /// line (dark ink on light ground, as the recognizer was trained),
538 /// resized to the recognizer's 48 px height at its aspect ratio (width
539 /// clamped to 8 px–[`crate::ocr_prep::REC_MAX_WIDTH`]) and read with the same deterministic
540 /// width-batching and recognition lanes as the page OCR — so a crop
541 /// yields what the page OCR would read from the same pixels.
542 ///
543 /// Returns exactly one [`RecognizedLine`] per crop, in input order; a
544 /// zero-sized crop reads as empty text with confidence `0.0`. Nothing is
545 /// filtered — the page OCR's `text_score` drop is
546 /// [`RecognizedLine::passes_text_score`], left to the caller — and no
547 /// orientation handling runs here (see [`Self::recognize_det_boxes`]).
548 pub fn recognize_crops(&mut self, crops: &[RgbImage]) -> Result<Vec<RecognizedLine>, String> {
549 let prepped: Vec<Option<PrepLine>> = crops.iter().map(crate::ocr_prep::prep_line).collect();
550 self.recognize_prepped(prepped)
551 }
552
553 /// Recognize the lines a text detector found on `img` (#939) — the
554 /// boxes [`DetModel::detect`](crate::ocr_det::DetModel) returns, in
555 /// `img` pixels — without running page layout. Each box is cropped as
556 /// the page OCR crops it ([`crate::ocr_prep::det_box_crop`]): RapidOCR's
557 /// warp of the box's quad, upright out of the detector's rotated
558 /// rectangle (tall lines turned 90°; #981) and, with
559 /// [`Self::set_textline_orientation`] on, a 180° turn where the
560 /// orientation classifier reads it upside down.
561 ///
562 /// One [`RecognizedLine`] per box, in input order (so `boxes[i]` is the
563 /// geometry of result `i`, in `img` pixels); a box with nothing inside
564 /// the image reads as empty with confidence `0.0`. Nothing is filtered,
565 /// as in [`Self::recognize_crops`].
566 pub fn recognize_det_boxes(
567 &mut self,
568 img: &RgbImage,
569 boxes: &[crate::ocr_det::DetBox],
570 ) -> Result<Vec<RecognizedLine>, String> {
571 let oriented = self.orient;
572 let mut crops: Vec<Option<RgbImage>> = boxes
573 .iter()
574 .map(|d| crate::ocr_prep::det_box_crop(img, d))
575 .collect();
576 if oriented {
577 self.upright(crops.iter_mut().flatten().collect())?;
578 }
579 let prepped = crops
580 .iter()
581 .map(|c| c.as_ref().and_then(crate::ocr_prep::prep_line))
582 .collect();
583 self.recognize_prepped(prepped)
584 }
585
586 /// Recognize the prepared lines, keeping a slot (empty, `0.0`) for each
587 /// `None`, so the results stay index-aligned with the caller's input.
588 fn recognize_prepped(
589 &mut self,
590 prepped: Vec<Option<PrepLine>>,
591 ) -> Result<Vec<RecognizedLine>, String> {
592 let mut slots = Vec::with_capacity(prepped.len());
593 let mut lines = Vec::with_capacity(prepped.len());
594 for p in prepped {
595 slots.push(p.as_ref().map(|_| lines.len()));
596 lines.extend(p);
597 }
598 let mut read = vec![(String::new(), 0.0f32); lines.len()];
599 crate::timing::timed("ocr.rec", || -> Result<(), String> {
600 for (i, r) in self.recognize_all(&lines)? {
601 read[i] = r;
602 }
603 Ok(())
604 })?;
605 Ok(slots
606 .into_iter()
607 .map(|slot| match slot {
608 Some(i) => {
609 let (text, confidence) = std::mem::take(&mut read[i]);
610 RecognizedLine {
611 text: text.trim().to_string(),
612 confidence,
613 }
614 }
615 None => RecognizedLine {
616 text: String::new(),
617 confidence: 0.0,
618 },
619 })
620 .collect())
621 }
622
623 /// Recognise every width batch of `lines`, dealt round-robin across the
624 /// lanes, and return `(line index, (text, confidence))` in batch order —
625 /// the same order the sequential loop produced, whatever the scheduling.
626 fn recognize_all(&mut self, lines: &[PrepLine]) -> Result<Vec<(usize, Recognized)>, String> {
627 let batches = width_batches(lines);
628 let lanes = self.recs.len().min(batches.len()).max(1);
629 let chars = &self.chars;
630 // One result slot per batch keeps the merge order independent of
631 // which lane finished first.
632 let mut per_batch: Vec<Option<Result<Vec<Recognized>, String>>> =
633 (0..batches.len()).map(|_| None).collect();
634 if lanes <= 1 {
635 for (slot, (w, chunk)) in per_batch.iter_mut().zip(&batches) {
636 *slot = Some(recognize_batch(&mut self.recs[0], chars, *w, chunk, lines));
637 }
638 } else {
639 std::thread::scope(|s| {
640 let handles: Vec<_> = self
641 .recs
642 .iter_mut()
643 .take(lanes)
644 .enumerate()
645 .map(|(lane, rec)| {
646 let batches = &batches;
647 s.spawn(move || {
648 batches
649 .iter()
650 .enumerate()
651 .filter(|(k, _)| k % lanes == lane)
652 .map(|(k, (w, chunk))| {
653 (k, recognize_batch(rec, chars, *w, chunk, lines))
654 })
655 .collect::<Vec<_>>()
656 })
657 })
658 .collect();
659 for h in handles {
660 for (k, r) in h.join().expect("ocr lane panicked") {
661 per_batch[k] = Some(r);
662 }
663 }
664 });
665 }
666 let mut out = Vec::with_capacity(lines.len());
667 for ((_, chunk), slot) in batches.iter().zip(per_batch) {
668 let texts = slot.expect("every batch is assigned a lane")?;
669 out.extend(chunk.iter().copied().zip(texts));
670 }
671 Ok(out)
672 }
673
674 /// Recognise a batch of prepared *same-width* lines in one session run.
675 ///
676 /// Only equal widths ever share a run: same-width batching is
677 /// bit-identical to one-at-a-time recognition (each sample keeps its own
678 /// data and per-sample kernel reduction order — verified empirically on
679 /// the scanned corpus), whereas width-padding leaks into the real
680 /// timesteps through the model's global-attention blocks and measurably
681 /// changes low-confidence characters.
682 /// Recognize `lines` and reduce to orientation-probe evidence: the
683 /// confidence-weighted character count `Σ(conf × chars)` plus the raw
684 /// character total (#225). Same deterministic width-batching as page OCR.
685 pub(crate) fn score_lines(&mut self, lines: &[PrepLine]) -> Result<(f32, usize), String> {
686 // Same accumulation order as the sequential loop (batch order), so
687 // the f32 sum is bit-identical regardless of lane scheduling.
688 let mut weighted = 0.0f32;
689 let mut chars = 0usize;
690 for (_, (text, conf)) in self.recognize_all(lines)? {
691 let n = text.trim().chars().count();
692 weighted += conf * n as f32;
693 chars += n;
694 }
695 Ok((weighted, chars))
696 }
697
698 /// OCR a page: produce text cells (page points) for every line found inside
699 /// the text regions, each paired with its recognition confidence (mean
700 /// emitted-character probability — feeds the page `ocr_score`, #183).
701 /// `scale` is image-px per page-point. `detected` — the text detector's
702 /// boxes, in image pixels of `img` — makes them the line source inside
703 /// the regions (#570, see [`prep_region_lines_det`]); `None` keeps the
704 /// projection segmentation.
705 pub fn ocr_page_with(
706 &mut self,
707 img: &RgbImage,
708 regions: &[Region],
709 scale: f32,
710 detected: Option<&[crate::ocr_det::DetBox]>,
711 ) -> Result<Vec<(TextCell, f32)>, String> {
712 // Gather every line crop on the page first (shared with the browser
713 // path), so equal-width lines can share a recognition run regardless
714 // of which region they came from.
715 let (bboxes, lines) = match detected {
716 Some(det) if self.orient => self.oriented_lines(img, regions, scale, det)?,
717 _ => crate::timing::timed("ocr.prep", || match detected {
718 Some(det) => prep_region_lines_det(img, regions, scale, det),
719 None => prep_region_lines(img, regions, scale),
720 }),
721 };
722
723 // Deterministic width-batching (shared with the wasm path), dealt
724 // across the recognition lanes.
725 let mut texts = vec![(String::new(), 0.0f32); lines.len()];
726 crate::timing::timed("ocr.rec", || -> Result<(), String> {
727 for (i, text) in self.recognize_all(&lines)? {
728 texts[i] = text;
729 }
730 Ok(())
731 })?;
732
733 // Emit cells in page order, exactly as the sequential walk did.
734 Ok(collect_cells(bboxes, texts))
735 }
736
737 /// Recognize the *word* crops inside the page's table regions (mirroring
738 /// the browser scanned path): [`ocr_page_with`](Self::ocr_page_with) deliberately
739 /// skips table labels, so a scanned table would otherwise reach the cell
740 /// matcher with no words at all and dissolve (#173). Returns word-level
741 /// [`TextCell`]s in page points.
742 pub fn ocr_table_words(
743 &mut self,
744 img: &RgbImage,
745 regions: &[Region],
746 scale: f32,
747 detected: Option<&[crate::ocr_det::DetBox]>,
748 ) -> Result<Vec<(TextCell, f32)>, String> {
749 let (bboxes, lines) = match detected {
750 Some(det) => prep_table_words_det(img, regions, scale, det),
751 None => prep_table_words(img, regions, scale),
752 };
753 let mut texts = vec![(String::new(), 0.0f32); lines.len()];
754 for (i, text) in self.recognize_all(&lines)? {
755 texts[i] = text;
756 }
757 Ok(collect_cells(bboxes, texts))
758 }
759}
760
761/// Pair recognized texts with their line boxes into page-point cells, in page
762/// order, dropping empty lines and those under [`text_score`].
763fn collect_cells(
764 bboxes: Vec<crate::ocr_prep::LineBox>,
765 texts: Vec<Recognized>,
766) -> Vec<(TextCell, f32)> {
767 let min_conf = crate::ocr_prep::text_score();
768 let mut cells = Vec::new();
769 for ((l, t, r, b), (text, conf)) in bboxes.into_iter().zip(texts) {
770 let text = text.trim().to_string();
771 if text.is_empty() || conf < min_conf {
772 continue;
773 }
774 cells.push((TextCell { text, l, t, r, b }, conf));
775 }
776 cells
777}
778
779/// Recognise a batch of prepared *same-width* lines in one run of `rec`.
780///
781/// Only equal widths ever share a run: same-width batching is bit-identical
782/// to one-at-a-time recognition (each sample keeps its own data and per-sample
783/// kernel reduction order — verified empirically on the scanned corpus),
784/// whereas width-padding leaks into the real timesteps through the model's
785/// global-attention blocks and measurably changes low-confidence characters.
786fn recognize_batch(
787 rec: &mut Session,
788 chars: &[String],
789 w: usize,
790 chunk: &[usize],
791 lines: &[PrepLine],
792) -> Result<Vec<(String, f32)>, String> {
793 let n = chunk.len();
794 let data = batch_input(w, chunk, lines);
795 let input = Tensor::from_array(([n, 3, REC_HEIGHT as usize, w], data))
796 .map_err(|e| format!("ocr: input tensor: {e}"))?;
797 let outputs = docling_onnx::run(rec, ort::inputs!["x" => input])
798 .map_err(|e| format!("ocr: rec inference: {e}"))?;
799 let (shape, probs) = outputs[0]
800 .try_extract_tensor::<f32>()
801 .map_err(|e| format!("ocr: extract rec: {e}"))?;
802 let t_len = shape[1] as usize;
803 let nc = shape[2] as usize;
804 Ok((0..n)
805 .map(|i| decode_row_scored(chars, &probs[i * t_len * nc..(i + 1) * t_len * nc], nc))
806 .collect())
807}
808
809#[cfg(test)]
810mod tests {
811 use super::*;
812
813 /// #889: under a Tesseract default, `DOCLING_RS_OCR_LANG` is Tesseract's
814 /// language list — a value naming no en/ch model picks English without
815 /// the PP-OCR warning; under PP-OCR an en/ch value still selects it.
816 #[test]
817 fn env_lang_under_a_tesseract_default_is_not_a_model_choice() {
818 let tess = || OcrEngine::Tesseract;
819 let pp = || OcrEngine::PpOcr;
820 assert_eq!(OcrLang::from_env_value(Some("rus+eng"), tess), OcrLang::En);
821 assert_eq!(OcrLang::from_env_value(Some("rus+eng"), pp), OcrLang::En);
822 assert_eq!(OcrLang::from_env_value(Some("zh-Hans"), tess), OcrLang::Ch);
823 assert_eq!(OcrLang::from_env_value(None, pp), OcrLang::En);
824 }
825
826 /// #388: BCP-47 tags and the ISO 639-2/3 codes for English and Chinese
827 /// resolve to the two recognizers, with or without docling's `iso:`
828 /// prefix and whatever the script/region subtags; other languages and
829 /// nonsense stay `None`.
830 #[test]
831 fn ocr_lang_accepts_bcp47_tags_for_the_two_recognizers() {
832 for id in [
833 "en",
834 "EN",
835 " en ",
836 "en-US",
837 "en_GB",
838 "eng",
839 "english",
840 "iso:en",
841 "ISO:en-GB",
842 "en-Latn-US",
843 ] {
844 assert_eq!(OcrLang::parse(id), Some(OcrLang::En), "{id:?}");
845 }
846 for id in [
847 "ch",
848 "zh",
849 "zho",
850 "chi",
851 "cmn",
852 "chinese",
853 "ch_sim",
854 "ch_tra",
855 "chinese_cht",
856 "zh-Hans",
857 "zh-Hant",
858 "zh-CN",
859 "zh-TW",
860 "zh-Hant-HK",
861 "zh_SG",
862 "iso:zh-Hans",
863 ] {
864 assert_eq!(OcrLang::parse(id), Some(OcrLang::Ch), "{id:?}");
865 }
866 for id in [
867 "", "de", "fr-FR", "ja", "deu", "cn", "latin", "iso:", "iso:und", "e",
868 ] {
869 assert_eq!(OcrLang::parse(id), None, "{id:?}");
870 }
871 }
872
873 /// #254: docling's four `OcrMode` ids parse; `full_page`/`layout_regions`
874 /// reduce to the force-full-page machinery, the default/pdf-aware pair to
875 /// the standard text-layer-aware path. Unknown ids parse to nothing.
876 #[test]
877 fn ocr_mode_ids_parse_and_map_to_forcing() {
878 for (id, mode, forces) in [
879 ("default", OcrMode::Default, false),
880 ("full_page", OcrMode::FullPage, true),
881 ("layout_regions", OcrMode::LayoutRegions, true),
882 (
883 "pdf_aware_layout_regions",
884 OcrMode::PdfAwareLayoutRegions,
885 false,
886 ),
887 ] {
888 assert_eq!(OcrMode::parse(id), Some(mode));
889 assert_eq!(mode.forces_full_page(), forces, "{id}");
890 }
891 assert_eq!(OcrMode::parse(" Full_Page "), Some(OcrMode::FullPage));
892 assert_eq!(OcrMode::parse("easyocr"), None);
893 assert_eq!(OcrMode::parse(""), None);
894 }
895
896 /// #460: the engine ids, and engine-aware `ocr_lang` validation — `deu`
897 /// is a Tesseract stem, not a PP-OCR model; `en` works under both.
898 #[test]
899 fn ocr_engine_ids_and_lang_validation() {
900 for id in ["ppocr", "PP-OCR", " rapidocr ", "default"] {
901 assert_eq!(OcrEngine::parse(id), Some(OcrEngine::PpOcr), "{id:?}");
902 }
903 for id in ["tesseract", "Tesseract_CLI", "tesserocr"] {
904 assert_eq!(OcrEngine::parse(id), Some(OcrEngine::Tesseract), "{id:?}");
905 }
906 assert_eq!(OcrEngine::parse("easyocr"), None);
907 assert!(OcrEngine::PpOcr.validate_lang("en").is_ok());
908 assert!(OcrEngine::PpOcr.validate_lang("deu").is_err());
909 assert!(OcrEngine::Tesseract.validate_lang("en").is_ok());
910 assert!(OcrEngine::Tesseract.validate_lang("deu+fra").is_ok());
911 assert!(OcrEngine::Tesseract.validate_lang("xx").is_err());
912 }
913}