Skip to main content

docling_pdf/
lib.rs

1//! PDF backend for docling.rs.
2//!
3//! A port of docling's standard PDF pipeline: pdfium extracts the text layer
4//! (cells with bounding boxes) and renders page images; a discriminative ONNX
5//! stack (layout detection, table structure, OCR) classifies regions; the cells
6//! are assembled in reading order into a [`DoclingDocument`].
7//!
8//! Current stages: pdfium text-cell extraction + page rendering ([`pdfium_backend`])
9//! and the deterministic text/reading-order assembly ([`assemble`]). The layout,
10//! table-structure and OCR ONNX stages land behind [`Pipeline`] next.
11
12// Without `ml` only the text-layer path runs; the shared assembly/label
13// helpers it doesn't exercise stay compiled for API stability (the full
14// build still flags genuinely dead code).
15#![cfg_attr(not(feature = "ml"), allow(dead_code))]
16
17// Reading-order assembly. Public under `ocr-prep` so the browser pipeline can
18// reuse the geometric table reconstruction and its reliability gate (#157).
19#[cfg(feature = "ocr-prep")]
20pub mod assemble;
21#[cfg(not(feature = "ocr-prep"))]
22mod assemble;
23mod dp_lines;
24#[cfg(feature = "ml")]
25pub mod enrich;
26// Public so sibling crates (e.g. docling-rag's ONNX embedder) can route their
27// own `ort` sessions through the same `DOCLING_RS_EP` selection.
28#[cfg(feature = "ml")]
29pub mod ep;
30pub mod layout;
31#[cfg(feature = "ml")]
32mod mets;
33#[cfg(feature = "ml")]
34mod ocr;
35#[cfg(feature = "ocr-prep")]
36pub mod ocr_prep;
37pub mod pdfium_backend;
38#[cfg(feature = "ml")]
39pub mod quality;
40mod reading_order;
41// Pure-Rust region resampling (page→1024px box-average, crop→448 bilinear) —
42// available to the browser TableFormer path (#157 stage 3), not just `ml`.
43#[cfg(feature = "ocr-prep")]
44pub mod resample;
45#[cfg(feature = "ocr-prep")]
46pub mod scanned;
47// Built-in standard-14 font metrics for the pure-Rust text parser (#187) —
48// no feature gate: the wasm/pdf-text path needs them like the native one.
49mod std14;
50#[cfg(feature = "ml")]
51pub mod tableformer;
52pub mod textparse;
53#[cfg(feature = "ocr-prep")]
54pub mod tf_core;
55// docling's TableFormer cell matcher — pure Rust, shared with the browser
56// TableFormer path (#157 stage 3).
57#[cfg(feature = "ocr-prep")]
58pub mod tf_match;
59pub mod timing;
60
61#[cfg(feature = "ml")]
62use std::collections::BTreeMap;
63use std::fmt;
64#[cfg(feature = "ml")]
65use std::sync::mpsc::{sync_channel, Receiver};
66#[cfg(feature = "ml")]
67use std::sync::{Arc, Mutex};
68
69use docling_core::DoclingDocument;
70#[cfg(feature = "ml")]
71use docling_core::Node;
72
73#[cfg(feature = "ml")]
74pub use mets::{convert_mets_gbs, convert_mets_gbs_with_options};
75#[cfg(feature = "ml")]
76pub use ocr::OcrLang;
77#[cfg(feature = "ml")]
78pub use pdfium_backend::PdfDocument;
79pub use pdfium_backend::{PdfPage, TextCell};
80
81/// Errors from the PDF backend. Detailed and surfaced (never silently skipped).
82#[derive(Debug)]
83pub enum PdfError {
84    /// pdfium failed to bind, open, or read the document.
85    Pdfium(String),
86    /// The layout ONNX model failed to load or run.
87    Layout(String),
88    /// The OCR ONNX model failed to load or run.
89    Ocr(String),
90}
91
92impl fmt::Display for PdfError {
93    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
94        match self {
95            PdfError::Pdfium(m) => write!(f, "pdf: pdfium error: {m}"),
96            PdfError::Layout(m) => write!(f, "pdf: {m}"),
97            PdfError::Ocr(m) => write!(f, "pdf: {m}"),
98        }
99    }
100}
101
102impl std::error::Error for PdfError {}
103
104#[cfg(feature = "ml")]
105impl From<pdfium_render::prelude::PdfiumError> for PdfError {
106    fn from(e: pdfium_render::prelude::PdfiumError) -> Self {
107        PdfError::Pdfium(e.to_string())
108    }
109}
110
111/// Convert a PDF's **embedded text layer only** — no pdfium, no ONNX, no
112/// threads: the pure-Rust content-stream parser ([`textparse`]) feeds the same
113/// orphan-region assembly the `no_ocr` pipeline flag uses, so text-layer PDFs
114/// come out identical to `--no-ocr` (flat, line-grouped paragraphs in reading
115/// order; no headings/lists/tables/pictures, and no hyperlink recovery).
116///
117/// This is the only conversion entry compiled without the `ml` feature (it is
118/// what a wasm32 build runs). A scanned/image-only PDF (no embedded text
119/// layer) yields an empty document rather than an error, same as `no_ocr` —
120/// callers can detect that and fall back to an OCR-capable build.
121pub fn convert_text_layer(bytes: &[u8], name: &str) -> Result<DoclingDocument, PdfError> {
122    convert_text_layer_pages(bytes, name, None)
123}
124
125/// [`convert_text_layer`] restricted to a **1-based inclusive** page window
126/// (issue #80's `--pages`); `None` converts everything. The window is
127/// validated the same way as [`Pipeline::pages`]: `first <= last`, 1-based,
128/// and it must select at least one existing page.
129pub fn convert_text_layer_pages(
130    bytes: &[u8],
131    name: &str,
132    pages: Option<(usize, usize)>,
133) -> Result<DoclingDocument, PdfError> {
134    if let Some((first, last)) = pages {
135        if first == 0 || last < first {
136            return Err(PdfError::Pdfium(format!(
137                "invalid page range {first}-{last} (pages are 1-based, first <= last)"
138            )));
139        }
140    }
141    let mut doc = DoclingDocument::new(name);
142    let mut total = 0usize;
143    let parsed = textparse::pdf_text_pages(bytes);
144    // A vestigial layer (a few typed-in form fields over scanned pages) is not
145    // the document's text: return the empty document, which callers already
146    // report as "no text layer" — so an OCR-capable caller falls back to OCR
147    // instead of proudly extracting thirteen characters.
148    if textparse::text_layer_is_vestigial(&parsed) {
149        return Ok(doc);
150    }
151    for (i, page) in parsed.into_iter().enumerate() {
152        total += 1;
153        if let Some((first, last)) = pages {
154            if i + 1 < first || i + 1 > last {
155                continue;
156            }
157        }
158        let mut regions = Vec::new();
159        assemble::add_orphan_regions(&mut regions, &page.cells);
160        let table_rows = vec![None; regions.len()];
161        let enrich_out = vec![None; regions.len()];
162        let (mut nodes, links) = assemble::assemble_page(&page, regions, &table_rows, &enrich_out);
163        assemble::stamp_page_no(&mut nodes, i + 1);
164        doc.nodes.extend(nodes);
165        doc.links.extend(links);
166    }
167    if let Some((first, last)) = pages {
168        if first > total {
169            return Err(PdfError::Pdfium(format!(
170                "page range {first}-{last} is outside the document ({total} page(s))"
171            )));
172        }
173    }
174    assemble::merge_continuations(&mut doc.nodes);
175    Ok(doc)
176}
177
178/// Threads ONNX inference may use, capped by `DOCLING_RS_PDF_THREADS` if set.
179/// Defaults to the available parallelism (ort otherwise picks a low number).
180#[cfg(feature = "ml")]
181pub(crate) fn intra_threads() -> usize {
182    if let Some(n) = std::env::var("DOCLING_RS_PDF_THREADS")
183        .ok()
184        .and_then(|v| v.parse::<usize>().ok())
185        .filter(|&n| n > 0)
186    {
187        return n;
188    }
189    std::thread::available_parallelism()
190        .map(|n| n.get())
191        .unwrap_or(1)
192}
193
194#[cfg(feature = "ml")]
195/// True when `DOCLING_RS_FP32` (any value but `0`) forces the full-precision
196/// models even where an INT8 variant sits next to the fp32 default.
197pub(crate) fn fp32_forced() -> bool {
198    std::env::var("DOCLING_RS_FP32")
199        .map(|v| v != "0")
200        .unwrap_or(false)
201}
202
203#[cfg(feature = "ml")]
204/// Should the int8 model defaults be skipped in favor of fp32? Either the
205/// user said so (`DOCLING_RS_FP32`), or a GPU execution provider is selected
206/// (#74) — the int8 exports are QDQ graphs calibrated for CPU kernels and
207/// only conformance-validated there. An explicit `DOCLING_*_ONNX` path
208/// override still wins over this at every call site.
209pub(crate) fn prefer_fp32() -> bool {
210    fp32_forced() || ep::prefers_fp32()
211}
212
213#[cfg(feature = "ml")]
214/// Resolve a default (CWD-relative) asset path. If it doesn't exist relative
215/// to the current directory, try next to the executable and one level above
216/// it (following symlinks — the layout `scripts/install/install.sh` produces:
217/// `/usr/local/bin/docling-rs` → `/usr/local/docling.rs/bin/docling-rs`
218/// with `models/` and `.pdfium/` in `/usr/local/docling.rs`). Lets an
219/// installed binary run from any working directory with no env vars; explicit
220/// env overrides never reach this. Returns `rel` unchanged when nothing
221/// exists anywhere, so callers' error messages keep the familiar path.
222pub(crate) fn resolve_asset(rel: &str) -> String {
223    if std::path::Path::new(rel).exists() {
224        return rel.to_string();
225    }
226    if let Some(dir) = std::env::current_exe()
227        .ok()
228        .and_then(|p| p.canonicalize().ok())
229        .and_then(|p| p.parent().map(std::path::Path::to_path_buf))
230    {
231        for base in [Some(dir.as_path()), dir.parent()].into_iter().flatten() {
232            let p = base.join(rel);
233            if p.exists() {
234                return p.to_string_lossy().into_owned();
235            }
236        }
237    }
238    rel.to_string()
239}
240
241/// One resolved runtime asset — which file a stage would load right now,
242/// given the CWD, the env overrides and the int8/fp32 preference.
243#[cfg(feature = "ml")]
244#[derive(Debug, Clone)]
245pub struct ModelEntry {
246    /// Pipeline stage, e.g. `layout`, `tableformer.decoder`, `ocr.rec`.
247    pub stage: &'static str,
248    /// The resolved path (absolute or CWD-relative, as it will be opened).
249    pub path: String,
250    /// Whether the file exists right now.
251    pub found: bool,
252    /// File size in bytes (0 when missing) — enough to tell an int8 quant
253    /// from an fp32 graph, or a stale model from a re-published one, at a
254    /// glance without hashing gigabytes per request.
255    pub bytes: u64,
256}
257
258/// Resolve the whole runtime model set **without loading anything** — the
259/// exact selection each stage performs at load time (layout honors the
260/// int8/fp32 preference, TableFormer its decoder ranking, OCR the language
261/// pair), plus the pdfium library. docling-serve exposes this at
262/// `/v1/config` and logs it at startup, so "the server picked up different
263/// models" is one `curl` away instead of a mystery of dissolved tables.
264/// Resolution is CWD-relative with an exe-dir fallback, so the answer can
265/// legitimately differ between two working directories.
266#[cfg(feature = "ml")]
267pub fn model_inventory() -> Vec<ModelEntry> {
268    fn entry(stage: &'static str, path: String) -> ModelEntry {
269        let meta = std::fs::metadata(&path).ok();
270        ModelEntry {
271            stage,
272            found: meta.is_some(),
273            bytes: meta.map(|m| m.len()).unwrap_or(0),
274            path,
275        }
276    }
277    let (enc, dec, bbx) = tableformer::resolved_paths();
278    let (rec, dict) = ocr::resolve_rec_pair(ocr::OcrLang::from_env());
279    let pdfium =
280        std::env::var("PDFIUM_DYNAMIC_LIB_PATH").unwrap_or_else(|_| resolve_asset(".pdfium/lib"));
281    vec![
282        entry(
283            "layout",
284            model_path(
285                "DOCLING_LAYOUT_ONNX",
286                "models/layout_heron.onnx",
287                "models/layout_heron_int8.onnx",
288            ),
289        ),
290        entry("tableformer.encoder", enc),
291        entry("tableformer.decoder", dec),
292        entry("tableformer.bbox", bbx),
293        entry("ocr.rec", rec),
294        entry("ocr.dict", dict),
295        entry("pdfium", pdfium),
296    ]
297}
298
299/// Resolve a model path: an explicit env override always wins; otherwise the
300/// INT8 variant of the default path when it exists on disk (the quantized
301/// models are conformance-validated — see docs/PDF_CONFORMANCE.md — and load/run
302/// markedly faster on CPU), unless `DOCLING_RS_FP32` opts back into full
303/// precision; else the fp32 default.
304#[cfg(feature = "ml")]
305pub(crate) fn model_path(env: &str, fp32_default: &str, int8_default: &str) -> String {
306    if let Ok(p) = std::env::var(env) {
307        return p;
308    }
309    if !prefer_fp32() {
310        let p = resolve_asset(int8_default);
311        if std::path::Path::new(&p).exists() {
312            return p;
313        }
314    }
315    resolve_asset(fp32_default)
316}
317
318/// Decode a standalone image with hard resource limits. A crafted image can
319/// declare enormous dimensions in a few-KB file; `image::load_from_memory`
320/// then tries to allocate the full pixel buffer (e.g. 60000×60000 → ~10 GB),
321/// and allocation failure aborts the whole process, bypassing the per-request
322/// panic catch. The 256 MiB alloc / 30000-px caps below turn that into a
323/// recoverable decode error instead. `DOCLING_RS_MAX_IMAGE_PIXELS` overrides
324/// the per-side pixel cap for the rare legitimately-huge scan.
325///
326/// Gated on `ml`: the only callers (`convert_image`, the METS backend) are
327/// ML-only, and the `image` crate is an `ml`-feature dependency — the
328/// text-layer wasm build has neither.
329#[cfg(feature = "ml")]
330pub(crate) fn decode_image_limited(bytes: &[u8]) -> Result<image::RgbImage, PdfError> {
331    let max_side: u32 = std::env::var("DOCLING_RS_MAX_IMAGE_PIXELS")
332        .ok()
333        .and_then(|v| v.parse().ok())
334        .unwrap_or(30_000);
335    decode_image_with_max_side(bytes, max_side)
336}
337
338#[cfg(feature = "ml")]
339fn decode_image_with_max_side(bytes: &[u8], max_side: u32) -> Result<image::RgbImage, PdfError> {
340    use image::ImageReader;
341    use std::io::Cursor;
342
343    let mut limits = image::Limits::default();
344    limits.max_image_width = Some(max_side);
345    limits.max_image_height = Some(max_side);
346    limits.max_alloc = Some(256 * 1024 * 1024);
347
348    let mut reader = ImageReader::new(Cursor::new(bytes))
349        .with_guessed_format()
350        .map_err(|e| PdfError::Pdfium(format!("image: {e}")))?;
351    reader.limits(limits);
352    Ok(reader
353        .decode()
354        .map_err(|e| PdfError::Pdfium(format!("image: {e}")))?
355        .into_rgb8())
356}
357
358#[cfg(feature = "ml")]
359/// One page's assembled output: typed nodes plus the page's hyperlinks (kept
360/// separate so pages processed out of order can be stitched back in page
361/// order) and its confidence scores (#183).
362type PageOut = (
363    Vec<Node>,
364    Vec<(String, String)>,
365    docling_core::confidence::PageConfidence,
366);
367
368#[cfg(feature = "ml")]
369/// The pool-wide TableFormer slot: one instance shared by every worker, loaded
370/// lazily on the first table region any worker sees. Tables appear on a
371/// minority of pages, so per-worker copies mostly multiplied ~0.4 GB of
372/// weights+arenas by the pool size for nothing; a single shared instance keeps
373/// the peak flat regardless of pool width, and a table's structure prediction
374/// is independent of which worker runs it, so output is byte-identical. The
375/// mutex serialises concurrent tables — the shared instance is loaded with the
376/// full intra-op thread budget to compensate (one wide TableFormer instead of
377/// several narrow ones).
378enum TfSlot {
379    /// Not attempted yet (no table seen so far).
380    Unloaded,
381    /// Load attempted, graphs absent — geometric fallback (warned once).
382    Missing,
383    Ready(tableformer::TableFormer),
384}
385
386#[cfg(feature = "ml")]
387type SharedTables = Arc<Mutex<TfSlot>>;
388
389#[cfg(feature = "ml")]
390/// The same lazy shared-slot pattern for the (rarer still) enrichment models:
391/// one instance per pipeline, loaded on the first region that needs it.
392enum EnrichSlot<T> {
393    Unloaded,
394    /// Load attempted, model files absent — enrichment skipped (warned once).
395    Missing,
396    Ready(T),
397}
398
399#[cfg(feature = "ml")]
400type SharedClassifier = Arc<Mutex<EnrichSlot<enrich::PictureClassifier>>>;
401#[cfg(feature = "ml")]
402type SharedCodeFormula = Arc<Mutex<EnrichSlot<enrich::CodeFormula>>>;
403
404#[cfg(feature = "ml")]
405/// The opt-in enrichment passes, mirroring docling's `PdfPipelineOptions`
406/// flags (`do_picture_classification`, `do_code_enrichment`,
407/// `do_formula_enrichment`). All off by default.
408#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
409pub struct EnrichmentOptions {
410    /// Classify each picture with DocumentFigureClassifier (26 classes).
411    pub picture_classification: bool,
412    /// Rewrite code blocks (and detect their language) with CodeFormulaV2.
413    pub code: bool,
414    /// Decode display formulas to LaTeX with CodeFormulaV2.
415    pub formula: bool,
416}
417
418#[cfg(feature = "ml")]
419impl EnrichmentOptions {
420    fn any(&self) -> bool {
421        self.picture_classification || self.code || self.formula
422    }
423}
424
425#[cfg(feature = "ml")]
426/// A self-contained set of the per-page models (layout, OCR). Each parallel
427/// page-worker owns its own `Worker` so inference runs concurrently without
428/// sharing an ONNX session (`ort`'s `Session::run` is `&mut self`); only the
429/// rarely-hit TableFormer is shared (see [`TfSlot`]).
430struct Worker {
431    /// `None` when `no_ocr` skips layout entirely — no model load, no inference.
432    layout: Option<layout::LayoutModel>,
433    ocr: Option<ocr::OcrModel>,
434    /// Shared TableFormer slot; `None` when `no_table_former`/`no_ocr` skip it.
435    tables: Option<SharedTables>,
436    /// Shared enrichment slots; `None` unless the corresponding flag is on.
437    classifier: Option<SharedClassifier>,
438    code_formula: Option<SharedCodeFormula>,
439    enrich: EnrichmentOptions,
440    /// Skip layout, OCR, and TableFormer; reconstruct text purely from the PDF's
441    /// embedded text layer. See [`Pipeline::no_ocr`].
442    no_ocr: bool,
443    /// Discard the embedded text layer and OCR every page. See
444    /// [`Pipeline::force_full_page_ocr`].
445    force_full_page_ocr: bool,
446    /// Keep text-panel pictures as pictures instead of demoting them to
447    /// paragraphs. See [`Pipeline::no_text_panels`].
448    no_text_panels: bool,
449    /// Which recognition model [`Self::ocr`] loads. See [`Pipeline::ocr_lang`].
450    ocr_lang: ocr::OcrLang,
451}
452
453#[cfg(feature = "ml")]
454impl Worker {
455    #[allow(clippy::too_many_arguments)] // mirrors the Pipeline's option set
456    fn load(
457        intra: usize,
458        tables: Option<SharedTables>,
459        enrich_slots: (Option<SharedClassifier>, Option<SharedCodeFormula>),
460        enrich: EnrichmentOptions,
461        no_ocr: bool,
462        force_full_page_ocr: bool,
463        no_text_panels: bool,
464        ocr_lang: ocr::OcrLang,
465    ) -> Result<Self, PdfError> {
466        Ok(Self {
467            layout: if no_ocr {
468                None
469            } else {
470                Some(layout::LayoutModel::load_with(intra).map_err(PdfError::Layout)?)
471            },
472            ocr: None,
473            tables,
474            classifier: enrich_slots.0,
475            code_formula: enrich_slots.1,
476            enrich,
477            no_ocr,
478            force_full_page_ocr,
479            no_text_panels,
480            ocr_lang,
481        })
482    }
483
484    /// Run layout (+ OCR for cell-less pages) + TableFormer and assemble page `n`
485    /// into its nodes and links. Pure given the page (mutates only the worker's
486    /// lazily-loaded OCR model), so it is safe to run concurrently across pages.
487    fn process(&mut self, n: usize, page: &mut PdfPage) -> Result<PageOut, PdfError> {
488        if self.no_ocr {
489            // Fastest path: no layout/OCR/TableFormer inference at all. The PDF's
490            // embedded text cells (if any) become flat, line-grouped paragraphs in
491            // reading order via the same orphan-region machinery that normally
492            // rescues text the detector missed — here it rescues *all* of it.
493            // Pages with no embedded text layer (scanned/image-only) yield nothing;
494            // convert those without `no_ocr`.
495            let parse = quality::parse_score(&page.cells);
496            let mut regions = Vec::new();
497            assemble::add_orphan_regions(&mut regions, &page.cells);
498            let table_rows = vec![None; regions.len()];
499            let enrich_out = vec![None; regions.len()];
500            let conf = quality::page_confidence(parse, &regions, &[]);
501            let (nodes, links) = timing::timed("assemble_page", || {
502                assemble::assemble_page(page, regions, &table_rows, &enrich_out)
503            });
504            return Ok((nodes, links, conf));
505        }
506        let regions = timing::timed("layout.predict", || {
507            self.layout
508                .as_mut()
509                .expect("layout model loaded unless no_ocr")
510                .predict(&page.image, page.width, page.height)
511        })
512        .map_err(|e| PdfError::Layout(format!("page {}: {e}", n + 1)))?;
513        self.finish_page(n, page, regions)
514    }
515
516    /// Layout-detect a whole batch of pages with one inference call (issue #73),
517    /// then run each page's remaining stages (OCR / TableFormer / enrichment /
518    /// assembly) per page. Index-aligned with `items`; a layout failure fails
519    /// every page in the batch (they shared the one inference call).
520    fn process_batch(&mut self, items: &mut [(usize, PdfPage)]) -> Vec<Result<PageOut, PdfError>> {
521        if self.no_ocr {
522            // No layout model to batch — the text-layer-only path is per page.
523            return items
524                .iter_mut()
525                .map(|(n, page)| {
526                    let n = *n;
527                    self.process(n, page)
528                })
529                .collect();
530        }
531        let inputs: Vec<(&image::RgbImage, f32, f32)> = items
532            .iter()
533            .map(|(_, page)| (&page.image, page.width, page.height))
534            .collect();
535        let batched = timing::timed("layout.predict", || {
536            self.layout
537                .as_mut()
538                .expect("layout model loaded unless no_ocr")
539                .predict_batch(&inputs)
540        });
541        match batched {
542            Ok(all) => items
543                .iter_mut()
544                .zip(all)
545                .map(|((n, page), regions)| self.finish_page(*n, page, regions))
546                .collect(),
547            Err(e) => items
548                .iter()
549                .map(|(n, _)| Err(PdfError::Layout(format!("page {}: {e}", n + 1))))
550                .collect(),
551        }
552    }
553
554    /// Everything after layout detection: per-label confidence thresholds,
555    /// overlap resolution, orphan-text recovery, OCR for cell-less pages,
556    /// TableFormer, enrichment, and page assembly.
557    fn finish_page(
558        &mut self,
559        n: usize,
560        page: &mut PdfPage,
561        regions: Vec<layout::Region>,
562    ) -> Result<PageOut, PdfError> {
563        // Force-OCR is exactly "pretend the text layer is not there": clear
564        // every cell kind the extractors produced before anything reads them,
565        // and the ordinary no-text-layer machinery below — full-page OCR,
566        // OCR-fed TableFormer matching — takes over unchanged. (`no_ocr` wins
567        // when both are set, mirroring docling, where `force_full_page_ocr`
568        // is a sub-option of `do_ocr`; the no-ocr path never reaches here.)
569        // Done here rather than in `process` so the batched layout path
570        // (`process_batch` → `finish_page`) honors the flag too.
571        // Parse quality is scored on the extracted text layer before force-OCR
572        // discards it (docling's page-preprocessing stage runs before OCR too,
573        // so its parse_score also reflects the original text layer).
574        let parse = quality::parse_score(&page.cells);
575        // Recognition confidences of every OCR'd cell on this page → ocr_score.
576        let mut ocr_confs: Vec<f32> = Vec::new();
577        if self.force_full_page_ocr {
578            page.cells.clear();
579            page.code_cells.clear();
580            page.word_cells.clear();
581        }
582        // Quant-robustness guard: the default int8 layout graph keeps its
583        // confidences near the 0.5 label thresholds, and a different CPU's
584        // quantized kernels can flip a whole page's detections under them —
585        // tables and paragraphs then dissolve into orphan one-liners while the
586        // same build converts the page perfectly elsewhere. When a dense
587        // digital page ends up with detections covering almost none of its
588        // text cells, re-run that one page on the fp32 graph (lazy-loaded,
589        // auto-int8 selection only) and keep whichever detections cover more.
590        let mut regions = regions;
591        if !page.cells.is_empty() {
592            let thresholded = |rs: &[layout::Region]| -> Vec<layout::Region> {
593                rs.iter()
594                    .filter(|r| r.score >= layout::label_threshold(r.label))
595                    .cloned()
596                    .collect()
597            };
598            let text_cells = page
599                .cells
600                .iter()
601                .filter(|c| !c.text.trim().is_empty())
602                .count();
603            let cov = assemble::layout_cell_coverage(&thresholded(&regions), &page.cells);
604            if text_cells >= 15 && cov < 0.5 {
605                let retry = self
606                    .layout
607                    .as_mut()
608                    .expect("layout model loaded unless no_ocr")
609                    .predict_fp32_fallback(&page.image, page.width, page.height)
610                    .map_err(|e| PdfError::Layout(format!("page {}: {e}", n + 1)))?;
611                if let Some(retry) = retry {
612                    let cov2 = assemble::layout_cell_coverage(&thresholded(&retry), &page.cells);
613                    if cov2 > cov {
614                        eprintln!(
615                            "docling-pdf: page {}: int8 layout covered {:.0}% of the text \
616                             cells; the fp32 retry covers {:.0}% — using it",
617                            n + 1,
618                            cov * 100.0,
619                            cov2 * 100.0
620                        );
621                        regions = retry;
622                    }
623                }
624            }
625        }
626        // docling's LayoutPostprocessor drops each detection below its label's
627        // confidence threshold (stricter than the 0.3 base the predictor keeps),
628        // before any overlap resolution. This removes the low-confidence tables /
629        // pictures / list-items that otherwise double-emit or mis-classify.
630        if std::env::var("DOCLING_RS_DEBUG_REGIONS").is_ok() {
631            for r in &regions {
632                eprintln!(
633                    "DBG raw {} {:.2} [{:.0},{:.0},{:.0},{:.0}]",
634                    r.label, r.score, r.l, r.t, r.r, r.b
635                );
636            }
637        }
638        regions.retain(|r| r.score >= layout::label_threshold(r.label));
639        // Resolve overlapping detections once, before OCR.
640        let mut regions = assemble::resolve(regions);
641        // Emit text the detector missed as orphan text regions (docling parity).
642        assemble::add_orphan_regions(&mut regions, &page.cells);
643        // Drop phantom empty low-confidence picture boxes (docling parity).
644        assemble::drop_false_pictures(&mut regions, &page.cells, page.width, page.height);
645        // A regular region fully inside a surviving table/index/picture is that
646        // special's child (a cell / in-figure label), not a separate block —
647        // remove it so it isn't emitted twice (docling parity).
648        assemble::drop_contained_regulars(&mut regions);
649        // No text layer → recognise text from the page image via OCR.
650        let ocred = page.cells.is_empty();
651        if ocred {
652            if self.ocr.is_none() {
653                self.ocr = Some(ocr::OcrModel::load(self.ocr_lang).map_err(PdfError::Ocr)?);
654            }
655            let cells = timing::timed("ocr.page", || {
656                self.ocr
657                    .as_mut()
658                    .unwrap()
659                    .ocr_page(&page.image, &regions, page.scale)
660            })
661            .map_err(|e| PdfError::Ocr(format!("page {}: {e}", n + 1)))?;
662            ocr_confs.extend(cells.iter().map(|(_, conf)| conf));
663            page.cells = cells.into_iter().map(|(cell, _)| cell).collect();
664            // Table interiors carry no words yet: region-scoped OCR skips
665            // table labels, and a scanned page has no pdfium text layer — so
666            // TableFormer's cell matcher got an empty word list and the table
667            // dissolved (#173). Recognize the table regions' word crops
668            // (mirroring the browser scanned path): `word_cells` feeds the
669            // matcher, and the same cells join `cells` so the geometric
670            // fallback and the table's region text see them too.
671            if regions.iter().any(|r| assemble::is_table_like(r.label)) {
672                let words = timing::timed("ocr.table_words", || {
673                    self.ocr
674                        .as_mut()
675                        .unwrap()
676                        .ocr_table_words(&page.image, &regions, page.scale)
677                })
678                .map_err(|e| PdfError::Ocr(format!("page {}: {e}", n + 1)))?;
679                ocr_confs.extend(words.iter().map(|(_, conf)| conf));
680                let words: Vec<_> = words.into_iter().map(|(cell, _)| cell).collect();
681                page.cells.extend(words.iter().cloned());
682                page.word_cells = words;
683            }
684        }
685        // Region-scoped OCR skips `picture` interiors, and a digital page's
686        // text layer cannot see into an embedded raster either — so a figure
687        // that is really a text box (terms-and-conditions exported as an
688        // image) lost its words on every page kind. Python docling OCRs the
689        // bitmap-covered areas of *every* page — even digital ones — once they
690        // exceed `bitmap_area_threshold` (5 % of the page); the browser paths
691        // already do. Recognize the big text-less crops here too; the panel
692        // demotion / orphan recovery below place the lines.
693        let mut pic_cells: Vec<pdfium_backend::TextCell> = Vec::new();
694        {
695            let page_area = (page.width * page.height).max(1.0);
696            let has_text = |r: &layout::Region| {
697                page.cells.iter().any(|c| {
698                    let ca = ((c.r - c.l) * (c.b - c.t)).max(1.0);
699                    let ix = (r.r.min(c.r) - r.l.max(c.l)).max(0.0);
700                    let iy = (r.b.min(c.b) - r.t.max(c.t)).max(0.0);
701                    !c.text.trim().is_empty() && ix * iy / ca > 0.5
702                })
703            };
704            // A captioned picture can never demote to a text panel (see
705            // recover_text_panels), and on digital pages its speculative OCR
706            // would be discarded anyway — don't pay for it.
707            let captioned = |r: &layout::Region| {
708                regions.iter().any(|c| {
709                    c.label == "caption"
710                        && c.r.min(r.r) - c.l.max(r.l) > 0.0
711                        && ((c.t >= r.b && c.t - r.b <= 25.0) || (r.t >= c.b && r.t - c.b <= 25.0))
712                })
713            };
714            let bare: Vec<layout::Region> = regions
715                .iter()
716                .filter(|r| {
717                    r.label == "picture"
718                        && (r.r - r.l) * (r.b - r.t) / page_area >= 0.05
719                        && !has_text(r)
720                        && (ocred || !captioned(r))
721                })
722                .map(|r| layout::Region {
723                    label: "text",
724                    ..r.clone()
725                })
726                .collect();
727            if !bare.is_empty() {
728                if self.ocr.is_none() {
729                    self.ocr = Some(ocr::OcrModel::load(self.ocr_lang).map_err(PdfError::Ocr)?);
730                }
731                let scored = timing::timed("ocr.pictures", || {
732                    self.ocr
733                        .as_mut()
734                        .unwrap()
735                        .ocr_page(&page.image, &bare, page.scale)
736                })
737                .map_err(|e| PdfError::Ocr(format!("page {}: {e}", n + 1)))?;
738                // Speculative in-picture OCR counts toward ocr_score only on
739                // OCR'd pages, where the recognized lines actually join the
740                // output; on a digital page they may be discarded below.
741                if ocred {
742                    ocr_confs.extend(scored.iter().map(|(_, conf)| conf));
743                }
744                pic_cells = scored.into_iter().map(|(cell, _)| cell).collect();
745                page.cells.extend(pic_cells.iter().cloned());
746            }
747        }
748        let cells_before_pic_ocr = page.cells.len() - pic_cells.len();
749        // A "picture" that is really a colored text panel — dense, wide,
750        // multi-line — reads out as paragraphs instead of shipping as pixels;
751        // sparse in-picture text (a chart's labels) keeps the crop and stays
752        // inside it as the picture's silent children (docling parity, #200).
753        // `no_text_panels` (#173) opts out entirely for image-extraction
754        // workflows.
755        if !self.no_text_panels {
756            assemble::recover_text_panels(&mut regions, &page.cells);
757        }
758        // On an OCR'd page, in-picture text that did NOT demote its picture
759        // mostly stays silent, exactly as in docling: its postprocess step
760        // "Remove regular clusters that are included in wrappers" walks
761        // SPECIAL_TYPES — which includes PICTURE — so an orphan text cluster
762        // >80 % contained in a kept picture becomes that picture's child and
763        // never reaches the serializer. Only border-straddlers (≤80 %
764        // containment) survive as text. Emitting *everything* here used to
765        // splice a chart's OCR'd axis ticks into the body text right next to
766        // the image chunk (#200) — so the orphan pass places the recognized
767        // lines, then the same containment drop that handled the first wave
768        // re-runs to swallow the in-picture ones.
769        if ocred && !pic_cells.is_empty() {
770            // Pictures (and wrappers) no longer count as claimers (#165), so
771            // the plain orphan pass places the recognized lines directly.
772            assemble::add_orphan_regions(&mut regions, &pic_cells);
773            assemble::drop_contained_regulars(&mut regions);
774        } else if !ocred && !pic_cells.is_empty() {
775            // Digital page, picture kept: its speculative OCR cells must not
776            // linger in the text-cell set (they were appended at the tail).
777            let kept: Vec<layout::Region> = regions
778                .iter()
779                .filter(|r| r.label == "picture")
780                .cloned()
781                .collect();
782            let tail = page.cells.split_off(cells_before_pic_ocr);
783            page.cells.extend(tail.into_iter().filter(|c| {
784                !kept.iter().any(|r| {
785                    let ca = ((c.r - c.l) * (c.b - c.t)).max(1.0);
786                    let ix = (r.r.min(c.r) - r.l.max(c.l)).max(0.0);
787                    let iy = (r.b.min(c.b) - r.t.max(c.t)).max(0.0);
788                    ix * iy / ca > 0.5
789                })
790            }));
791        }
792        // TableFormer structure per table region (else geometric fallback). The
793        // shared slot is only locked (and lazily loaded) when the page actually
794        // has a table, so table-free documents never pay for TableFormer at all.
795        let mut table_rows: Vec<Option<Vec<Vec<String>>>> = vec![None; regions.len()];
796        if let Some(slot) = self.tables.as_ref() {
797            if regions.iter().any(|r| assemble::is_table_like(r.label)) {
798                timing::timed("tableformer", || {
799                    let mut guard = slot.lock().unwrap();
800                    if matches!(*guard, TfSlot::Unloaded) {
801                        // Full intra-op width: tables serialise on this mutex, so
802                        // the one instance gets the whole thread budget.
803                        *guard = match tableformer::TableFormer::load_with(intra_threads()) {
804                            Some(tf) => TfSlot::Ready(tf),
805                            None => TfSlot::Missing,
806                        };
807                    }
808                    if let TfSlot::Ready(tf) = &mut *guard {
809                        for (i, r) in regions.iter().enumerate() {
810                            if assemble::is_table_like(r.label) {
811                                table_rows[i] = tf.predict_table_rows(
812                                    &page.image,
813                                    [r.l, r.t, r.r, r.b],
814                                    &page.word_cells,
815                                );
816                            }
817                        }
818                    }
819                });
820            }
821        }
822        if std::env::var("DOCLING_RS_DEBUG_REGIONS").is_ok() {
823            for (i, r) in regions.iter().enumerate() {
824                eprintln!(
825                    "DBG final {} {:.2} [{:.0},{:.0},{:.0},{:.0}] rows={:?}",
826                    r.label,
827                    r.score,
828                    r.l,
829                    r.t,
830                    r.r,
831                    r.b,
832                    table_rows[i]
833                        .as_ref()
834                        .map(|t| (t.len(), t.first().map(|r| r.len())))
835                );
836            }
837            eprintln!(
838                "DBG cells={} words={}",
839                page.cells.len(),
840                page.word_cells.len()
841            );
842        }
843        // Enrichment passes (opt-in): DocumentPictureClassifier over picture
844        // regions, CodeFormulaV2 over code/formula regions. Same shared-slot
845        // shape as TableFormer — one lazily-loaded instance per pipeline, only
846        // ever locked when a page actually has a matching region.
847        let mut enrich_out: Vec<Option<assemble::Enrichment>> = vec![None; regions.len()];
848        if let Some(slot) = self.classifier.as_ref() {
849            if regions.iter().any(|r| r.label == "picture") {
850                timing::timed("picture_classifier", || {
851                    let mut guard = slot.lock().unwrap();
852                    if matches!(*guard, EnrichSlot::Unloaded) {
853                        *guard = match enrich::PictureClassifier::load_with(intra_threads()) {
854                            Some(m) => EnrichSlot::Ready(m),
855                            None => EnrichSlot::Missing,
856                        };
857                    }
858                    if let EnrichSlot::Ready(model) = &mut *guard {
859                        for (i, r) in regions.iter().enumerate() {
860                            if r.label != "picture" {
861                                continue;
862                            }
863                            let Some(crop) = assemble::crop_region_scaled(
864                                page,
865                                [r.l, r.t, r.r, r.b],
866                                enrich::CLASSIFIER_SCALE,
867                            ) else {
868                                continue;
869                            };
870                            match model.classify(&crop) {
871                                Ok(classes) => {
872                                    enrich_out[i] =
873                                        Some(assemble::Enrichment::PictureClasses(classes));
874                                }
875                                Err(e) => eprintln!("docling-pdf: page {}: {e}", n + 1),
876                            }
877                        }
878                    }
879                });
880            }
881        }
882        if let Some(slot) = self.code_formula.as_ref() {
883            let wants = |label: &str| {
884                (label == "code" && self.enrich.code) || (label == "formula" && self.enrich.formula)
885            };
886            if regions.iter().any(|r| wants(r.label)) {
887                timing::timed("code_formula", || {
888                    let mut guard = slot.lock().unwrap();
889                    if matches!(*guard, EnrichSlot::Unloaded) {
890                        *guard = match enrich::CodeFormula::load_with(intra_threads()) {
891                            Some(m) => EnrichSlot::Ready(m),
892                            None => EnrichSlot::Missing,
893                        };
894                    }
895                    if let EnrichSlot::Ready(model) = &mut *guard {
896                        for (i, r) in regions.iter().enumerate() {
897                            if !wants(r.label) {
898                                continue;
899                            }
900                            // docling crops the postprocessed cluster box — the
901                            // union of the region's text cells, not the raw
902                            // detector box — expanded by 18% per side, at
903                            // ~120 dpi.
904                            let [bl, bt, br, bb] = assemble::region_cell_bbox(r, &page.cells)
905                                .unwrap_or([r.l, r.t, r.r, r.b]);
906                            let (w, h) = (br - bl, bb - bt);
907                            let ex = enrich::CODE_FORMULA_EXPANSION;
908                            let bbox = [bl - w * ex, bt - h * ex, br + w * ex, bb + h * ex];
909                            let Some(crop) = assemble::crop_region_scaled(
910                                page,
911                                bbox,
912                                enrich::CODE_FORMULA_SCALE,
913                            ) else {
914                                continue;
915                            };
916                            let kind = if r.label == "code" {
917                                enrich::CodeFormulaKind::Code
918                            } else {
919                                enrich::CodeFormulaKind::Formula
920                            };
921                            match model.predict(&crop, kind) {
922                                Ok(text) => {
923                                    enrich_out[i] = Some(match kind {
924                                        enrich::CodeFormulaKind::Code => {
925                                            let (code, language) =
926                                                enrich::extract_code_language(&text);
927                                            assemble::Enrichment::Code {
928                                                language,
929                                                text: code,
930                                            }
931                                        }
932                                        enrich::CodeFormulaKind::Formula => {
933                                            assemble::Enrichment::Formula { latex: text }
934                                        }
935                                    });
936                                }
937                                Err(e) => eprintln!("docling-pdf: page {}: {e}", n + 1),
938                            }
939                        }
940                    }
941                });
942            }
943        }
944        // Score the final region set (docling assigns layout_score over the
945        // postprocessed clusters — the same set assemble_page consumes).
946        let conf = quality::page_confidence(parse, &regions, &ocr_confs);
947        let (nodes, links) = timing::timed("assemble_page", || {
948            assemble::assemble_page(page, regions, &table_rows, &enrich_out)
949        });
950        Ok((nodes, links, conf))
951    }
952}
953
954#[cfg(feature = "ml")]
955/// Per-worker ONNX intra-op threads. The layout model is memory-bandwidth bound,
956/// so on a typical machine two threads per worker (sharing one in-cache copy of
957/// the weights) extracts more throughput than one fat model or many single-thread
958/// workers. `DOCLING_RS_PDF_INTRA` overrides for per-machine tuning.
959fn pdf_intra() -> usize {
960    if let Some(n) = std::env::var("DOCLING_RS_PDF_INTRA")
961        .ok()
962        .and_then(|v| v.parse::<usize>().ok())
963        .filter(|&n| n > 0)
964    {
965        return n;
966    }
967    if intra_threads() >= 2 {
968        2
969    } else {
970        1
971    }
972}
973
974#[cfg(feature = "ml")]
975/// How many page-workers to spin up for a multi-page PDF. `DOCLING_RS_PDF_WORKERS`
976/// overrides; otherwise size the pool so `workers × intra ≈ cores`, capped at 4 so
977/// a worst-case pool holds a bounded amount of model memory (~0.4 GB per worker)
978/// and does not oversaturate the memory bus with model-weight traffic.
979fn pdf_worker_count() -> usize {
980    if let Some(n) = std::env::var("DOCLING_RS_PDF_WORKERS")
981        .ok()
982        .and_then(|v| v.parse::<usize>().ok())
983        .filter(|&n| n > 0)
984    {
985        return n;
986    }
987    (intra_threads() / pdf_intra()).clamp(1, 4)
988}
989
990#[cfg(feature = "ml")]
991/// Max pages a worker layout-detects with one batched inference call (issue
992/// #73). Workers drain the work channel opportunistically up to this size —
993/// whatever is already rendered gets batched, so batching never *waits* for
994/// pages and adds no latency when rendering is the bottleneck.
995///
996/// Default: 4 on 8+ cores, 1 (per-page) below. Measured on a 4-core box the
997/// batch only adds cache pressure and costs pipeline overlap (2 workers × 2
998/// threads: 8.1 s/conv at batch=1 vs 9.3 s at batch=4 on the 9-page
999/// 2206.01062 fixture); the single-session amortization it buys needs the
1000/// wider thread budget of a many-core machine. Output is bit-identical at
1001/// every batch size, so this is purely a throughput knob.
1002/// `DOCLING_RS_PDF_LAYOUT_BATCH` overrides; `1` restores per-page inference.
1003fn pdf_layout_batch() -> usize {
1004    std::env::var("DOCLING_RS_PDF_LAYOUT_BATCH")
1005        .ok()
1006        .and_then(|v| v.parse::<usize>().ok())
1007        .filter(|&n| n > 0)
1008        .unwrap_or_else(|| if intra_threads() >= 8 { 4 } else { 1 })
1009}
1010
1011#[cfg(feature = "ml")]
1012/// Minimum page count before a PDF is worth the parallel worker pool. Below this,
1013/// the serial primary (running its model on every core) is faster than fanning out
1014/// — the helper pool's one-time model-load cost only pays off once enough pages
1015/// share it. `DOCLING_RS_PDF_PARALLEL_MIN` overrides.
1016fn pdf_parallel_min() -> usize {
1017    std::env::var("DOCLING_RS_PDF_PARALLEL_MIN")
1018        .ok()
1019        .and_then(|v| v.parse::<usize>().ok())
1020        .filter(|&n| n > 0)
1021        .unwrap_or(6)
1022}
1023
1024#[cfg(feature = "ml")]
1025/// A reusable PDF pipeline. The **primary** worker runs its models on every core,
1026/// so a single-page / small / image / METS input is converted at full intra-op
1027/// speed with no pool to load. A document with enough pages instead fans out
1028/// across a **pool** of narrower workers processed concurrently. Both load lazily
1029/// and are cached for reuse, so a one-shot conversion only pays for what it uses.
1030pub struct Pipeline {
1031    /// Full-intra worker for the serial path; loaded on first serial use.
1032    primary: Option<Worker>,
1033    /// Narrower workers (≈cores/`target_workers` threads each) for the parallel
1034    /// path; loaded on first multi-page use and cached.
1035    pool: Vec<Worker>,
1036    /// The single TableFormer instance every worker shares (see [`TfSlot`]).
1037    tables: SharedTables,
1038    /// The shared enrichment-model slots (same pattern as [`TfSlot`]).
1039    classifier: SharedClassifier,
1040    code_formula: SharedCodeFormula,
1041    /// Desired pool size for multi-page documents.
1042    target_workers: usize,
1043    /// Page count at/above which the parallel pool is worth its load cost.
1044    parallel_min: usize,
1045    /// Skip loading/running TableFormer; table regions fall back to geometric
1046    /// reconstruction. See [`Pipeline::no_table_former`].
1047    no_table_former: bool,
1048    /// Skip layout, OCR, and TableFormer entirely. See [`Pipeline::no_ocr`].
1049    no_ocr: bool,
1050    /// OCR every page even when it carries a text layer. See
1051    /// [`Pipeline::force_full_page_ocr`].
1052    force_full_page_ocr: bool,
1053    /// Never demote text-panel pictures. See [`Pipeline::no_text_panels`].
1054    no_text_panels: bool,
1055    /// Opt-in enrichment passes. See [`Pipeline::enrichments`].
1056    enrich: EnrichmentOptions,
1057    /// 1-based inclusive page window to convert. See [`Pipeline::pages`].
1058    page_range: Option<(usize, usize)>,
1059    /// OCR recognition language. See [`Pipeline::ocr_lang`].
1060    ocr_lang: ocr::OcrLang,
1061}
1062
1063#[cfg(feature = "ml")]
1064impl Pipeline {
1065    /// Construct the pipeline. Models load lazily on first use (full-intra primary
1066    /// for serial inputs, the helper pool for multi-page PDFs), so nothing is
1067    /// loaded that a given document doesn't need.
1068    pub fn new() -> Result<Self, PdfError> {
1069        Ok(Self {
1070            primary: None,
1071            pool: Vec::new(),
1072            tables: Arc::new(Mutex::new(TfSlot::Unloaded)),
1073            classifier: Arc::new(Mutex::new(EnrichSlot::Unloaded)),
1074            code_formula: Arc::new(Mutex::new(EnrichSlot::Unloaded)),
1075            target_workers: pdf_worker_count(),
1076            parallel_min: pdf_parallel_min(),
1077            no_table_former: false,
1078            no_ocr: false,
1079            force_full_page_ocr: false,
1080            no_text_panels: false,
1081            enrich: EnrichmentOptions::default(),
1082            page_range: None,
1083            ocr_lang: ocr::OcrLang::from_env(),
1084        })
1085    }
1086
1087    /// Convert only pages `first..=last` (**1-based**, like the page numbers a
1088    /// PDF viewer shows — issue #80's `--pages A-B`). Out-of-range pages are
1089    /// skipped before rasterization, so the cost is proportional to the window,
1090    /// not the document. `last` past the end of the document clamps; a window
1091    /// that selects no pages at all (`first` beyond the last page) is an error
1092    /// at convert time. `None` (the default) converts everything.
1093    pub fn pages(mut self, range: Option<(usize, usize)>) -> Self {
1094        self.page_range = range;
1095        self
1096    }
1097
1098    /// In-place variant of [`pages`](Self::pages) for a long-lived pipeline
1099    /// (e.g. docling-serve's warm instance) that applies a per-request window
1100    /// without rebuilding — unlike the model switches, the window is pure
1101    /// configuration. Set it before every conversion; it stays until changed.
1102    pub fn set_pages(&mut self, range: Option<(usize, usize)>) {
1103        self.page_range = range;
1104    }
1105
1106    /// OCR recognition language (see [`OcrLang`]): English by default, `ch`
1107    /// for the multilingual docling-conformance model. `None` keeps the
1108    /// process default (`DOCLING_RS_OCR_LANG`, else English). Set before the
1109    /// first conversion; for a warm pipeline use
1110    /// [`set_ocr_lang`](Self::set_ocr_lang).
1111    pub fn ocr_lang(mut self, lang: Option<ocr::OcrLang>) -> Self {
1112        self.set_ocr_lang(lang);
1113        self
1114    }
1115
1116    /// In-place variant of [`ocr_lang`](Self::ocr_lang) for a long-lived
1117    /// pipeline (docling-serve's warm instance). Unlike the page window this
1118    /// is a *model* switch: any worker whose cached recognition model was
1119    /// loaded for a different language drops it, to be lazily reloaded on the
1120    /// next OCR-needing page (cheap — the rec models are ~10 MB).
1121    pub fn set_ocr_lang(&mut self, lang: Option<ocr::OcrLang>) {
1122        let lang = lang.unwrap_or_else(ocr::OcrLang::from_env);
1123        self.ocr_lang = lang;
1124        for worker in self.primary.iter_mut().chain(self.pool.iter_mut()) {
1125            if worker.ocr_lang != lang {
1126                worker.ocr_lang = lang;
1127                worker.ocr = None;
1128            }
1129        }
1130    }
1131
1132    /// Resolve the configured 1-based window against a page count into the
1133    /// 0-based inclusive form the backend walks, validating it selects at
1134    /// least one existing page.
1135    fn resolve_range(&self, total: usize) -> Result<Option<(usize, usize)>, PdfError> {
1136        let Some((first, last)) = self.page_range else {
1137            return Ok(None);
1138        };
1139        if first == 0 || last < first {
1140            return Err(PdfError::Pdfium(format!(
1141                "invalid page range {first}-{last} (pages are 1-based, first <= last)"
1142            )));
1143        }
1144        if first > total {
1145            return Err(PdfError::Pdfium(format!(
1146                "page range {first}-{last} is outside the document ({total} page(s))"
1147            )));
1148        }
1149        Ok(Some((first - 1, last.min(total) - 1)))
1150    }
1151
1152    /// Enable the opt-in enrichment passes (docling's
1153    /// `do_picture_classification` / `do_code_enrichment` /
1154    /// `do_formula_enrichment`). Each enabled pass lazily loads its model on
1155    /// the first matching region; a missing model warns once and is skipped.
1156    /// Set before the first conversion (no effect on already-loaded workers).
1157    pub fn enrichments(mut self, opts: EnrichmentOptions) -> Self {
1158        self.enrich = opts;
1159        self
1160    }
1161
1162    /// Skip loading and running the TableFormer table-structure model. Table
1163    /// regions still get emitted, but reconstructed geometrically from cell
1164    /// positions instead of via the ONNX model's predicted structure — faster
1165    /// (no model load, no per-table inference) at the cost of table fidelity.
1166    /// No effect if a worker is already loaded; set this before the first
1167    /// conversion.
1168    pub fn no_table_former(mut self, disable: bool) -> Self {
1169        self.no_table_former = disable;
1170        self
1171    }
1172
1173    /// Keep every detected `picture` region as a picture. By default an
1174    /// *uncaptioned* picture that reads like a dense, uniform text panel (a
1175    /// terms-and-conditions box exported as an image) is demoted into
1176    /// paragraphs (#157); a chart the layout mislabels can still trip that
1177    /// heuristic on scanned pages, and image-extraction workflows may simply
1178    /// want every crop — this flag disables the demotion entirely (#173).
1179    /// No effect on already-loaded workers; set before the first conversion.
1180    pub fn no_text_panels(mut self, disable: bool) -> Self {
1181        self.no_text_panels = disable;
1182        self
1183    }
1184
1185    /// Skip layout detection, OCR, and TableFormer entirely — no model load, no
1186    /// inference of any kind. The PDF's embedded text cells are grouped by line
1187    /// and emitted as plain paragraphs in reading order: no headings, lists,
1188    /// tables, code blocks, or pictures, since that structure comes from the
1189    /// layout model. The fastest possible PDF path, but pages with no embedded
1190    /// text layer (scanned/image-only PDFs) yield no text at all — convert those
1191    /// without this flag. Implies `no_table_former`. No effect if a worker is
1192    /// already loaded; set this before the first conversion.
1193    pub fn no_ocr(mut self, disable: bool) -> Self {
1194        self.no_ocr = disable;
1195        self
1196    }
1197
1198    /// OCR every page from its rendered image even when the page carries an
1199    /// embedded text layer — docling's `force_full_page_ocr`. The escape hatch
1200    /// for text layers that exist but lie: broken encodings, subset fonts with
1201    /// garbage mappings, a scanned form with a few typed-in fields. Ignored
1202    /// when [`no_ocr`](Self::no_ocr) is set, mirroring docling (there
1203    /// `force_full_page_ocr` is a sub-option of `do_ocr`).
1204    pub fn force_full_page_ocr(mut self, force: bool) -> Self {
1205        self.force_full_page_ocr = force;
1206        self
1207    }
1208
1209    /// The shared TableFormer slot handed to each worker, or `None` when the
1210    /// pipeline options skip TableFormer entirely.
1211    fn tables_slot(&self) -> Option<SharedTables> {
1212        if self.no_table_former || self.no_ocr {
1213            None
1214        } else {
1215            Some(Arc::clone(&self.tables))
1216        }
1217    }
1218
1219    /// The shared enrichment slots for a worker (`None` per model unless its
1220    /// flag is on; `no_ocr` skips layout, so there are no regions to enrich).
1221    fn enrich_slots(&self) -> (Option<SharedClassifier>, Option<SharedCodeFormula>) {
1222        if self.no_ocr || !self.enrich.any() {
1223            return (None, None);
1224        }
1225        (
1226            self.enrich
1227                .picture_classification
1228                .then(|| Arc::clone(&self.classifier)),
1229            (self.enrich.code || self.enrich.formula).then(|| Arc::clone(&self.code_formula)),
1230        )
1231    }
1232
1233    /// Eagerly load the models (the full-intra serial worker: layout + OCR, and
1234    /// the shared TableFormer unless disabled) so the first conversion doesn't pay
1235    /// the load cost. Idempotent; respects `no_ocr` / `no_table_former` (with
1236    /// `no_ocr` there is nothing to load). The docling.rs analogue of docling's
1237    /// `DocumentConverter.initialize_pipeline`.
1238    pub fn warm_up(&mut self) -> Result<(), PdfError> {
1239        self.primary()?;
1240        Ok(())
1241    }
1242
1243    /// The full-intra serial worker, loaded on first use.
1244    fn primary(&mut self) -> Result<&mut Worker, PdfError> {
1245        if self.primary.is_none() {
1246            self.primary = Some(Worker::load(
1247                intra_threads(),
1248                self.tables_slot(),
1249                self.enrich_slots(),
1250                self.enrich,
1251                self.no_ocr,
1252                self.force_full_page_ocr,
1253                self.no_text_panels,
1254                self.ocr_lang,
1255            )?);
1256        }
1257        Ok(self.primary.as_mut().unwrap())
1258    }
1259
1260    /// Convert a PDF (bytes) to a [`DoclingDocument`]. A document with fewer than
1261    /// `parallel_min` pages (or a pool size of 1) streams through the full-intra
1262    /// primary; a larger one renders on this thread (pdfium is not thread-safe) and
1263    /// fans the pages out across the worker pool, reassembled in page order so the
1264    /// output is byte-identical to the serial path.
1265    pub fn convert(
1266        &mut self,
1267        bytes: &[u8],
1268        password: Option<&str>,
1269        name: &str,
1270    ) -> Result<DoclingDocument, PdfError> {
1271        let pages = pdfium_backend::page_count(bytes, password)?;
1272        let range = self.resolve_range(pages)?;
1273        // Serial vs parallel is decided by the pages actually converted: a
1274        // 3-page window over a 500-page PDF should not pay the pool load.
1275        let selected = range.map_or(pages, |(a, b)| b - a + 1);
1276        let doc = if self.target_workers >= 2 && selected >= self.parallel_min {
1277            self.convert_parallel(bytes, password, name, range)?
1278        } else {
1279            self.convert_serial(bytes, password, name, range)?
1280        };
1281        timing::report();
1282        Ok(doc)
1283    }
1284
1285    /// Stream pages one at a time through the primary worker — render → process →
1286    /// drop — so the document holds ~one page bitmap (~5 MB) at a time.
1287    fn convert_serial(
1288        &mut self,
1289        bytes: &[u8],
1290        password: Option<&str>,
1291        name: &str,
1292        range: Option<(usize, usize)>,
1293    ) -> Result<DoclingDocument, PdfError> {
1294        let mut doc = DoclingDocument::new(name);
1295        let mut confs = std::collections::BTreeMap::new();
1296        let render_image = !self.no_ocr;
1297        let worker = self.primary()?;
1298        pdfium_backend::for_each_page(
1299            bytes,
1300            password,
1301            render_image,
1302            range,
1303            |n, _total, mut page| {
1304                let (mut nodes, links, conf) = worker.process(n, &mut page)?;
1305                assemble::stamp_page_no(&mut nodes, n + 1);
1306                doc.nodes.extend(nodes);
1307                doc.links.extend(links);
1308                confs.insert(n + 1, conf);
1309                Ok::<(), PdfError>(())
1310            },
1311        )?;
1312        assemble::merge_continuations(&mut doc.nodes);
1313        doc.confidence = Some(docling_core::ConfidenceReport::from_pages(confs));
1314        Ok(doc)
1315    }
1316
1317    /// Render pages serially on this thread (pdfium) and process them in parallel
1318    /// across the worker pool. A bounded channel applies backpressure so only a
1319    /// handful of page bitmaps are resident at once; results carry their page
1320    /// index and are reassembled in order, so the output is byte-identical to the
1321    /// serial path.
1322    fn convert_parallel(
1323        &mut self,
1324        bytes: &[u8],
1325        password: Option<&str>,
1326        name: &str,
1327        range: Option<(usize, usize)>,
1328    ) -> Result<DoclingDocument, PdfError> {
1329        self.ensure_pool()?;
1330        let n_workers = self.pool.len();
1331        let render_image = !self.no_ocr;
1332        let layout_batch = pdf_layout_batch();
1333        // Bound sized so every worker can accumulate a full layout batch while
1334        // rendering stays ahead (and never below the pre-#73 render-ahead of
1335        // two pages per worker); still a hard cap on resident page bitmaps.
1336        let (work_tx, work_rx) = sync_channel::<(usize, PdfPage)>(n_workers * layout_batch.max(2));
1337        let work_rx: Arc<Mutex<Receiver<(usize, PdfPage)>>> = Arc::new(Mutex::new(work_rx));
1338        let results: Arc<Mutex<Vec<(usize, PageOut)>>> = Arc::new(Mutex::new(Vec::new()));
1339        let first_err: Arc<Mutex<Option<PdfError>>> = Arc::new(Mutex::new(None));
1340
1341        // Move the pool into the scope so each worker gets an exclusive `&mut`.
1342        let mut workers = std::mem::take(&mut self.pool);
1343        std::thread::scope(|s| {
1344            for worker in workers.iter_mut() {
1345                let work_rx = Arc::clone(&work_rx);
1346                let results = Arc::clone(&results);
1347                let first_err = Arc::clone(&first_err);
1348                s.spawn(move || loop {
1349                    // Hold the receiver lock only for the recv (plus a non-blocking
1350                    // drain up to the layout batch size); release before the (long)
1351                    // per-page work so other workers can pull concurrently.
1352                    let mut batch = Vec::new();
1353                    {
1354                        let rx = work_rx.lock().unwrap();
1355                        match rx.recv() {
1356                            Ok(item) => {
1357                                batch.push(item);
1358                                while batch.len() < layout_batch {
1359                                    match rx.try_recv() {
1360                                        Ok(item) => batch.push(item),
1361                                        Err(_) => break,
1362                                    }
1363                                }
1364                            }
1365                            Err(_) => break,
1366                        }
1367                    }
1368                    let outs = worker.process_batch(&mut batch);
1369                    for ((idx, _), out) in batch.iter().zip(outs) {
1370                        match out {
1371                            Ok(out) => results.lock().unwrap().push((*idx, out)),
1372                            Err(e) => {
1373                                let mut slot = first_err.lock().unwrap();
1374                                if slot.is_none() {
1375                                    *slot = Some(e);
1376                                }
1377                            }
1378                        }
1379                    }
1380                });
1381            }
1382            // Render on this thread and feed the workers; backpressure blocks here
1383            // when the channel is full. Dropping `work_tx` afterwards signals the
1384            // workers (recv → Err) to finish.
1385            let render = pdfium_backend::for_each_page(
1386                bytes,
1387                password,
1388                render_image,
1389                range,
1390                |i, _total, page| {
1391                    work_tx
1392                        .send((i, page))
1393                        .map_err(|_| PdfError::Pdfium("page-worker channel closed".into()))
1394                },
1395            );
1396            drop(work_tx);
1397            if let Err(e) = render {
1398                let mut slot = first_err.lock().unwrap();
1399                if slot.is_none() {
1400                    *slot = Some(e);
1401                }
1402            }
1403        });
1404        // Threads have joined; restore the pool for the next conversion.
1405        self.pool = workers;
1406
1407        if let Some(e) = first_err.lock().unwrap().take() {
1408            return Err(e);
1409        }
1410        let mut results = Arc::try_unwrap(results)
1411            .unwrap_or_else(|arc| Mutex::new(arc.lock().unwrap().clone()))
1412            .into_inner()
1413            .unwrap();
1414        results.sort_by_key(|(idx, _)| *idx);
1415        let mut doc = DoclingDocument::new(name);
1416        let mut confs = std::collections::BTreeMap::new();
1417        for (idx, (mut nodes, links, conf)) in results {
1418            assemble::stamp_page_no(&mut nodes, idx + 1);
1419            doc.nodes.extend(nodes);
1420            doc.links.extend(links);
1421            confs.insert(idx + 1, conf);
1422        }
1423        assemble::merge_continuations(&mut doc.nodes);
1424        doc.confidence = Some(docling_core::ConfidenceReport::from_pages(confs));
1425        Ok(doc)
1426    }
1427
1428    /// Convert a PDF in **streaming** mode: `emit` is called with each finalized,
1429    /// in-document-order batch of nodes (and that span's recovered links) as pages
1430    /// complete, so a caller can serialize Markdown page by page instead of waiting
1431    /// for the whole document. The batches are exactly the buffered [`convert`]'s
1432    /// nodes, split at safe block boundaries by [`assemble::StreamAssembler`] — the
1433    /// parallel path reorders pages back into document order before emitting, so
1434    /// the output is identical regardless of worker scheduling.
1435    ///
1436    /// `emit` runs on the calling thread (never a worker), so it needn't be `Send`
1437    /// and its backpressure throttles the whole pipeline. Returning `Err` from
1438    /// `emit` aborts the conversion with that error.
1439    pub fn convert_streaming<F>(
1440        &mut self,
1441        bytes: &[u8],
1442        password: Option<&str>,
1443        name: &str,
1444        emit: F,
1445    ) -> Result<(), PdfError>
1446    where
1447        F: FnMut(Vec<Node>, Vec<(String, String)>) -> Result<(), PdfError>,
1448    {
1449        let _ = name; // page nodes carry no name; the caller owns the document name.
1450        let pages = pdfium_backend::page_count(bytes, password)?;
1451        let range = self.resolve_range(pages)?;
1452        let selected = range.map_or(pages, |(a, b)| b - a + 1);
1453        let r = if self.target_workers >= 2 && selected >= self.parallel_min {
1454            self.convert_streaming_parallel(bytes, password, range, emit)
1455        } else {
1456            self.convert_streaming_serial(bytes, password, range, emit)
1457        };
1458        timing::report();
1459        r
1460    }
1461
1462    /// Serial streaming: render → process → emit, one page at a time, holding back
1463    /// only the tail that might still merge into the next page.
1464    fn convert_streaming_serial<F>(
1465        &mut self,
1466        bytes: &[u8],
1467        password: Option<&str>,
1468        range: Option<(usize, usize)>,
1469        mut emit: F,
1470    ) -> Result<(), PdfError>
1471    where
1472        F: FnMut(Vec<Node>, Vec<(String, String)>) -> Result<(), PdfError>,
1473    {
1474        let mut asm = assemble::StreamAssembler::new();
1475        let render_image = !self.no_ocr;
1476        let worker = self.primary()?;
1477        pdfium_backend::for_each_page(
1478            bytes,
1479            password,
1480            render_image,
1481            range,
1482            |n, _total, mut page| {
1483                // Confidence is dropped on the streaming path: the report is
1484                // only complete once every page has run, which defeats
1485                // page-by-page emission — buffered `convert` carries it.
1486                let (nodes, links, _conf) = worker.process(n, &mut page)?;
1487                emit(asm.push(nodes), links)
1488            },
1489        )?;
1490        emit(asm.finish(), Vec::new())
1491    }
1492
1493    /// Parallel streaming: pages render serially on a dedicated thread (pdfium is
1494    /// not thread-safe) and process across the worker pool; results carry their
1495    /// page index and are reordered on the calling thread into a
1496    /// [`assemble::StreamAssembler`], which emits each page in document order as
1497    /// soon as its predecessors have arrived. Bounded channels keep only a handful
1498    /// of pages resident and let `emit`'s backpressure reach the renderer.
1499    fn convert_streaming_parallel<F>(
1500        &mut self,
1501        bytes: &[u8],
1502        password: Option<&str>,
1503        range: Option<(usize, usize)>,
1504        mut emit: F,
1505    ) -> Result<(), PdfError>
1506    where
1507        F: FnMut(Vec<Node>, Vec<(String, String)>) -> Result<(), PdfError>,
1508    {
1509        self.ensure_pool()?;
1510        let n_workers = self.pool.len();
1511        let render_image = !self.no_ocr;
1512        let layout_batch = pdf_layout_batch();
1513        // Bound sized so every worker can accumulate a full layout batch while
1514        // rendering stays ahead (and never below the pre-#73 render-ahead of
1515        // two pages per worker); still a hard cap on resident page bitmaps.
1516        let (work_tx, work_rx) = sync_channel::<(usize, PdfPage)>(n_workers * layout_batch.max(2));
1517        let work_rx: Arc<Mutex<Receiver<(usize, PdfPage)>>> = Arc::new(Mutex::new(work_rx));
1518        // Workers and the renderer report here; the calling thread drains it in
1519        // page order. Bounded so workers block (bounding resident bitmaps) when the
1520        // consumer falls behind.
1521        let (res_tx, res_rx) = sync_channel::<Result<(usize, PageOut), PdfError>>(n_workers * 2);
1522
1523        let mut workers = std::mem::take(&mut self.pool);
1524        let mut asm = assemble::StreamAssembler::new();
1525        let mut first_err: Option<PdfError> = None;
1526
1527        std::thread::scope(|s| {
1528            // Workers: pull a batch of pages (whatever is already rendered, up
1529            // to the layout batch size), process it, report (index-tagged)
1530            // results.
1531            for worker in workers.iter_mut() {
1532                let work_rx = Arc::clone(&work_rx);
1533                let res_tx = res_tx.clone();
1534                s.spawn(move || 'outer: loop {
1535                    let mut batch = Vec::new();
1536                    {
1537                        let rx = work_rx.lock().unwrap();
1538                        match rx.recv() {
1539                            Ok(item) => {
1540                                batch.push(item);
1541                                while batch.len() < layout_batch {
1542                                    match rx.try_recv() {
1543                                        Ok(item) => batch.push(item),
1544                                        Err(_) => break,
1545                                    }
1546                                }
1547                            }
1548                            Err(_) => break,
1549                        }
1550                    }
1551                    let outs = worker.process_batch(&mut batch);
1552                    for ((idx, _), out) in batch.iter().zip(outs) {
1553                        if res_tx.send(out.map(|o| (*idx, o))).is_err() {
1554                            break 'outer; // consumer gone
1555                        }
1556                    }
1557                });
1558            }
1559            // Renderer: feed pages to the pool on its own thread (pdfium stays on a
1560            // single thread); report a render error through the same channel.
1561            {
1562                let res_tx = res_tx.clone();
1563                s.spawn(move || {
1564                    let render = pdfium_backend::for_each_page(
1565                        bytes,
1566                        password,
1567                        render_image,
1568                        range,
1569                        |i, _total, page| {
1570                            work_tx
1571                                .send((i, page))
1572                                .map_err(|_| PdfError::Pdfium("page-worker channel closed".into()))
1573                        },
1574                    );
1575                    drop(work_tx); // signal workers to finish
1576                    if let Err(e) = render {
1577                        let _ = res_tx.send(Err(e));
1578                    }
1579                });
1580            }
1581            // Drop our own sender so the channel closes once the threads finish.
1582            drop(res_tx);
1583
1584            // Collector (this thread): reorder into document order and emit.
1585            // With a page window, indices start at the window's first page.
1586            let mut buffer: BTreeMap<usize, PageOut> = BTreeMap::new();
1587            let mut next = range.map_or(0, |(first, _)| first);
1588            for msg in res_rx.iter() {
1589                match msg {
1590                    Err(e) => {
1591                        if first_err.is_none() {
1592                            first_err = Some(e);
1593                        }
1594                    }
1595                    Ok((idx, out)) => {
1596                        buffer.insert(idx, out);
1597                        if first_err.is_some() {
1598                            continue; // keep draining so the threads can exit
1599                        }
1600                        while let Some((nodes, links, _conf)) = buffer.remove(&next) {
1601                            if let Err(e) = emit(asm.push(nodes), links) {
1602                                first_err = Some(e);
1603                                break;
1604                            }
1605                            next += 1;
1606                        }
1607                    }
1608                }
1609            }
1610        });
1611        // Threads have joined; restore the pool for the next conversion.
1612        self.pool = workers;
1613
1614        if let Some(e) = first_err {
1615            return Err(e);
1616        }
1617        emit(asm.finish(), Vec::new())
1618    }
1619
1620    /// Lazily grow the pool to `target_workers`, loading the new workers
1621    /// concurrently (model load is mostly I/O + mmap, so N loads overlap to roughly
1622    /// one load's wall-time). Cached for reuse across documents.
1623    fn ensure_pool(&mut self) -> Result<(), PdfError> {
1624        let need = self.target_workers.saturating_sub(self.pool.len());
1625        if need == 0 {
1626            return Ok(());
1627        }
1628        let intra = pdf_intra();
1629        let no_ocr = self.no_ocr;
1630        let force = self.force_full_page_ocr;
1631        let ntp = self.no_text_panels;
1632        let ocr_lang = self.ocr_lang;
1633        let enrich = self.enrich;
1634        let tables = self.tables_slot();
1635        let enrich_slots = self.enrich_slots();
1636        let loaded: Vec<Result<Worker, PdfError>> = std::thread::scope(|s| {
1637            let handles: Vec<_> = (0..need)
1638                .map(|_| {
1639                    let tables = tables.clone();
1640                    let enrich_slots = enrich_slots.clone();
1641                    s.spawn(move || {
1642                        Worker::load(
1643                            intra,
1644                            tables,
1645                            enrich_slots,
1646                            enrich,
1647                            no_ocr,
1648                            force,
1649                            ntp,
1650                            ocr_lang,
1651                        )
1652                    })
1653                })
1654                .collect();
1655            handles.into_iter().map(|h| h.join().unwrap()).collect()
1656        });
1657        for w in loaded {
1658            self.pool.push(w?);
1659        }
1660        Ok(())
1661    }
1662
1663    /// Convert a standalone image (PNG/JPEG/TIFF/WebP/…) as a single page —
1664    /// docling routes images through the same layout+OCR pipeline as a PDF page.
1665    pub fn convert_image(&mut self, bytes: &[u8], name: &str) -> Result<DoclingDocument, PdfError> {
1666        let image = decode_image_limited(bytes)?;
1667        let (w, h) = image.dimensions();
1668        // The image is its own page rendered at 1 px per "point" (scale 1.0); a
1669        // standalone image has no text layer, so OCR supplies the cells.
1670        let page = PdfPage {
1671            width: w as f32,
1672            height: h as f32,
1673            scale: 1.0,
1674            cells: Vec::new(),
1675            code_cells: Vec::new(),
1676            word_cells: Vec::new(),
1677            image,
1678            links: Vec::new(),
1679        };
1680        self.process_pages(vec![page], name)
1681    }
1682
1683    /// Run layout (+ OCR for cell-less pages) and assemble each already-rendered
1684    /// page (image / METS inputs, which are small and already materialised).
1685    fn process_pages(
1686        &mut self,
1687        mut pages: Vec<PdfPage>,
1688        name: &str,
1689    ) -> Result<DoclingDocument, PdfError> {
1690        let mut doc = DoclingDocument::new(name);
1691        let mut confs = std::collections::BTreeMap::new();
1692        let worker = self.primary()?;
1693        for (n, page) in pages.iter_mut().enumerate() {
1694            let (mut nodes, links, conf) = worker.process(n, page)?;
1695            assemble::stamp_page_no(&mut nodes, n + 1);
1696            doc.nodes.extend(nodes);
1697            doc.links.extend(links);
1698            confs.insert(n + 1, conf);
1699        }
1700        assemble::merge_continuations(&mut doc.nodes);
1701        doc.confidence = Some(docling_core::ConfidenceReport::from_pages(confs));
1702        Ok(doc)
1703    }
1704}
1705
1706#[cfg(feature = "ml")]
1707/// Convenience one-shot conversion (loads the pipeline per call). Errors are
1708/// detailed and surfaced (never silently skipped).
1709pub fn convert(
1710    bytes: &[u8],
1711    password: Option<&str>,
1712    name: &str,
1713) -> Result<DoclingDocument, PdfError> {
1714    convert_with_options(
1715        bytes,
1716        password,
1717        name,
1718        false,
1719        false,
1720        false,
1721        false,
1722        EnrichmentOptions::default(),
1723        None,
1724        None,
1725    )
1726}
1727
1728#[cfg(feature = "ml")]
1729/// Like [`convert`], but optionally skips loading/running TableFormer (see
1730/// [`Pipeline::no_table_former`]) and/or layout+OCR+TableFormer entirely (see
1731/// [`Pipeline::no_ocr`]), and/or enables the enrichment passes (see
1732/// [`Pipeline::enrichments`]).
1733// One positional per pipeline switch mirrors the Pipeline builder; growing
1734// past clippy's arity cap is the price of keeping this one-shot signature
1735// stable-ish instead of churning callers into an options struct mid-series.
1736#[allow(clippy::too_many_arguments)]
1737pub fn convert_with_options(
1738    bytes: &[u8],
1739    password: Option<&str>,
1740    name: &str,
1741    no_table_former: bool,
1742    no_ocr: bool,
1743    force_full_page_ocr: bool,
1744    no_text_panels: bool,
1745    enrich: EnrichmentOptions,
1746    pages: Option<(usize, usize)>,
1747    ocr_lang: Option<OcrLang>,
1748) -> Result<DoclingDocument, PdfError> {
1749    Pipeline::new()?
1750        .no_table_former(no_table_former)
1751        .no_ocr(no_ocr)
1752        .force_full_page_ocr(force_full_page_ocr)
1753        .no_text_panels(no_text_panels)
1754        .enrichments(enrich)
1755        .pages(pages)
1756        .ocr_lang(ocr_lang)
1757        .convert(bytes, password, name)
1758}
1759
1760#[cfg(feature = "ml")]
1761/// Convenience one-shot image conversion (loads the pipeline per call).
1762pub fn convert_image(bytes: &[u8], name: &str) -> Result<DoclingDocument, PdfError> {
1763    convert_image_with_options(
1764        bytes,
1765        name,
1766        false,
1767        false,
1768        false,
1769        EnrichmentOptions::default(),
1770        None,
1771    )
1772}
1773
1774#[cfg(feature = "ml")]
1775/// Like [`convert_image`], but optionally skips loading/running TableFormer (see
1776/// [`Pipeline::no_table_former`]) and/or layout+OCR+TableFormer entirely (see
1777/// [`Pipeline::no_ocr`]), and/or enables the enrichment passes.
1778pub fn convert_image_with_options(
1779    bytes: &[u8],
1780    name: &str,
1781    no_table_former: bool,
1782    no_ocr: bool,
1783    no_text_panels: bool,
1784    enrich: EnrichmentOptions,
1785    ocr_lang: Option<OcrLang>,
1786) -> Result<DoclingDocument, PdfError> {
1787    Pipeline::new()?
1788        .no_table_former(no_table_former)
1789        .no_ocr(no_ocr)
1790        .no_text_panels(no_text_panels)
1791        .enrichments(enrich)
1792        .ocr_lang(ocr_lang)
1793        .convert_image(bytes, name)
1794}
1795
1796#[cfg(feature = "ml")]
1797/// Convert pre-segmented pages (image + already-known text cells, e.g. METS/hOCR
1798/// scans) through the shared layout + assembly pipeline.
1799pub fn convert_pages(pages: Vec<PdfPage>, name: &str) -> Result<DoclingDocument, PdfError> {
1800    convert_pages_with_options(
1801        pages,
1802        name,
1803        false,
1804        false,
1805        false,
1806        EnrichmentOptions::default(),
1807    )
1808}
1809
1810#[cfg(feature = "ml")]
1811/// Like [`convert_pages`], but optionally skips loading/running TableFormer (see
1812/// [`Pipeline::no_table_former`]) and/or layout+OCR+TableFormer entirely (see
1813/// [`Pipeline::no_ocr`]), and/or enables the enrichment passes.
1814pub fn convert_pages_with_options(
1815    pages: Vec<PdfPage>,
1816    name: &str,
1817    no_table_former: bool,
1818    no_ocr: bool,
1819    no_text_panels: bool,
1820    enrich: EnrichmentOptions,
1821) -> Result<DoclingDocument, PdfError> {
1822    Pipeline::new()?
1823        .no_table_former(no_table_former)
1824        .no_text_panels(no_text_panels)
1825        .no_ocr(no_ocr)
1826        .enrichments(enrich)
1827        .process_pages(pages, name)
1828}
1829
1830#[cfg(feature = "ml")]
1831#[cfg(all(test, feature = "ml"))]
1832mod image_limit_tests {
1833    use super::decode_image_with_max_side;
1834
1835    /// A small valid PNG encoded via the `image` crate (robust vs. a hand-rolled
1836    /// byte literal).
1837    fn png_bytes(w: u32, h: u32) -> Vec<u8> {
1838        use std::io::Cursor;
1839        let img = image::RgbImage::new(w, h);
1840        let mut out = Vec::new();
1841        img.write_to(&mut Cursor::new(&mut out), image::ImageFormat::Png)
1842            .unwrap();
1843        out
1844    }
1845
1846    #[test]
1847    fn normal_image_decodes_under_the_cap() {
1848        let img = decode_image_with_max_side(&png_bytes(8, 8), 30_000).expect("8x8 decodes");
1849        assert_eq!(img.dimensions(), (8, 8));
1850    }
1851
1852    #[test]
1853    fn dimensions_over_the_cap_are_rejected_not_aborted() {
1854        // A per-side cap below the image's declared size must yield a
1855        // recoverable Err, never an allocation-abort — the mechanism that stops
1856        // a crafted image declaring 60000×60000 from OOM-killing the process.
1857        let r = decode_image_with_max_side(&png_bytes(8, 8), 4);
1858        assert!(
1859            r.is_err(),
1860            "decode must fail under the pixel cap, not abort"
1861        );
1862    }
1863}
1864
1865#[cfg(test)]
1866mod median_tests {
1867    #[test]
1868    fn median_of_empty_is_zero_not_a_panic() {
1869        // A crafted table can leave a row/column with zero matched cells; the
1870        // even-count branch would index values[0 - 1] and panic (→ remote crash
1871        // via docling-serve) without the empty guard.
1872        assert_eq!(super::tf_match::median_for_test(&mut []), 0.0);
1873        assert_eq!(super::tf_match::median_for_test(&mut [4.0, 2.0]), 3.0);
1874        assert_eq!(super::tf_match::median_for_test(&mut [5.0, 1.0, 3.0]), 3.0);
1875    }
1876}
1877
1878#[cfg(test)]
1879mod send_check {
1880    /// The Node bindings (`docling-node`) run a shared [`super::Pipeline`] on
1881    /// libuv worker threads (`Arc<Mutex<Pipeline>>`), which is only sound while
1882    /// `Pipeline: Send` holds — this fails to compile if a non-`Send` field
1883    /// (e.g. an `Rc` or a raw pdfium handle) ever lands in the pipeline.
1884    fn assert_send<T: Send>() {}
1885
1886    #[test]
1887    fn pipeline_is_send() {
1888        assert_send::<super::Pipeline>();
1889    }
1890}