Skip to main content

docling_pdf/
tableformer.rs

1//! TableFormer: table-structure recovery via docling-ibm-models, exported to
2//! ONNX by `scripts/install/export_tableformer.py`. The image encoder + tag-transformer
3//! encoder run once to a memory tensor; the decoder is then stepped
4//! autoregressively to emit an OTSL structure-token sequence (the same model
5//! docling runs). See docs/PDF_CONFORMANCE.md.
6
7use crate::pdfium_backend::TextCell;
8// The ONNX-free half (preprocessing, structure corrections, bbox bookkeeping,
9// span merge, OTSL→grid) lives in tf_core so the browser build (#157 stage 3)
10// runs the same logic; this file owns the three `ort` sessions and the
11// owned-value KV-cache fast path.
12use crate::tf_core::{
13    argmax, build_table_cells, correct, merge_spans, preprocess_input, BboxBook, TableCell, END,
14    MAX_STEPS, START, UCEL,
15};
16use image::RgbImage;
17use ort::session::Session;
18use ort::value::{DynValue, Tensor, TensorRef};
19
20const SIDE: usize = crate::tf_core::SIDE as usize;
21const EMBED_DIM: usize = crate::tf_core::EMBED_DIM;
22/// Decoder geometry, fixed by the exported TableModel04_rs graph: the cached
23/// decoder threads a `[N_LAYERS, past, 1, EMBED_DIM]` per-layer state cache.
24const N_LAYERS: usize = 6;
25/// Cells per bbox-head run (see `finish_table`): bounds its broadcast
26/// intermediates to ~BBOX_CHUNK_CELLS × 784 × 512 × 4 bytes per tensor.
27/// 256 halves the peak on ~1,000-cell tables for <1% extra time (smaller
28/// chunks repeat the head's encoder-side work more often); output is
29/// byte-identical to the single run.
30const BBOX_CHUNK_CELLS: usize = 256;
31
32/// Split the flat `[n, EMBED_DIM]` hidden states into bbox-head runs of at
33/// most [`BBOX_CHUNK_CELLS`] whole cells.
34fn bbox_chunks(hiddens: &[f32]) -> impl Iterator<Item = &[f32]> {
35    hiddens.chunks(BBOX_CHUNK_CELLS * EMBED_DIM)
36}
37
38/// Resolve the encoder / decoder / bbox files exactly as [`TableFormer::load`]
39/// will (shared with `model_inventory`, so diagnostics can never drift from
40/// what actually loads). Explicit `DOCLING_TABLEFORMER_*` overrides win; the
41/// decoder otherwise picks by preference — INT8 variants first unless
42/// `DOCLING_RS_FP32` opts out, and within a precision the true-KV-cache
43/// export (`decoder_kv*`, one token per step, O(past) step cost) ranks ahead
44/// of the legacy layer-output-cache graph it matches byte-for-byte (91/91
45/// snapshot corpus exact with either; the KV graph re-measured ~13–17% faster
46/// warm, so speed wins the default and the legacy file stays as the smaller
47/// fallback). `decoder_kv` ranks ABOVE `decoder_int8`: the #97 hoisted fp32
48/// KV graph is faster than the quantized legacy graph on every machine
49/// measured, and it is byte-exact (its own int8 variant is not produced — see
50/// quantize_models.py). `decoder_kv_layers` ranks just above `decoder_kv`:
51/// the same graph with per-layer caches (#871, [`DecoderStyle::KvLayered`]),
52/// bit-identical and without per-step allocations that grow with the table.
53pub fn resolved_paths() -> (String, String, String) {
54    // The encoder ranks its fp16-weight repack (`encoder_fp16.onnx`, #374 —
55    // the same graph with the weights stored as fp16 and cast back to fp32
56    // at load, ~half the download, fp32 compute) ahead of the fp32 file
57    // unless `DOCLING_RS_FP32` opts out; an explicit override wins.
58    let enc = docling_core::env::nonempty("DOCLING_TABLEFORMER_ENCODER").unwrap_or_else(|| {
59        let candidates: &[&str] = if crate::prefer_fp32() {
60            &[".models/tableformer/encoder.onnx"]
61        } else {
62            &[
63                ".models/tableformer/encoder_fp16.onnx",
64                ".models/tableformer/encoder.onnx",
65            ]
66        };
67        candidates
68            .iter()
69            .map(|p| crate::resolve_asset(p))
70            .find(|p| std::path::Path::new(p).exists())
71            .unwrap_or_else(|| crate::resolve_asset(".models/tableformer/encoder.onnx"))
72    });
73    let dec = docling_core::env::nonempty("DOCLING_TABLEFORMER_DECODER").unwrap_or_else(|| {
74        let candidates: &[&str] = if crate::prefer_fp32() {
75            &[
76                ".models/tableformer/decoder_kv_layers.onnx",
77                ".models/tableformer/decoder_kv.onnx",
78                ".models/tableformer/decoder.onnx",
79            ]
80        } else {
81            &[
82                ".models/tableformer/decoder_kv_layers.onnx",
83                ".models/tableformer/decoder_kv.onnx",
84                ".models/tableformer/decoder_int8.onnx",
85                ".models/tableformer/decoder.onnx",
86            ]
87        };
88        candidates
89            .iter()
90            .map(|p| crate::resolve_asset(p))
91            .find(|p| std::path::Path::new(p).exists())
92            .unwrap_or_else(|| ".models/tableformer/decoder.onnx".to_string())
93    });
94    let bbx = docling_core::env::nonempty("DOCLING_TABLEFORMER_BBOX")
95        .unwrap_or_else(|| crate::resolve_asset(".models/tableformer/bbox.onnx"));
96    (enc, dec, bbx)
97}
98
99pub struct TableFormer {
100    encoder: Session,
101    decoder: Session,
102    bbox: Session,
103    /// Which decoder graph flavour is loaded, detected from the session's
104    /// input names (so an explicit `DOCLING_TABLEFORMER_DECODER` override
105    /// works with any of them).
106    style: DecoderStyle,
107    /// The `KvHoisted` decoder's `tag` input has a symbolic batch axis (the
108    /// dynamic-batch `decoder_kv.onnx` export): a page's tables decode
109    /// together, one step for all of them — see [`Self::predict_tables_on`].
110    /// The older fixed-`[1,1]` export decodes the tables one after another.
111    batched: bool,
112    /// The `KvLayered` loop runs through an `IoBinding` that keeps the caches
113    /// and the cross tensors where the provider computes ([`DeviceCache`]):
114    /// on under a CUDA-class provider (CUDA, TensorRT); DirectML only with
115    /// `DOCLING_RS_TF_IOBINDING=1`, which forces it either way (also on CPU,
116    /// which is how it is tested without a GPU; `0` turns it off). Cleared — for good — when a
117    /// table's first bound step fails, so the host-buffer loop takes over.
118    iobinding: bool,
119    /// The decoder session was built without the GPU provider (DirectML):
120    /// its tensors live on the host whatever the other sessions use.
121    decoder_on_host: bool,
122}
123
124/// The three decoder-graph generations the loop supports.
125#[derive(Clone, Copy, PartialEq, Eq)]
126enum DecoderStyle {
127    /// `decoder.onnx`: layer-output cache; feeds the full `tags` prefix and a
128    /// single `cache` every step.
129    Legacy,
130    /// The pre-#97 `decoder_kv.onnx`: one tag per step, `cache_k`/`cache_v`,
131    /// with the stacked `cross_k`/`cross_v` re-split inside every step.
132    KvStacked,
133    /// The #97 `decoder_kv.onnx`: one tag per step, and the constant cross
134    /// tensors arrive as 2×`N_LAYERS` per-layer inputs (`cross_kt_i` already
135    /// transposed for q·Kᵀ, `cross_v_i`), computed once per table by the
136    /// encoder — the step graph does no work proportional to their size.
137    KvHoisted,
138    /// `decoder_kv_layers.onnx` (#871, `scripts/install/kv_layers.py`): the
139    /// `KvHoisted` graph with the self-attention cache per layer — `Kᵀ` and
140    /// `V` in and out for each layer instead of two stacked tensors — whose
141    /// outputs land in buffers the loop owns ([`LayerCache`]).
142    KvLayered,
143}
144
145impl DecoderStyle {
146    /// One tag per step with the per-layer cross tensors the encoder emits.
147    fn hoisted(self) -> bool {
148        matches!(self, DecoderStyle::KvHoisted | DecoderStyle::KvLayered)
149    }
150}
151
152/// KV-cache geometry fixed by the `decoder_kv.onnx` export
153/// (`[N_LAYERS, 1, KV_HEADS, past, KV_HEAD_DIM]`, `KV_HEADS × KV_HEAD_DIM = EMBED_DIM`).
154const KV_HEADS: usize = 8;
155const KV_HEAD_DIM: usize = 64;
156
157/// The autoregressive decode state: `a` is the legacy layer-output cache, or
158/// `cache_k` for the KV graph; `b` is `cache_v` (KV graph only). `None` = first
159/// step (the zero-`past` empties are allocated per table by [`TableFormer::empty_cache`]).
160#[derive(Default)]
161struct DecodeCache {
162    a: Option<DynValue>,
163    b: Option<DynValue>,
164    /// The `KvLayered` caches, set up on the first step.
165    layers: Option<LayerCache>,
166    /// The `KvLayered` caches on the device, when the loop binds them there.
167    device: Option<DeviceCache>,
168}
169
170/// The `KvLayered` decode under a GPU provider (CUDA, TensorRT, DirectML):
171/// an `IoBinding` holding the
172/// step's inputs where the provider computes. With plain runs, every step
173/// handed ONNX Runtime host tensors to copy to the device and got host
174/// tensors back — the 12 constant cross tensors (~19 MB per table, the same
175/// every step) and both directions of the KV cache (~4 MB each way at 160
176/// tokens, growing) — so on CUDA a step cost 12.6 ms against the ~3 ms its
177/// 49 small GEMMs take. Here the cross tensors are bound once per decode
178/// (one copy), the cache outputs stay in device memory and are bound as the
179/// next step's inputs, and only `logits`/`hidden` (kilobytes) come back.
180/// Same graph, same kernels: the output is the plain loop's.
181struct DeviceCache {
182    binding: ort::session::IoBinding,
183    rows: usize,
184    past: usize,
185    /// The last step's cache outputs, where the provider wrote them.
186    cache: Vec<DynValue>,
187    /// Zero-`past` first-step inputs.
188    empty: Vec<Tensor<f32>>,
189    /// Where the cache outputs are bound: the GPU provider's device memory,
190    /// or the CPU when `DOCLING_RS_TF_IOBINDING` forces the loop without one.
191    device: ort::memory::MemoryInfo<'static>,
192}
193
194/// The `KvLayered` decoder's self-attention caches, in memory docling owns
195/// (#871). With ONNX Runtime allocating them, every step of a table made a
196/// cache one token longer than the last and freed the previous one —
197/// hundreds of ever-larger blocks per table, which on linux/arm64 with the
198/// CPU arena off fragmented glibc's heap until a 6 GiB container ran out of
199/// memory. Here each step reads its caches from one of two buffers and ONNX
200/// Runtime writes the one-token-longer caches straight into the other (the
201/// outputs are preallocated over it, [`docling_onnx::borrowed_f32`]); the
202/// next step swaps them. Each buffer holds the `2 × N_LAYERS` tensors in
203/// regions of `cap` floats — per layer `Kᵀ` `[rows, H, hd, past]`, then `V`
204/// `[rows, H, past, hd]` — sized for [`MAX_STEPS`] tokens: zeroed (`calloc`,
205/// so mapped lazily), only the prefix a table reaches becomes resident, and
206/// they are released when the table (or the page's batch) is done.
207struct LayerCache {
208    rows: usize,
209    past: usize,
210    cap: usize,
211    bufs: [Vec<f32>; 2],
212    /// Which of `bufs` holds the current caches.
213    cur: usize,
214    /// Zero-`past` first-step inputs (ort's slice constructors reject a
215    /// 0-length dim, the session allocator does not).
216    empty: Vec<Tensor<f32>>,
217}
218
219impl LayerCache {
220    fn new(alloc: &ort::memory::Allocator, rows: usize) -> Result<Self, String> {
221        let cap = rows * KV_HEADS * MAX_STEPS * KV_HEAD_DIM;
222        let mut empty = Vec::with_capacity(2 * N_LAYERS);
223        for j in 0..2 * N_LAYERS {
224            empty.push(
225                Tensor::<f32>::new(alloc, Self::shape(j, rows, 0))
226                    .map_err(|e| format!("tableformer: empty kv cache: {e}"))?,
227            );
228        }
229        Ok(Self {
230            rows,
231            past: 0,
232            cap,
233            bufs: [vec![0.0; 2 * N_LAYERS * cap], vec![0.0; 2 * N_LAYERS * cap]],
234            cur: 0,
235            empty,
236        })
237    }
238
239    /// Tensor `j`'s shape at `past` tokens: `Kᵀ` for the first `N_LAYERS`.
240    fn shape(j: usize, rows: usize, past: usize) -> [usize; 4] {
241        if j < N_LAYERS {
242            [rows, KV_HEADS, KV_HEAD_DIM, past]
243        } else {
244            [rows, KV_HEADS, past, KV_HEAD_DIM]
245        }
246    }
247
248    fn input_name(j: usize) -> String {
249        if j < N_LAYERS {
250            format!("cache_kt_{j}")
251        } else {
252            format!("cache_v_{}", j - N_LAYERS)
253        }
254    }
255}
256
257/// Zero-`past` first-step cache tensors: `(cache, None)` for the legacy graph,
258/// `(cache_k, Some(cache_v))` for the KV graph.
259type EmptyCache = (Tensor<f32>, Option<Tensor<f32>>);
260
261/// Encoder outputs that drive the cached decode loop: the per-layer cross-attention
262/// K/V (projected from the image memory once, constant across decode steps) and
263/// `enc_out` for the bbox decoder. Kept as owned `ort` values so each decode step
264/// (and the bbox run) borrows them directly — no per-step extract/copy/re-wrap.
265struct EncodeOut {
266    /// Stacked `[N_LAYERS,1,H,S,hd]` cross K/V — the `Legacy`/`KvStacked`
267    /// decoders' inputs. `None` for `KvHoisted`, which reads the per-layer
268    /// tensors instead: the stacked pair is 2×9.6 MB per table, and a page's
269    /// tables are now all held encoded at once for the batched loop.
270    ck: Option<DynValue>,
271    cv: Option<DynValue>,
272    eo: DynValue,
273    /// `KvHoisted` only: per-layer `[cross_kt_0..N, cross_v_0..N]`, index-aligned
274    /// with the decoder's input names, borrowed by every decode step.
275    per_layer: Vec<(String, DynValue)>,
276}
277
278impl TableFormer {
279    /// Load the exported encoder/decoder/bbox ONNX graphs (env overrides, else
280    /// `.models/tableformer/{encoder,decoder,bbox}.onnx`). Returns `None` if any is
281    /// absent, so the pipeline falls back to geometric reconstruction.
282    pub fn load() -> Option<Self> {
283        Self::load_with(crate::intra_threads())
284    }
285
286    /// Like [`load`](Self::load) but with an explicit intra-op thread count, so a
287    /// parallel page-worker pool can run each table model on fewer threads (the
288    /// throughput comes from running pages concurrently, not from one fat model).
289    ///
290    /// See [`resolved_paths`] for the encoder/decoder/bbox file selection.
291    pub fn load_with(intra: usize) -> Option<Self> {
292        // (resolution shared with the model inventory — see resolved_paths)
293        let (enc, dec, bbx) = resolved_paths();
294        if crate::timing::enabled() {
295            eprintln!("docling-pdf: tableformer decoder: {dec}");
296        }
297        if [&enc, &dec, &bbx]
298            .iter()
299            .any(|p| !std::path::Path::new(p).exists())
300        {
301            // The geometric fallback is a supported, intentional configuration
302            // (docling has no ML table-structure equivalent baked in either), so
303            // this stays a single quiet stderr note rather than an error — but it
304            // fires every process (not per-worker) so a CWD-relative default that
305            // silently misses its files (a very easy mistake for anything not run
306            // from the repo root, e.g. an embedding app) is at least visible once.
307            warn_missing_once(&enc, &dec, &bbx);
308            return None;
309        }
310        // The decoder's KV-cache grows by one entry every autoregressive step, so
311        // its input shapes differ on every `run()` call. ONNX Runtime's memory
312        // pattern optimizer assumes stable shapes to plan buffer reuse; disabling
313        // it for this session avoids repeatedly re-validating/re-touching that
314        // plan (and the external-weights file) on each step. The bbox head has
315        // the same problem one level up: its `tag_h` input is `[ncells, 512]`
316        // and every table has a different cell count, so with the pattern
317        // planner on each run re-plans — and on this graph the plan is *worse*
318        // than none: 290 ms vs 54 ms for a 100-cell table, 560 vs 94 ms for
319        // 200 cells (ORT 1.22, 4 threads). It was 0.26 s per table on the
320        // corpus, more than the encoder.
321        //
322        // The decoder runs on ONE intra-op thread. A step is 49 small GEMMs
323        // over a single token — it streams the layer weights, it does not
324        // compute — so extra threads only add synchronisation: measured 4.1 ms
325        // per step on 1 thread vs 5.5 on 4 (7.1 vs 4.9 once the cache is 100+
326        // long). In the pool it also stops a table decode from taking all the
327        // cores away from the other workers' layout inference. And a
328        // single-thread session has a fixed reduction order, so table
329        // structure no longer varies run-to-run on near-tie tokens the way
330        // multi-threaded float sums let it (the conformance scripts pin one
331        // thread for exactly that reason; the default now matches them). The
332        // encoder keeps the shared budget: one 448×448 CNN + transformer pass
333        // per table, 680 ms single-threaded vs 165 on four.
334        // The decoder runs on the CPU under DirectML. As exported, every
335        // decoder graph fails inside the provider (`Reshape node_view`,
336        // 0x8007023E, on both adapters of the first DirectML machine that
337        // tried), and every table fell back to geometric structure: DirectML
338        // rejects the `allowzero=1` torch's exporter writes on each `view`
339        // Reshape. With it cleared — equivalent here, no target shape holds a
340        // 0 — the decoder runs there, but slower than on the CPU: 36.9 ms per
341        // step against 13.9 on the same laptop (2203.01017v2, 141 steps). A
342        // step is ~150 tiny ops over one token, kernel launches dominate, so
343        // the encoder and the bbox head stay on the GPU and only the decoder
344        // moves.
345        let decoder_on_host = docling_onnx::is_directml();
346        let build = |path: &str,
347                     mem_pattern: bool,
348                     threads: usize,
349                     host: bool|
350         -> Result<Session, String> {
351            let builder = docling_onnx::session_builder()?
352                .with_intra_threads(threads)
353                .map_err(|e| e.to_string())?
354                .with_memory_pattern(mem_pattern)
355                .map_err(|e| e.to_string())?;
356            let variant = if mem_pattern {
357                "mem_pattern"
358            } else {
359                "no_mem_pattern"
360            };
361            let builder = if host {
362                docling_onnx::apply_host(builder)?
363            } else {
364                docling_onnx::apply(builder)?
365            };
366            docling_onnx::commit(builder, path, variant)
367                .map_err(|e| format!("tableformer load {path}: {e}"))
368        };
369        match (
370            build(&enc, true, intra, false),
371            build(&dec, false, 1, decoder_on_host),
372            build(&bbx, false, intra, false),
373        ) {
374            (Ok(encoder), Ok(decoder), Ok(bbox)) => {
375                let has = |n: &str| decoder.inputs().iter().any(|i| i.name() == n);
376                let style = if has("cache_kt_0") {
377                    DecoderStyle::KvLayered
378                } else if has("cross_kt_0") {
379                    DecoderStyle::KvHoisted
380                } else if has("cache_k") {
381                    DecoderStyle::KvStacked
382                } else {
383                    DecoderStyle::Legacy
384                };
385                if style.hoisted() && !encoder.outputs().iter().any(|o| o.name() == "cross_kt_0") {
386                    eprintln!(
387                        "docling-pdf: tableformer decoder needs per-layer cross tensors \
388                         (cross_kt_*) the encoder doesn't emit — re-download or re-export \
389                         the model set (scripts/install/export_tableformer.py); \
390                         falling back to geometric tables"
391                    );
392                    return None;
393                }
394                // Dynamic batch axis on `tag` ⇒ the export batches decode
395                // steps across tables (ort reports a symbolic dim as -1).
396                let batched = style.hoisted()
397                    && decoder.inputs().iter().any(|i| {
398                        i.name() == "tag"
399                            && matches!(i.dtype(), ort::value::ValueType::Tensor { shape, .. }
400                                if shape.first().is_some_and(|d| *d < 0))
401                    });
402                if crate::timing::enabled() && batched {
403                    eprintln!("docling-pdf: tableformer decoder batches a page's tables per step");
404                }
405                if crate::timing::enabled() && decoder_on_host {
406                    eprintln!("docling-pdf: tableformer decoder runs on the CPU (DirectML)");
407                }
408                let iobinding = style == DecoderStyle::KvLayered
409                    && match docling_core::env::nonempty("DOCLING_RS_TF_IOBINDING") {
410                        Some(_) => docling_core::env::flag("DOCLING_RS_TF_IOBINDING"),
411                        // Not when the decoder was kept on the host
412                        // (DirectML, above): nothing to keep on a device.
413                        None => !decoder_on_host && docling_onnx::device_memory_info().is_some(),
414                    };
415                if crate::timing::enabled() && iobinding {
416                    eprintln!("docling-pdf: tableformer decoder keeps its caches on the device");
417                }
418                Some(Self {
419                    encoder,
420                    decoder,
421                    bbox,
422                    style,
423                    batched,
424                    iobinding,
425                    decoder_on_host,
426                })
427            }
428            _ => None,
429        }
430    }
431
432    /// Run the image encoder and capture what the cached decoder loop needs: each
433    /// decoder layer's cross-attention K/V (projected from the image memory once,
434    /// shape `[N_LAYERS,1,H,S,head_dim]`) and `enc_out` for the bbox decoder.
435    fn encode(&mut self, img: &RgbImage) -> Result<EncodeOut, String> {
436        let input = crate::timing::timed("tf.preprocess", || preprocess(img))?;
437        let mut enc_out = crate::timing::timed("tf.encoder", || {
438            docling_onnx::run(&mut self.encoder, ort::inputs!["image" => input])
439                .map_err(|e| format!("tableformer: encode: {e}"))
440        })?;
441        let mut per_layer = Vec::new();
442        if self.style.hoisted() {
443            for prefix in ["cross_kt_", "cross_v_"] {
444                for i in 0.. {
445                    let name = format!("{prefix}{i}");
446                    match enc_out.remove(&name) {
447                        Some(v) => per_layer.push((name, v)),
448                        None => break,
449                    }
450                }
451            }
452            if per_layer.is_empty() {
453                return Err("tableformer: encoder emitted no cross_kt_* outputs".into());
454            }
455        }
456        let mut grab = |name: &str| -> Result<DynValue, String> {
457            enc_out
458                .remove(name)
459                .ok_or_else(|| format!("tableformer: encoder output {name} missing"))
460        };
461        let hoisted = self.style.hoisted();
462        Ok(EncodeOut {
463            ck: if hoisted {
464                None
465            } else {
466                Some(grab("cross_k")?)
467            },
468            cv: if hoisted {
469                None
470            } else {
471                Some(grab("cross_v")?)
472            },
473            eo: grab("enc_out")?,
474            per_layer,
475        })
476    }
477
478    /// One doubly-cached decode step: feed the current `tags`, the constant cross
479    /// K/V, and the growing self-attention `cache`; return the raw argmax tag and
480    /// the last token's hidden state, advancing the cache. The cache stays an owned
481    /// `ort` value — the previous step's `out_cache` output is fed back directly,
482    /// never extracted or copied (it grows every step, so per-step copies were
483    /// O(steps²) float traffic). `empty_cache` is the zero-`past` value used on the
484    /// first step (ort's array constructors reject a 0-length dim, so it is
485    /// allocated through the session allocator by the caller).
486    fn decode_step(
487        &mut self,
488        tags: &[i64],
489        enc: &EncodeOut,
490        cache: &mut DecodeCache,
491        empty: &EmptyCache,
492    ) -> Result<(i64, Vec<f32>), String> {
493        crate::timing::timed("tf.decode_step", || {
494            self.decode_step_inner(tags, enc, cache, empty)
495        })
496    }
497
498    fn decode_step_inner(
499        &mut self,
500        tags: &[i64],
501        enc: &EncodeOut,
502        cache: &mut DecodeCache,
503        empty: &EmptyCache,
504    ) -> Result<(i64, Vec<f32>), String> {
505        if self.style.hoisted() {
506            // #97 graph: one tag; the constant per-layer cross tensors are
507            // borrowed views — the step pays nothing proportional to them.
508            let last = *tags.last().expect("decode starts from <start>");
509            let (raws, hidden) = self.step_kv(&[last], &enc.per_layer, cache, empty)?;
510            return Ok((raws[0], hidden));
511        }
512        let (ck, cv) = match (enc.ck.as_ref(), enc.cv.as_ref()) {
513            (Some(k), Some(v)) => (k, v),
514            _ => return Err("tableformer: stacked cross K/V missing".into()),
515        };
516        let mut dout = match self.style {
517            DecoderStyle::KvHoisted | DecoderStyle::KvLayered => unreachable!("handled above"),
518            DecoderStyle::KvStacked => {
519                // Pre-#97 KV graph: feed only the newly emitted tag; the projected
520                // K/V for the whole prefix live in cache_k/cache_v and are fed
521                // back as-is.
522                let last = *tags.last().expect("decode starts from <start>");
523                let tag_t = Tensor::from_array(([1usize, 1usize], vec![last]))
524                    .map_err(|e| format!("tableformer: tag: {e}"))?;
525                match (cache.a.as_ref(), cache.b.as_ref()) {
526                    (Some(k), Some(v)) => docling_onnx::run(
527                        &mut self.decoder,
528                        ort::inputs![
529                        "tag" => tag_t, "cross_k" => ck, "cross_v" => cv,
530                        "cache_k" => k, "cache_v" => v],
531                    ),
532                    _ => docling_onnx::run(
533                        &mut self.decoder,
534                        ort::inputs![
535                        "tag" => tag_t, "cross_k" => ck, "cross_v" => cv,
536                        "cache_k" => &empty.0,
537                        "cache_v" => empty.1.as_ref().expect("kv empty cache has both halves")],
538                    ),
539                }
540            }
541            DecoderStyle::Legacy => {
542                let tags_t = Tensor::from_array(([tags.len(), 1usize], tags.to_vec()))
543                    .map_err(|e| format!("tableformer: tags: {e}"))?;
544                match cache.a.as_ref() {
545                    None => docling_onnx::run(
546                        &mut self.decoder,
547                        ort::inputs![
548                        "tags" => tags_t, "cross_k" => ck, "cross_v" => cv,
549                        "cache" => &empty.0],
550                    ),
551                    Some(c) => docling_onnx::run(
552                        &mut self.decoder,
553                        ort::inputs![
554                        "tags" => tags_t, "cross_k" => ck, "cross_v" => cv,
555                        "cache" => c],
556                    ),
557                }
558            }
559        }
560        .map_err(|e| format!("tableformer: decode: {e}"))?;
561        let (_, logits) = dout["logits"]
562            .try_extract_tensor::<f32>()
563            .map_err(|e| format!("tableformer: logits: {e}"))?;
564        let raw = argmax(logits) as i64;
565        let (_, hidden) = dout["hidden"]
566            .try_extract_tensor::<f32>()
567            .map_err(|e| format!("tableformer: hidden: {e}"))?;
568        let hidden = hidden.to_vec();
569        if self.style != DecoderStyle::Legacy {
570            cache.a = Some(
571                dout.remove("out_cache_k")
572                    .ok_or_else(|| "tableformer: out_cache_k missing".to_string())?,
573            );
574            cache.b = Some(
575                dout.remove("out_cache_v")
576                    .ok_or_else(|| "tableformer: out_cache_v missing".to_string())?,
577            );
578        } else {
579            cache.a = Some(
580                dout.remove("out_cache")
581                    .ok_or_else(|| "tableformer: decoder output out_cache missing".to_string())?,
582            );
583        }
584        Ok((raw, hidden))
585    }
586
587    /// One step of a one-tag-per-step graph over `tags.len()` rows, through
588    /// whichever cache layout the loaded decoder has.
589    fn step_kv(
590        &mut self,
591        tags: &[i64],
592        per_layer: &[(String, DynValue)],
593        cache: &mut DecodeCache,
594        empty: &EmptyCache,
595    ) -> Result<(Vec<i64>, Vec<f32>), String> {
596        if self.style == DecoderStyle::KvLayered && self.iobinding {
597            match self.step_kv_device(tags, per_layer, cache) {
598                Ok(step) => return Ok(step),
599                // A provider that will not hand out its device memory to a
600                // binding (the device-resident path is only verified on CUDA
601                // and on CPU) fails on a decode's first step, before any
602                // state lives there: decode on with host caches, here and for
603                // every later table. A later failure is a real one.
604                Err(e) if cache.device.as_ref().is_none_or(|d| d.past == 0) => {
605                    eprintln!(
606                        "docling-pdf: tableformer: device-resident decode unavailable ({e}); \
607                         decoding with host caches"
608                    );
609                    self.iobinding = false;
610                    cache.device = None;
611                }
612                Err(e) => return Err(e),
613            }
614        }
615        if self.style == DecoderStyle::KvLayered {
616            self.step_kv_layered(tags, per_layer, cache)
617        } else {
618            self.step_kv_hoisted(tags, per_layer, cache, empty)
619        }
620    }
621
622    /// One `KvLayered` step through the decode's [`DeviceCache`] binding.
623    fn step_kv_device(
624        &mut self,
625        tags: &[i64],
626        per_layer: &[(String, DynValue)],
627        cache: &mut DecodeCache,
628    ) -> Result<(Vec<i64>, Vec<f32>), String> {
629        let rows = tags.len();
630        let err = |what: &str, e: ort::Error| format!("tableformer: {what}: {e}");
631        let dc = match &mut cache.device {
632            Some(dc) => dc,
633            None => {
634                let mut binding = self
635                    .decoder
636                    .create_binding()
637                    .map_err(|e| err("io binding", e))?;
638                // The constant cross tensors: copied to the device once here,
639                // and every step of this decode reads that copy.
640                for (name, v) in per_layer {
641                    binding
642                        .bind_input(name.as_str(), v)
643                        .map_err(|e| err(name, e))?;
644                }
645                binding
646                    .synchronize_inputs()
647                    .map_err(|e| err("cross inputs", e))?;
648                let alloc = self.decoder.allocator();
649                let mut empty = Vec::with_capacity(2 * N_LAYERS);
650                for j in 0..2 * N_LAYERS {
651                    empty.push(
652                        Tensor::<f32>::new(alloc, LayerCache::shape(j, rows, 0))
653                            .map_err(|e| err("empty kv cache", e))?,
654                    );
655                }
656                cache.device.insert(DeviceCache {
657                    binding,
658                    rows,
659                    past: 0,
660                    cache: Vec::new(),
661                    empty,
662                    device: if self.decoder_on_host {
663                        ort::memory::MemoryInfo::default()
664                    } else {
665                        docling_onnx::device_memory_info().unwrap_or_default()
666                    },
667                })
668            }
669        };
670        if dc.rows != rows || dc.past >= MAX_STEPS {
671            return Err(format!(
672                "tableformer: kv cache for {} rows at {} tokens, asked for {rows} rows one more",
673                dc.rows, dc.past
674            ));
675        }
676        let tag_t =
677            Tensor::from_array(([rows, 1usize], tags.to_vec())).map_err(|e| err("tag", e))?;
678        dc.binding
679            .bind_input("tag", &tag_t)
680            .map_err(|e| err("tag", e))?;
681        for j in 0..2 * N_LAYERS {
682            let name = LayerCache::input_name(j);
683            if dc.past == 0 {
684                dc.binding.bind_input(name.as_str(), &dc.empty[j])
685            } else {
686                dc.binding.bind_input(name.as_str(), &dc.cache[j])
687            }
688            .map_err(|e| err(&name, e))?;
689            // Re-bound every step: the output is one token longer each time,
690            // so ONNX Runtime must allocate it afresh rather than reuse the
691            // previous step's value.
692            let out = format!("out_{name}");
693            dc.binding
694                .bind_output_to_device(out.as_str(), &dc.device)
695                .map_err(|e| err(&out, e))?;
696        }
697        let host = ort::memory::MemoryInfo::default();
698        for name in ["logits", "hidden"] {
699            dc.binding
700                .bind_output_to_device(name, &host)
701                .map_err(|e| err(name, e))?;
702        }
703        // The CUDA arena is shrunk at a decode's first step only (#842's run
704        // option): shrinking after every step handed the step's working
705        // memory back to the driver and allocated it again on the next one —
706        // a cudaFree + cudaMalloc pair per token. Once per decode still
707        // returns the previous table's chunks before this one grows.
708        let shrink = if dc.past == 0 {
709            docling_onnx::run_options()
710        } else {
711            None
712        };
713        let mut dout = match shrink {
714            Some(opts) => self.decoder.run_binding_with_options(&dc.binding, opts),
715            None => self.decoder.run_binding(&dc.binding),
716        }
717        .map_err(|e| err("decode", e))?;
718        let (_, logits) = dout["logits"]
719            .try_extract_tensor::<f32>()
720            .map_err(|e| err("logits", e))?;
721        let vocab = logits.len() / rows;
722        let raws: Vec<i64> = logits
723            .chunks_exact(vocab)
724            .map(|row| argmax(row) as i64)
725            .collect();
726        let (_, hidden) = dout["hidden"]
727            .try_extract_tensor::<f32>()
728            .map_err(|e| err("hidden", e))?;
729        let hidden = hidden.to_vec();
730        let mut next = Vec::with_capacity(2 * N_LAYERS);
731        for j in 0..2 * N_LAYERS {
732            let out = format!("out_{}", LayerCache::input_name(j));
733            next.push(
734                dout.remove(out.as_str())
735                    .ok_or_else(|| format!("tableformer: {out} missing"))?,
736            );
737        }
738        drop(dout);
739        dc.cache = next;
740        dc.past += 1;
741        Ok((raws, hidden))
742    }
743
744    /// One `KvLayered` step: [`Self::step_kv_hoisted`] with the caches read
745    /// from and written into the [`LayerCache`] buffers — ONNX Runtime
746    /// allocates nothing that grows with the table (#871).
747    fn step_kv_layered(
748        &mut self,
749        tags: &[i64],
750        per_layer: &[(String, DynValue)],
751        cache: &mut DecodeCache,
752    ) -> Result<(Vec<i64>, Vec<f32>), String> {
753        let rows = tags.len();
754        let lc = match &mut cache.layers {
755            Some(lc) => lc,
756            None => cache
757                .layers
758                .insert(LayerCache::new(self.decoder.allocator(), rows)?),
759        };
760        if lc.rows != rows || lc.past >= MAX_STEPS {
761            return Err(format!(
762                "tableformer: kv cache for {} rows at {} tokens, asked for {rows} rows one more",
763                lc.rows, lc.past
764            ));
765        }
766        let (past, cap) = (lc.past, lc.cap);
767        let tag_t = Tensor::from_array(([rows, 1usize], tags.to_vec()))
768            .map_err(|e| format!("tableformer: tag: {e}"))?;
769        let [first, second] = &mut lc.bufs;
770        let (src, dst) = if lc.cur == 0 {
771            (&*first, second)
772        } else {
773            (&*second, first)
774        };
775        let mut inputs: Vec<(
776            std::borrow::Cow<'_, str>,
777            ort::session::SessionInputValue<'_>,
778        )> = Vec::with_capacity(1 + 2 * N_LAYERS + per_layer.len());
779        inputs.push(("tag".into(), tag_t.into()));
780        let in_len = rows * KV_HEADS * past * KV_HEAD_DIM;
781        for j in 0..2 * N_LAYERS {
782            let name = LayerCache::input_name(j);
783            if past == 0 {
784                inputs.push((name.into(), (&lc.empty[j]).into()));
785            } else {
786                let view = TensorRef::from_array_view((
787                    LayerCache::shape(j, rows, past),
788                    &src[j * cap..j * cap + in_len],
789                ))
790                .map_err(|e| format!("tableformer: {name}: {e}"))?;
791                inputs.push((name.into(), view.into()));
792            }
793        }
794        for (name, v) in per_layer {
795            inputs.push((name.as_str().into(), v.into()));
796        }
797        let alloc = self.decoder.allocator();
798        let mut outputs = ort::session::OutputSelector::default();
799        for (j, region) in dst.chunks_exact_mut(cap).enumerate() {
800            // SAFETY: `dst` is borrowed mutably for this whole step and
801            // nothing else touches it; the values over it live in `opts` and
802            // `dout`, both dropped at the end of this function, before the
803            // next step reads the buffer (or the cache frees it).
804            let out = unsafe {
805                docling_onnx::borrowed_f32(region, &LayerCache::shape(j, rows, past + 1), alloc)
806            }
807            .map_err(|e| format!("tableformer: kv cache output: {e}"))?;
808            outputs = outputs.preallocate(format!("out_{}", LayerCache::input_name(j)), out);
809        }
810        let opts = docling_onnx::run_options_with_outputs(outputs)?;
811        let dout = self
812            .decoder
813            .run_with_options(inputs, &opts)
814            .map_err(|e| format!("tableformer: decode: {e}"))?;
815        let (_, logits) = dout["logits"]
816            .try_extract_tensor::<f32>()
817            .map_err(|e| format!("tableformer: logits: {e}"))?;
818        let vocab = logits.len() / rows;
819        let raws: Vec<i64> = logits
820            .chunks_exact(vocab)
821            .map(|row| argmax(row) as i64)
822            .collect();
823        let (_, hidden) = dout["hidden"]
824            .try_extract_tensor::<f32>()
825            .map_err(|e| format!("tableformer: hidden: {e}"))?;
826        let hidden = hidden.to_vec();
827        drop(dout);
828        drop(opts);
829        lc.past += 1;
830        lc.cur ^= 1;
831        Ok((raws, hidden))
832    }
833
834    /// One `KvHoisted` step over `tags.len()` rows — one table per row. `tags`
835    /// holds each row's last emitted tag, `per_layer` the cross tensors with a
836    /// matching leading batch axis (the encoder's own `[1,…]` outputs for a
837    /// single table, or [`Self::batch_cross`]'s concatenation), and the cache
838    /// grows `[N_LAYERS, rows, H, past, hd]` in lockstep. Returns each row's raw
839    /// argmax tag and the `[rows, EMBED_DIM]` hidden states, flattened.
840    fn step_kv_hoisted(
841        &mut self,
842        tags: &[i64],
843        per_layer: &[(String, DynValue)],
844        cache: &mut DecodeCache,
845        empty: &EmptyCache,
846    ) -> Result<(Vec<i64>, Vec<f32>), String> {
847        let rows = tags.len();
848        let tag_t = Tensor::from_array(([rows, 1usize], tags.to_vec()))
849            .map_err(|e| format!("tableformer: tag: {e}"))?;
850        let mut inputs: Vec<(
851            std::borrow::Cow<'_, str>,
852            ort::session::SessionInputValue<'_>,
853        )> = Vec::with_capacity(3 + per_layer.len());
854        inputs.push(("tag".into(), tag_t.into()));
855        match (cache.a.as_ref(), cache.b.as_ref()) {
856            (Some(k), Some(v)) => {
857                inputs.push(("cache_k".into(), k.into()));
858                inputs.push(("cache_v".into(), v.into()));
859            }
860            _ => {
861                inputs.push(("cache_k".into(), (&empty.0).into()));
862                inputs.push((
863                    "cache_v".into(),
864                    empty
865                        .1
866                        .as_ref()
867                        .expect("kv empty cache has both halves")
868                        .into(),
869                ));
870            }
871        }
872        for (name, v) in per_layer {
873            inputs.push((name.as_str().into(), v.into()));
874        }
875        let mut dout = docling_onnx::run(&mut self.decoder, inputs)
876            .map_err(|e| format!("tableformer: decode: {e}"))?;
877        let (_, logits) = dout["logits"]
878            .try_extract_tensor::<f32>()
879            .map_err(|e| format!("tableformer: logits: {e}"))?;
880        let vocab = logits.len() / rows;
881        let raws: Vec<i64> = logits
882            .chunks_exact(vocab)
883            .map(|row| argmax(row) as i64)
884            .collect();
885        let (_, hidden) = dout["hidden"]
886            .try_extract_tensor::<f32>()
887            .map_err(|e| format!("tableformer: hidden: {e}"))?;
888        let hidden = hidden.to_vec();
889        cache.a = Some(
890            dout.remove("out_cache_k")
891                .ok_or_else(|| "tableformer: out_cache_k missing".to_string())?,
892        );
893        cache.b = Some(
894            dout.remove("out_cache_v")
895                .ok_or_else(|| "tableformer: out_cache_v missing".to_string())?,
896        );
897        Ok((raws, hidden))
898    }
899
900    /// Stack the per-layer cross tensors of several encoded tables along the
901    /// batch axis (`[1,H,hd,S]` × B → `[B,H,hd,S]`, same for `cross_v`), index-
902    /// aligned with the decoder's input names. One copy per page — ~20 MB per
903    /// table, nothing next to the decode steps it lets the tables share.
904    fn batch_cross(encs: &[EncodeOut]) -> Result<Vec<(String, DynValue)>, String> {
905        let b = encs.len();
906        let mut out = Vec::with_capacity(encs[0].per_layer.len());
907        for j in 0..encs[0].per_layer.len() {
908            let name = encs[0].per_layer[j].0.clone();
909            let mut data: Vec<f32> = Vec::new();
910            let mut dims = [b, 0, 0, 0];
911            for enc in encs {
912                let (shape, v) = enc.per_layer[j]
913                    .1
914                    .try_extract_tensor::<f32>()
915                    .map_err(|e| format!("tableformer: {name}: {e}"))?;
916                if shape.len() != 4 || shape[0] != 1 {
917                    return Err(format!("tableformer: {name}: unexpected shape {shape:?}"));
918                }
919                dims[1..].copy_from_slice(&[
920                    shape[1] as usize,
921                    shape[2] as usize,
922                    shape[3] as usize,
923                ]);
924                data.reserve(v.len() * b);
925                data.extend_from_slice(v);
926            }
927            let t = Tensor::from_array((dims, data))
928                .map_err(|e| format!("tableformer: {name}: {e}"))?;
929            out.push((name, t.into_dyn()));
930        }
931        Ok(out)
932    }
933
934    /// Decode `encs.len()` tables in lockstep: every step runs the decoder once
935    /// over all of them (a step is 49 weight-streaming GEMMs over one token per
936    /// row — B rows cost about what one does). The caches start empty for
937    /// every row and grow together, so nothing is ever padded or masked; a
938    /// table that emits `<end>` simply keeps its row (fed `END`, output
939    /// ignored) until the last one finishes. Row b of every op is exactly the
940    /// single-table computation, so each table's tokens and hidden states are
941    /// bit-identical to decoding it alone (asserted by the export script's
942    /// batching gate; the corpus snapshots pin it end-to-end).
943    fn decode_batch(&mut self, encs: &[EncodeOut]) -> Result<Vec<BboxBook>, String> {
944        let b = encs.len();
945        let cross = Self::batch_cross(encs)?;
946        let mut books: Vec<BboxBook> = (0..b).map(|_| BboxBook::new()).collect();
947        let mut active = vec![true; b];
948        let mut last = vec![START; b];
949        let mut cache = DecodeCache::default();
950        let empty = self.empty_cache(b)?;
951        crate::timing::timed("tf.decode_loop", || -> Result<(), String> {
952            // Each active table's `otsl` grows by one per step, so a shared
953            // step counter is the per-table `otsl.len() < MAX_STEPS` bound.
954            for _ in 0..MAX_STEPS {
955                if !active.iter().any(|a| *a) {
956                    break;
957                }
958                let (raws, hidden) = crate::timing::timed("tf.decode_step", || {
959                    self.step_kv(&last, &cross, &mut cache, &empty)
960                })?;
961                for t in 0..b {
962                    if !active[t] {
963                        continue;
964                    }
965                    let h = &hidden[t * EMBED_DIM..(t + 1) * EMBED_DIM];
966                    if books[t].step(raws[t], h) {
967                        last[t] = *books[t].tags.last().expect("step pushed a tag");
968                    } else {
969                        active[t] = false;
970                        last[t] = END;
971                    }
972                }
973            }
974            Ok(())
975        })?;
976        Ok(books)
977    }
978
979    /// The zero-`past` first-step cache(s) for `rows` tables, allocated through
980    /// the session allocator (ort's array constructors reject a 0-length dim;
981    /// the C API does allow it).
982    fn empty_cache(&self, rows: usize) -> Result<EmptyCache, String> {
983        let alloc = self.decoder.allocator();
984        if self.style != DecoderStyle::Legacy {
985            let mk = || {
986                Tensor::<f32>::new(alloc, [N_LAYERS, rows, KV_HEADS, 0usize, KV_HEAD_DIM])
987                    .map_err(|e| format!("tableformer: empty kv cache: {e}"))
988            };
989            Ok((mk()?, Some(mk()?)))
990        } else {
991            let c = Tensor::<f32>::new(alloc, [N_LAYERS, 0usize, 1, EMBED_DIM])
992                .map_err(|e| format!("tableformer: empty cache: {e}"))?;
993            Ok((c, None))
994        }
995    }
996
997    /// Predict the OTSL structure-token sequence for a table-region image.
998    pub fn predict_otsl(&mut self, img: &RgbImage) -> Result<Vec<i64>, String> {
999        let enc = self.encode(img)?;
1000        // Structure corrections live in tf_core::correct (shared with the wasm
1001        // path); docling's line_num is never incremented, so xcel→lcel fires on
1002        // every row.
1003        let mut tags: Vec<i64> = vec![START];
1004        let mut out: Vec<i64> = Vec::new();
1005        let mut prev_ucel = false;
1006        let mut cache = DecodeCache::default();
1007        let empty = self.empty_cache(1)?;
1008        while out.len() < MAX_STEPS {
1009            let (raw, _hidden) = self.decode_step(&tags, &enc, &mut cache, &empty)?;
1010            let tag = correct(raw, prev_ucel);
1011            if tag == END {
1012                break;
1013            }
1014            out.push(tag);
1015            tags.push(tag);
1016            prev_ucel = tag == UCEL;
1017        }
1018        Ok(out)
1019    }
1020
1021    /// Full structure prediction: OTSL grid cells with per-cell boxes (in the 448
1022    /// image, normalized cxcywh). Collects per-cell decoder hidden states using
1023    /// docling's exact bbox bookkeeping (skip-after-row-break, first-lcel of a
1024    /// horizontal span), runs the bbox decoder, merges span boxes, then lays the
1025    /// cells onto the OTSL grid with row/col spans.
1026    pub fn predict_table_structure(&mut self, img: &RgbImage) -> Result<Vec<TableCell>, String> {
1027        let enc = self.encode(img)?;
1028
1029        // The autoregressive loop's bbox bookkeeping lives in tf_core::BboxBook
1030        // (shared with the wasm path); this loop only steps the decoder.
1031        let mut book = BboxBook::new();
1032        let mut cache = DecodeCache::default();
1033        let empty = self.empty_cache(1)?;
1034        crate::timing::timed("tf.decode_loop", || -> Result<(), String> {
1035            while book.otsl.len() < MAX_STEPS {
1036                let (raw, hidden) = self.decode_step(&book.tags, &enc, &mut cache, &empty)?;
1037                if !book.step(raw, &hidden) {
1038                    break;
1039                }
1040            }
1041            Ok(())
1042        })?;
1043        self.finish_table(book, &enc.eo)
1044    }
1045
1046    /// The bbox stage after a table's decode loop: run the bbox decoder over
1047    /// the collected per-cell hidden states, merge span boxes, lay the cells
1048    /// onto the OTSL grid.
1049    fn finish_table(
1050        &mut self,
1051        mut book: BboxBook,
1052        eo: &DynValue,
1053    ) -> Result<Vec<TableCell>, String> {
1054        // docling#4592: a structure with no row break, or cut mid-row at the
1055        // step budget, is discarded with a warning (upstream logs at warning
1056        // level too). docling then keeps the text as one child of an empty
1057        // table; the port's empty result sends the region down the geometric
1058        // table path instead, which keeps the words *and* a grid
1059        // (docs/PDF_CONFORMANCE.md, "Degenerate TableFormer sequences").
1060        if let Some(reason) = book.degenerate_reason() {
1061            eprintln!(
1062                "docling-pdf: tableformer: discarding a predicted structure of {} tags because \
1063                 {reason}; geometric table fallback",
1064                book.otsl.len()
1065            );
1066            return Ok(Vec::new());
1067        }
1068        if book.n == 0 {
1069            return Ok(Vec::new());
1070        }
1071        // One run over all cells materializes [ncells, 784, 512] fp32
1072        // intermediates (~1.6 GB each at 1,000 cells). Cells are independent
1073        // (each box/class depends on its own `tag_h` row and `enc_out`), so
1074        // running them in chunks bounds the peak without changing the output.
1075        let hiddens = std::mem::take(&mut book.hiddens);
1076        let mut boxes: Vec<[f32; 4]> = Vec::with_capacity(book.n);
1077        let mut classes: Vec<i64> = Vec::with_capacity(book.n);
1078        for rows in bbox_chunks(&hiddens) {
1079            let tag_h = Tensor::from_array(([rows.len() / EMBED_DIM, EMBED_DIM], rows.to_vec()))
1080                .map_err(|e| format!("tableformer: tag_h: {e}"))?;
1081            let bout = crate::timing::timed("tf.bbox", || {
1082                docling_onnx::run(
1083                    &mut self.bbox,
1084                    ort::inputs!["enc_out" => eo, "tag_h" => tag_h],
1085                )
1086                .map_err(|e| format!("tableformer: bbox: {e}"))
1087            })?;
1088            let (_, raw) = bout["boxes"]
1089                .try_extract_tensor::<f32>()
1090                .map_err(|e| format!("tableformer: boxes: {e}"))?;
1091            boxes.extend(raw.chunks_exact(4).map(|c| [c[0], c[1], c[2], c[3]]));
1092            // Per-cell class logits [n, 3] → argmax (docling's `outputs_class`).
1093            let (_, craw) = bout["classes"]
1094                .try_extract_tensor::<f32>()
1095                .map_err(|e| format!("tableformer: classes: {e}"))?;
1096            classes.extend(craw.chunks_exact(3).map(|c| argmax(c) as i64));
1097        }
1098        let (merged, merged_classes) = merge_spans(&boxes, &classes, &book.merge);
1099        Ok(build_table_cells(&book.otsl, &merged, &merged_classes))
1100    }
1101
1102    /// Predict a table region's Markdown grid: crop the region (docling's
1103    /// page→1024px box-average then bbox crop), run the structure model, then
1104    /// match the page's word cells into the predicted cells with docling's
1105    /// matching post-processor ([`crate::tf_match`]) and expand spans into a
1106    /// dense `rows × cols` grid. `region` is `(l, t, r, b)` in page points
1107    /// (top-left). Returns `None` if no structure is predicted.
1108    pub fn predict_table_rows(
1109        &mut self,
1110        page_image: &RgbImage,
1111        region: [f32; 4],
1112        words: &[TextCell],
1113    ) -> Option<crate::tf_core::TableGrid> {
1114        let page1024 = Self::page_1024(page_image);
1115        self.predict_table_rows_on(page_image.height(), &page1024, region, words)
1116    }
1117
1118    /// The page rendered at 1024 px height (cv2.INTER_AREA), the frame every
1119    /// table crop of that page is cut from. Computed once per page by the
1120    /// pipeline and shared across its tables — the resample is a full-page
1121    /// f64 box filter, 110–170 ms on the corpus pages, and it used to run
1122    /// again for every table on the page.
1123    pub fn page_1024(page_image: &RgbImage) -> RgbImage {
1124        let sf = 1024.0 / page_image.height() as f32;
1125        let pw = (page_image.width() as f32 * sf) as u32;
1126        crate::timing::timed("tableformer.inter_area", || {
1127            crate::resample::inter_area(page_image, pw, 1024)
1128        })
1129    }
1130
1131    /// [`predict_table_rows`](Self::predict_table_rows) with the page's
1132    /// 1024-px frame already built ([`page_1024`](Self::page_1024));
1133    /// `page_h` is the source page image's pixel height.
1134    pub fn predict_table_rows_on(
1135        &mut self,
1136        page_h: u32,
1137        page1024: &RgbImage,
1138        region: [f32; 4],
1139        words: &[TextCell],
1140    ) -> Option<crate::tf_core::TableGrid> {
1141        let crop = Self::crop_region(page_h, page1024, region)?;
1142        let cells = crate::timing::timed("tableformer.structure", || {
1143            self.predict_table_structure(&crop)
1144        })
1145        .ok()?;
1146        if cells.is_empty() {
1147            return None;
1148        }
1149        // The ort-free tail (word matching + grid assembly) is shared with the
1150        // browser path in tf_core.
1151        crate::tf_core::table_rows(&cells, region, words)
1152    }
1153
1154    /// Every table of a page at once: [`predict_table_rows_on`](Self::predict_table_rows_on)
1155    /// per region, except that with the dynamic-batch decoder the tables'
1156    /// decode steps are shared — each table is encoded on its own, then one
1157    /// decode loop steps all of them together ([`Self::decode_batch`]), then
1158    /// each runs its own bbox head. Per table the result is bit-identical to
1159    /// the one-at-a-time path; a page with a single table takes exactly that
1160    /// path. Should the batched run fail (an ort error), the tables are
1161    /// retried one by one so a page never loses all of its tables to one
1162    /// shared step.
1163    pub fn predict_tables_on(
1164        &mut self,
1165        page_h: u32,
1166        page1024: &RgbImage,
1167        regions: &[[f32; 4]],
1168        words: &[TextCell],
1169    ) -> Vec<Option<crate::tf_core::TableGrid>> {
1170        let mut out: Vec<Option<crate::tf_core::TableGrid>> = vec![None; regions.len()];
1171        let crops: Vec<(usize, RgbImage)> = regions
1172            .iter()
1173            .enumerate()
1174            .filter_map(|(i, r)| Self::crop_region(page_h, page1024, *r).map(|c| (i, c)))
1175            .collect();
1176        if self.batched && crops.len() > 1 {
1177            let batched = crate::timing::timed("tableformer.structure", || {
1178                self.predict_structures_batched(crops.iter().map(|(_, c)| c))
1179            });
1180            match batched {
1181                Ok(cells) => {
1182                    for ((i, _), cells) in crops.iter().zip(cells) {
1183                        if !cells.is_empty() {
1184                            out[*i] = crate::tf_core::table_rows(&cells, regions[*i], words);
1185                        }
1186                    }
1187                    return out;
1188                }
1189                Err(e) => docling_core::debug_log!(
1190                    "docling-pdf: tableformer batched decode failed ({e}); decoding tables one by one"
1191                ),
1192            }
1193        }
1194        for (i, crop) in &crops {
1195            let cells = crate::timing::timed("tableformer.structure", || {
1196                self.predict_table_structure(crop)
1197            });
1198            if let Ok(cells) = cells {
1199                if !cells.is_empty() {
1200                    out[*i] = crate::tf_core::table_rows(&cells, regions[*i], words);
1201                }
1202            }
1203        }
1204        out
1205    }
1206
1207    /// [`predict_table_structure`](Self::predict_table_structure) for several
1208    /// crops with the decode steps shared across them.
1209    fn predict_structures_batched<'a>(
1210        &mut self,
1211        crops: impl Iterator<Item = &'a RgbImage>,
1212    ) -> Result<Vec<Vec<TableCell>>, String> {
1213        let mut encs = Vec::new();
1214        for crop in crops {
1215            encs.push(self.encode(crop)?);
1216        }
1217        let books = self.decode_batch(&encs)?;
1218        books
1219            .into_iter()
1220            .zip(&encs)
1221            .map(|(book, enc)| self.finish_table(book, &enc.eo))
1222            .collect()
1223    }
1224
1225    /// Crop the table bbox out of the 1024px frame. docling's coordinate
1226    /// chain, rounding included: the cluster bbox is rounded to integer page
1227    /// points *first* (`round(cluster.bbox.l) * scale`, banker's rounding),
1228    /// scaled by 2 (its table-structure page scale), then by `1024 / <2x
1229    /// page-image height>`, and the crop indices round again. Rounding after
1230    /// scaling instead shifts some crops by a pixel — enough to change
1231    /// TableFormer's cell boxes on tall tables (redp5110's TOC). `None` for a
1232    /// region that collapses to an empty crop.
1233    fn crop_region(page_h: u32, page1024: &RgbImage, region: [f32; 4]) -> Option<RgbImage> {
1234        let k = 2.0 * 1024.0 / page_h as f64;
1235        let px = |v: f32| (v as f64).round_ties_even() * k;
1236        let x = (px(region[0]).round_ties_even()).max(0.0) as u32;
1237        let y = (px(region[1]).round_ties_even()).max(0.0) as u32;
1238        let x2 = (px(region[2]).round_ties_even() as u32).min(page1024.width());
1239        let y2 = (px(region[3]).round_ties_even() as u32).min(page1024.height());
1240        if x2 <= x || y2 <= y {
1241            return None;
1242        }
1243        Some(image::imageops::crop_imm(page1024, x, y, x2 - x, y2 - y).to_image())
1244    }
1245}
1246
1247/// Note once per process that TableFormer's ONNX graphs weren't found, so tables
1248/// fall back to geometric reconstruction. The default paths are relative
1249/// (`.models/tableformer/*.onnx`), which only resolves when the process's current
1250/// directory happens to be the repo root — a very easy miss for anything else
1251/// (an embedding app, a binding invoked from a different working directory, …),
1252/// and previously failed with no signal at all.
1253fn warn_missing_once(enc: &str, dec: &str, bbx: &str) {
1254    static WARNED: std::sync::Once = std::sync::Once::new();
1255    WARNED.call_once(|| {
1256        eprintln!(
1257            "docling.rs: TableFormer models not found (checked {enc}, {dec}, {bbx}); \
1258             tables will use geometric reconstruction instead of ML table-structure \
1259             recognition. Set DOCLING_TABLEFORMER_ENCODER / DOCLING_TABLEFORMER_DECODER \
1260             / DOCLING_TABLEFORMER_BBOX to enable it (see README.md)."
1261        );
1262    });
1263}
1264
1265/// docling's preprocessing: bilinear (cv2.INTER_LINEAR) resize the crop to 448²,
1266/// normalize `(x/255 − mean)/std`, laid out as (C, W, H) — docling transposes
1267/// (2,1,0), so width is the major spatial axis. The page→1024px box-average
1268/// (cv2.INTER_AREA) is the caller's job.
1269fn preprocess(img: &RgbImage) -> Result<Tensor<f32>, String> {
1270    Tensor::from_array(([1usize, 3, SIDE, SIDE], preprocess_input(img)))
1271        .map_err(|e| format!("tableformer: input: {e}"))
1272}
1273
1274#[cfg(test)]
1275mod bbox_chunk_tests {
1276    use super::*;
1277
1278    #[test]
1279    fn chunks_cover_all_cells_in_order_on_cell_boundaries() {
1280        for n in [1usize, 255, 256, 257, 980, 1000] {
1281            let h: Vec<f32> = (0..n * EMBED_DIM).map(|i| i as f32).collect();
1282            let chunks: Vec<&[f32]> = bbox_chunks(&h).collect();
1283            assert!(chunks.iter().all(|c| c.len() % EMBED_DIM == 0));
1284            assert!(chunks
1285                .iter()
1286                .all(|c| c.len() <= BBOX_CHUNK_CELLS * EMBED_DIM));
1287            assert_eq!(chunks.concat(), h, "n={n}");
1288            assert_eq!(chunks.len(), n.div_ceil(BBOX_CHUNK_CELLS));
1289        }
1290    }
1291}