docling_pdf/tableformer.rs
1//! TableFormer: table-structure recovery via docling-ibm-models, exported to
2//! ONNX by `scripts/install/export_tableformer.py`. The image encoder + tag-transformer
3//! encoder run once to a memory tensor; the decoder is then stepped
4//! autoregressively to emit an OTSL structure-token sequence (the same model
5//! docling runs). See docs/PDF_CONFORMANCE.md.
6
7use crate::pdfium_backend::TextCell;
8// The ONNX-free half (preprocessing, structure corrections, bbox bookkeeping,
9// span merge, OTSL→grid) lives in tf_core so the browser build (#157 stage 3)
10// runs the same logic; this file owns the three `ort` sessions and the
11// owned-value KV-cache fast path.
12use crate::tf_core::{
13 argmax, build_table_cells, correct, merge_spans, preprocess_input, BboxBook, TableCell, END,
14 MAX_STEPS, START, UCEL,
15};
16use image::RgbImage;
17use ort::session::Session;
18use ort::value::{DynValue, Tensor, TensorRef};
19
20const SIDE: usize = crate::tf_core::SIDE as usize;
21const EMBED_DIM: usize = crate::tf_core::EMBED_DIM;
22/// Decoder geometry, fixed by the exported TableModel04_rs graph: the cached
23/// decoder threads a `[N_LAYERS, past, 1, EMBED_DIM]` per-layer state cache.
24const N_LAYERS: usize = 6;
25/// Cells per bbox-head run (see `finish_table`): bounds its broadcast
26/// intermediates to ~BBOX_CHUNK_CELLS × 784 × 512 × 4 bytes per tensor.
27/// 256 halves the peak on ~1,000-cell tables for <1% extra time (smaller
28/// chunks repeat the head's encoder-side work more often); output is
29/// byte-identical to the single run.
30const BBOX_CHUNK_CELLS: usize = 256;
31
32/// Split the flat `[n, EMBED_DIM]` hidden states into bbox-head runs of at
33/// most [`BBOX_CHUNK_CELLS`] whole cells.
34fn bbox_chunks(hiddens: &[f32]) -> impl Iterator<Item = &[f32]> {
35 hiddens.chunks(BBOX_CHUNK_CELLS * EMBED_DIM)
36}
37
38/// Resolve the encoder / decoder / bbox files exactly as [`TableFormer::load`]
39/// will (shared with `model_inventory`, so diagnostics can never drift from
40/// what actually loads). Explicit `DOCLING_TABLEFORMER_*` overrides win; the
41/// decoder otherwise picks by preference — INT8 variants first unless
42/// `DOCLING_RS_FP32` opts out, and within a precision the true-KV-cache
43/// export (`decoder_kv*`, one token per step, O(past) step cost) ranks ahead
44/// of the legacy layer-output-cache graph it matches byte-for-byte (91/91
45/// snapshot corpus exact with either; the KV graph re-measured ~13–17% faster
46/// warm, so speed wins the default and the legacy file stays as the smaller
47/// fallback). `decoder_kv` ranks ABOVE `decoder_int8`: the #97 hoisted fp32
48/// KV graph is faster than the quantized legacy graph on every machine
49/// measured, and it is byte-exact (its own int8 variant is not produced — see
50/// quantize_models.py). `decoder_kv_layers` ranks just above `decoder_kv`:
51/// the same graph with per-layer caches (#871, [`DecoderStyle::KvLayered`]),
52/// bit-identical and without per-step allocations that grow with the table.
53pub fn resolved_paths() -> (String, String, String) {
54 // The encoder ranks its fp16-weight repack (`encoder_fp16.onnx`, #374 —
55 // the same graph with the weights stored as fp16 and cast back to fp32
56 // at load, ~half the download, fp32 compute) ahead of the fp32 file
57 // unless `DOCLING_RS_FP32` opts out; an explicit override wins.
58 let enc = docling_core::env::nonempty("DOCLING_TABLEFORMER_ENCODER").unwrap_or_else(|| {
59 let candidates: &[&str] = if crate::prefer_fp32() {
60 &[".models/tableformer/encoder.onnx"]
61 } else {
62 &[
63 ".models/tableformer/encoder_fp16.onnx",
64 ".models/tableformer/encoder.onnx",
65 ]
66 };
67 candidates
68 .iter()
69 .map(|p| crate::resolve_asset(p))
70 .find(|p| std::path::Path::new(p).exists())
71 .unwrap_or_else(|| crate::resolve_asset(".models/tableformer/encoder.onnx"))
72 });
73 let dec = docling_core::env::nonempty("DOCLING_TABLEFORMER_DECODER").unwrap_or_else(|| {
74 let candidates: &[&str] = if crate::prefer_fp32() {
75 &[
76 ".models/tableformer/decoder_kv_layers.onnx",
77 ".models/tableformer/decoder_kv.onnx",
78 ".models/tableformer/decoder.onnx",
79 ]
80 } else {
81 &[
82 ".models/tableformer/decoder_kv_layers.onnx",
83 ".models/tableformer/decoder_kv.onnx",
84 ".models/tableformer/decoder_int8.onnx",
85 ".models/tableformer/decoder.onnx",
86 ]
87 };
88 candidates
89 .iter()
90 .map(|p| crate::resolve_asset(p))
91 .find(|p| std::path::Path::new(p).exists())
92 .unwrap_or_else(|| ".models/tableformer/decoder.onnx".to_string())
93 });
94 let bbx = docling_core::env::nonempty("DOCLING_TABLEFORMER_BBOX")
95 .unwrap_or_else(|| crate::resolve_asset(".models/tableformer/bbox.onnx"));
96 (enc, dec, bbx)
97}
98
99pub struct TableFormer {
100 encoder: Session,
101 decoder: Session,
102 bbox: Session,
103 /// Which decoder graph flavour is loaded, detected from the session's
104 /// input names (so an explicit `DOCLING_TABLEFORMER_DECODER` override
105 /// works with any of them).
106 style: DecoderStyle,
107 /// The `KvHoisted` decoder's `tag` input has a symbolic batch axis (the
108 /// dynamic-batch `decoder_kv.onnx` export): a page's tables decode
109 /// together, one step for all of them — see [`Self::predict_tables_on`].
110 /// The older fixed-`[1,1]` export decodes the tables one after another.
111 batched: bool,
112 /// The `KvLayered` loop runs through an `IoBinding` that keeps the caches
113 /// and the cross tensors where the provider computes ([`DeviceCache`]):
114 /// on under a CUDA-class provider (CUDA, TensorRT); DirectML only with
115 /// `DOCLING_RS_TF_IOBINDING=1`, which forces it either way (also on CPU,
116 /// which is how it is tested without a GPU; `0` turns it off). Cleared — for good — when a
117 /// table's first bound step fails, so the host-buffer loop takes over.
118 iobinding: bool,
119 /// The decoder session was built without the GPU provider (DirectML):
120 /// its tensors live on the host whatever the other sessions use.
121 decoder_on_host: bool,
122}
123
124/// The three decoder-graph generations the loop supports.
125#[derive(Clone, Copy, PartialEq, Eq)]
126enum DecoderStyle {
127 /// `decoder.onnx`: layer-output cache; feeds the full `tags` prefix and a
128 /// single `cache` every step.
129 Legacy,
130 /// The pre-#97 `decoder_kv.onnx`: one tag per step, `cache_k`/`cache_v`,
131 /// with the stacked `cross_k`/`cross_v` re-split inside every step.
132 KvStacked,
133 /// The #97 `decoder_kv.onnx`: one tag per step, and the constant cross
134 /// tensors arrive as 2×`N_LAYERS` per-layer inputs (`cross_kt_i` already
135 /// transposed for q·Kᵀ, `cross_v_i`), computed once per table by the
136 /// encoder — the step graph does no work proportional to their size.
137 KvHoisted,
138 /// `decoder_kv_layers.onnx` (#871, `scripts/install/kv_layers.py`): the
139 /// `KvHoisted` graph with the self-attention cache per layer — `Kᵀ` and
140 /// `V` in and out for each layer instead of two stacked tensors — whose
141 /// outputs land in buffers the loop owns ([`LayerCache`]).
142 KvLayered,
143}
144
145impl DecoderStyle {
146 /// One tag per step with the per-layer cross tensors the encoder emits.
147 fn hoisted(self) -> bool {
148 matches!(self, DecoderStyle::KvHoisted | DecoderStyle::KvLayered)
149 }
150}
151
152/// KV-cache geometry fixed by the `decoder_kv.onnx` export
153/// (`[N_LAYERS, 1, KV_HEADS, past, KV_HEAD_DIM]`, `KV_HEADS × KV_HEAD_DIM = EMBED_DIM`).
154const KV_HEADS: usize = 8;
155const KV_HEAD_DIM: usize = 64;
156
157/// The autoregressive decode state: `a` is the legacy layer-output cache, or
158/// `cache_k` for the KV graph; `b` is `cache_v` (KV graph only). `None` = first
159/// step (the zero-`past` empties are allocated per table by [`TableFormer::empty_cache`]).
160#[derive(Default)]
161struct DecodeCache {
162 a: Option<DynValue>,
163 b: Option<DynValue>,
164 /// The `KvLayered` caches, set up on the first step.
165 layers: Option<LayerCache>,
166 /// The `KvLayered` caches on the device, when the loop binds them there.
167 device: Option<DeviceCache>,
168}
169
170/// The `KvLayered` decode under a GPU provider (CUDA, TensorRT, DirectML):
171/// an `IoBinding` holding the
172/// step's inputs where the provider computes. With plain runs, every step
173/// handed ONNX Runtime host tensors to copy to the device and got host
174/// tensors back — the 12 constant cross tensors (~19 MB per table, the same
175/// every step) and both directions of the KV cache (~4 MB each way at 160
176/// tokens, growing) — so on CUDA a step cost 12.6 ms against the ~3 ms its
177/// 49 small GEMMs take. Here the cross tensors are bound once per decode
178/// (one copy), the cache outputs stay in device memory and are bound as the
179/// next step's inputs, and only `logits`/`hidden` (kilobytes) come back.
180/// Same graph, same kernels: the output is the plain loop's.
181struct DeviceCache {
182 binding: ort::session::IoBinding,
183 rows: usize,
184 past: usize,
185 /// The last step's cache outputs, where the provider wrote them.
186 cache: Vec<DynValue>,
187 /// Zero-`past` first-step inputs.
188 empty: Vec<Tensor<f32>>,
189 /// Where the cache outputs are bound: the GPU provider's device memory,
190 /// or the CPU when `DOCLING_RS_TF_IOBINDING` forces the loop without one.
191 device: ort::memory::MemoryInfo<'static>,
192}
193
194/// The `KvLayered` decoder's self-attention caches, in memory docling owns
195/// (#871). With ONNX Runtime allocating them, every step of a table made a
196/// cache one token longer than the last and freed the previous one —
197/// hundreds of ever-larger blocks per table, which on linux/arm64 with the
198/// CPU arena off fragmented glibc's heap until a 6 GiB container ran out of
199/// memory. Here each step reads its caches from one of two buffers and ONNX
200/// Runtime writes the one-token-longer caches straight into the other (the
201/// outputs are preallocated over it, [`docling_onnx::borrowed_f32`]); the
202/// next step swaps them. Each buffer holds the `2 × N_LAYERS` tensors in
203/// regions of `cap` floats — per layer `Kᵀ` `[rows, H, hd, past]`, then `V`
204/// `[rows, H, past, hd]` — sized for [`MAX_STEPS`] tokens: zeroed (`calloc`,
205/// so mapped lazily), only the prefix a table reaches becomes resident, and
206/// they are released when the table (or the page's batch) is done.
207struct LayerCache {
208 rows: usize,
209 past: usize,
210 cap: usize,
211 bufs: [Vec<f32>; 2],
212 /// Which of `bufs` holds the current caches.
213 cur: usize,
214 /// Zero-`past` first-step inputs (ort's slice constructors reject a
215 /// 0-length dim, the session allocator does not).
216 empty: Vec<Tensor<f32>>,
217}
218
219impl LayerCache {
220 fn new(alloc: &ort::memory::Allocator, rows: usize) -> Result<Self, String> {
221 let cap = rows * KV_HEADS * MAX_STEPS * KV_HEAD_DIM;
222 let mut empty = Vec::with_capacity(2 * N_LAYERS);
223 for j in 0..2 * N_LAYERS {
224 empty.push(
225 Tensor::<f32>::new(alloc, Self::shape(j, rows, 0))
226 .map_err(|e| format!("tableformer: empty kv cache: {e}"))?,
227 );
228 }
229 Ok(Self {
230 rows,
231 past: 0,
232 cap,
233 bufs: [vec![0.0; 2 * N_LAYERS * cap], vec![0.0; 2 * N_LAYERS * cap]],
234 cur: 0,
235 empty,
236 })
237 }
238
239 /// Tensor `j`'s shape at `past` tokens: `Kᵀ` for the first `N_LAYERS`.
240 fn shape(j: usize, rows: usize, past: usize) -> [usize; 4] {
241 if j < N_LAYERS {
242 [rows, KV_HEADS, KV_HEAD_DIM, past]
243 } else {
244 [rows, KV_HEADS, past, KV_HEAD_DIM]
245 }
246 }
247
248 fn input_name(j: usize) -> String {
249 if j < N_LAYERS {
250 format!("cache_kt_{j}")
251 } else {
252 format!("cache_v_{}", j - N_LAYERS)
253 }
254 }
255}
256
257/// Zero-`past` first-step cache tensors: `(cache, None)` for the legacy graph,
258/// `(cache_k, Some(cache_v))` for the KV graph.
259type EmptyCache = (Tensor<f32>, Option<Tensor<f32>>);
260
261/// Encoder outputs that drive the cached decode loop: the per-layer cross-attention
262/// K/V (projected from the image memory once, constant across decode steps) and
263/// `enc_out` for the bbox decoder. Kept as owned `ort` values so each decode step
264/// (and the bbox run) borrows them directly — no per-step extract/copy/re-wrap.
265struct EncodeOut {
266 /// Stacked `[N_LAYERS,1,H,S,hd]` cross K/V — the `Legacy`/`KvStacked`
267 /// decoders' inputs. `None` for `KvHoisted`, which reads the per-layer
268 /// tensors instead: the stacked pair is 2×9.6 MB per table, and a page's
269 /// tables are now all held encoded at once for the batched loop.
270 ck: Option<DynValue>,
271 cv: Option<DynValue>,
272 eo: DynValue,
273 /// `KvHoisted` only: per-layer `[cross_kt_0..N, cross_v_0..N]`, index-aligned
274 /// with the decoder's input names, borrowed by every decode step.
275 per_layer: Vec<(String, DynValue)>,
276}
277
278impl TableFormer {
279 /// Load the exported encoder/decoder/bbox ONNX graphs (env overrides, else
280 /// `.models/tableformer/{encoder,decoder,bbox}.onnx`). Returns `None` if any is
281 /// absent, so the pipeline falls back to geometric reconstruction.
282 pub fn load() -> Option<Self> {
283 Self::load_with(crate::intra_threads())
284 }
285
286 /// Like [`load`](Self::load) but with an explicit intra-op thread count, so a
287 /// parallel page-worker pool can run each table model on fewer threads (the
288 /// throughput comes from running pages concurrently, not from one fat model).
289 ///
290 /// See [`resolved_paths`] for the encoder/decoder/bbox file selection.
291 pub fn load_with(intra: usize) -> Option<Self> {
292 // (resolution shared with the model inventory — see resolved_paths)
293 let (enc, dec, bbx) = resolved_paths();
294 if crate::timing::enabled() {
295 eprintln!("docling-pdf: tableformer decoder: {dec}");
296 }
297 if [&enc, &dec, &bbx]
298 .iter()
299 .any(|p| !std::path::Path::new(p).exists())
300 {
301 // The geometric fallback is a supported, intentional configuration
302 // (docling has no ML table-structure equivalent baked in either), so
303 // this stays a single quiet stderr note rather than an error — but it
304 // fires every process (not per-worker) so a CWD-relative default that
305 // silently misses its files (a very easy mistake for anything not run
306 // from the repo root, e.g. an embedding app) is at least visible once.
307 warn_missing_once(&enc, &dec, &bbx);
308 return None;
309 }
310 // The decoder's KV-cache grows by one entry every autoregressive step, so
311 // its input shapes differ on every `run()` call. ONNX Runtime's memory
312 // pattern optimizer assumes stable shapes to plan buffer reuse; disabling
313 // it for this session avoids repeatedly re-validating/re-touching that
314 // plan (and the external-weights file) on each step. The bbox head has
315 // the same problem one level up: its `tag_h` input is `[ncells, 512]`
316 // and every table has a different cell count, so with the pattern
317 // planner on each run re-plans — and on this graph the plan is *worse*
318 // than none: 290 ms vs 54 ms for a 100-cell table, 560 vs 94 ms for
319 // 200 cells (ORT 1.22, 4 threads). It was 0.26 s per table on the
320 // corpus, more than the encoder.
321 //
322 // The decoder runs on ONE intra-op thread. A step is 49 small GEMMs
323 // over a single token — it streams the layer weights, it does not
324 // compute — so extra threads only add synchronisation: measured 4.1 ms
325 // per step on 1 thread vs 5.5 on 4 (7.1 vs 4.9 once the cache is 100+
326 // long). In the pool it also stops a table decode from taking all the
327 // cores away from the other workers' layout inference. And a
328 // single-thread session has a fixed reduction order, so table
329 // structure no longer varies run-to-run on near-tie tokens the way
330 // multi-threaded float sums let it (the conformance scripts pin one
331 // thread for exactly that reason; the default now matches them). The
332 // encoder keeps the shared budget: one 448×448 CNN + transformer pass
333 // per table, 680 ms single-threaded vs 165 on four.
334 // The decoder runs on the CPU under DirectML. As exported, every
335 // decoder graph fails inside the provider (`Reshape node_view`,
336 // 0x8007023E, on both adapters of the first DirectML machine that
337 // tried), and every table fell back to geometric structure: DirectML
338 // rejects the `allowzero=1` torch's exporter writes on each `view`
339 // Reshape. With it cleared — equivalent here, no target shape holds a
340 // 0 — the decoder runs there, but slower than on the CPU: 36.9 ms per
341 // step against 13.9 on the same laptop (2203.01017v2, 141 steps). A
342 // step is ~150 tiny ops over one token, kernel launches dominate, so
343 // the encoder and the bbox head stay on the GPU and only the decoder
344 // moves.
345 let decoder_on_host = docling_onnx::is_directml();
346 let build = |path: &str,
347 mem_pattern: bool,
348 threads: usize,
349 host: bool|
350 -> Result<Session, String> {
351 let builder = docling_onnx::session_builder()?
352 .with_intra_threads(threads)
353 .map_err(|e| e.to_string())?
354 .with_memory_pattern(mem_pattern)
355 .map_err(|e| e.to_string())?;
356 let variant = if mem_pattern {
357 "mem_pattern"
358 } else {
359 "no_mem_pattern"
360 };
361 let builder = if host {
362 docling_onnx::apply_host(builder)?
363 } else {
364 docling_onnx::apply(builder)?
365 };
366 docling_onnx::commit(builder, path, variant)
367 .map_err(|e| format!("tableformer load {path}: {e}"))
368 };
369 match (
370 build(&enc, true, intra, false),
371 build(&dec, false, 1, decoder_on_host),
372 build(&bbx, false, intra, false),
373 ) {
374 (Ok(encoder), Ok(decoder), Ok(bbox)) => {
375 let has = |n: &str| decoder.inputs().iter().any(|i| i.name() == n);
376 let style = if has("cache_kt_0") {
377 DecoderStyle::KvLayered
378 } else if has("cross_kt_0") {
379 DecoderStyle::KvHoisted
380 } else if has("cache_k") {
381 DecoderStyle::KvStacked
382 } else {
383 DecoderStyle::Legacy
384 };
385 if style.hoisted() && !encoder.outputs().iter().any(|o| o.name() == "cross_kt_0") {
386 eprintln!(
387 "docling-pdf: tableformer decoder needs per-layer cross tensors \
388 (cross_kt_*) the encoder doesn't emit — re-download or re-export \
389 the model set (scripts/install/export_tableformer.py); \
390 falling back to geometric tables"
391 );
392 return None;
393 }
394 // Dynamic batch axis on `tag` ⇒ the export batches decode
395 // steps across tables (ort reports a symbolic dim as -1).
396 let batched = style.hoisted()
397 && decoder.inputs().iter().any(|i| {
398 i.name() == "tag"
399 && matches!(i.dtype(), ort::value::ValueType::Tensor { shape, .. }
400 if shape.first().is_some_and(|d| *d < 0))
401 });
402 if crate::timing::enabled() && batched {
403 eprintln!("docling-pdf: tableformer decoder batches a page's tables per step");
404 }
405 if crate::timing::enabled() && decoder_on_host {
406 eprintln!("docling-pdf: tableformer decoder runs on the CPU (DirectML)");
407 }
408 let iobinding = style == DecoderStyle::KvLayered
409 && match docling_core::env::nonempty("DOCLING_RS_TF_IOBINDING") {
410 Some(_) => docling_core::env::flag("DOCLING_RS_TF_IOBINDING"),
411 // Not when the decoder was kept on the host
412 // (DirectML, above): nothing to keep on a device.
413 None => !decoder_on_host && docling_onnx::device_memory_info().is_some(),
414 };
415 if crate::timing::enabled() && iobinding {
416 eprintln!("docling-pdf: tableformer decoder keeps its caches on the device");
417 }
418 Some(Self {
419 encoder,
420 decoder,
421 bbox,
422 style,
423 batched,
424 iobinding,
425 decoder_on_host,
426 })
427 }
428 _ => None,
429 }
430 }
431
432 /// Run the image encoder and capture what the cached decoder loop needs: each
433 /// decoder layer's cross-attention K/V (projected from the image memory once,
434 /// shape `[N_LAYERS,1,H,S,head_dim]`) and `enc_out` for the bbox decoder.
435 fn encode(&mut self, img: &RgbImage) -> Result<EncodeOut, String> {
436 let input = crate::timing::timed("tf.preprocess", || preprocess(img))?;
437 let mut enc_out = crate::timing::timed("tf.encoder", || {
438 docling_onnx::run(&mut self.encoder, ort::inputs!["image" => input])
439 .map_err(|e| format!("tableformer: encode: {e}"))
440 })?;
441 let mut per_layer = Vec::new();
442 if self.style.hoisted() {
443 for prefix in ["cross_kt_", "cross_v_"] {
444 for i in 0.. {
445 let name = format!("{prefix}{i}");
446 match enc_out.remove(&name) {
447 Some(v) => per_layer.push((name, v)),
448 None => break,
449 }
450 }
451 }
452 if per_layer.is_empty() {
453 return Err("tableformer: encoder emitted no cross_kt_* outputs".into());
454 }
455 }
456 let mut grab = |name: &str| -> Result<DynValue, String> {
457 enc_out
458 .remove(name)
459 .ok_or_else(|| format!("tableformer: encoder output {name} missing"))
460 };
461 let hoisted = self.style.hoisted();
462 Ok(EncodeOut {
463 ck: if hoisted {
464 None
465 } else {
466 Some(grab("cross_k")?)
467 },
468 cv: if hoisted {
469 None
470 } else {
471 Some(grab("cross_v")?)
472 },
473 eo: grab("enc_out")?,
474 per_layer,
475 })
476 }
477
478 /// One doubly-cached decode step: feed the current `tags`, the constant cross
479 /// K/V, and the growing self-attention `cache`; return the raw argmax tag and
480 /// the last token's hidden state, advancing the cache. The cache stays an owned
481 /// `ort` value — the previous step's `out_cache` output is fed back directly,
482 /// never extracted or copied (it grows every step, so per-step copies were
483 /// O(steps²) float traffic). `empty_cache` is the zero-`past` value used on the
484 /// first step (ort's array constructors reject a 0-length dim, so it is
485 /// allocated through the session allocator by the caller).
486 fn decode_step(
487 &mut self,
488 tags: &[i64],
489 enc: &EncodeOut,
490 cache: &mut DecodeCache,
491 empty: &EmptyCache,
492 ) -> Result<(i64, Vec<f32>), String> {
493 crate::timing::timed("tf.decode_step", || {
494 self.decode_step_inner(tags, enc, cache, empty)
495 })
496 }
497
498 fn decode_step_inner(
499 &mut self,
500 tags: &[i64],
501 enc: &EncodeOut,
502 cache: &mut DecodeCache,
503 empty: &EmptyCache,
504 ) -> Result<(i64, Vec<f32>), String> {
505 if self.style.hoisted() {
506 // #97 graph: one tag; the constant per-layer cross tensors are
507 // borrowed views — the step pays nothing proportional to them.
508 let last = *tags.last().expect("decode starts from <start>");
509 let (raws, hidden) = self.step_kv(&[last], &enc.per_layer, cache, empty)?;
510 return Ok((raws[0], hidden));
511 }
512 let (ck, cv) = match (enc.ck.as_ref(), enc.cv.as_ref()) {
513 (Some(k), Some(v)) => (k, v),
514 _ => return Err("tableformer: stacked cross K/V missing".into()),
515 };
516 let mut dout = match self.style {
517 DecoderStyle::KvHoisted | DecoderStyle::KvLayered => unreachable!("handled above"),
518 DecoderStyle::KvStacked => {
519 // Pre-#97 KV graph: feed only the newly emitted tag; the projected
520 // K/V for the whole prefix live in cache_k/cache_v and are fed
521 // back as-is.
522 let last = *tags.last().expect("decode starts from <start>");
523 let tag_t = Tensor::from_array(([1usize, 1usize], vec![last]))
524 .map_err(|e| format!("tableformer: tag: {e}"))?;
525 match (cache.a.as_ref(), cache.b.as_ref()) {
526 (Some(k), Some(v)) => docling_onnx::run(
527 &mut self.decoder,
528 ort::inputs![
529 "tag" => tag_t, "cross_k" => ck, "cross_v" => cv,
530 "cache_k" => k, "cache_v" => v],
531 ),
532 _ => docling_onnx::run(
533 &mut self.decoder,
534 ort::inputs![
535 "tag" => tag_t, "cross_k" => ck, "cross_v" => cv,
536 "cache_k" => &empty.0,
537 "cache_v" => empty.1.as_ref().expect("kv empty cache has both halves")],
538 ),
539 }
540 }
541 DecoderStyle::Legacy => {
542 let tags_t = Tensor::from_array(([tags.len(), 1usize], tags.to_vec()))
543 .map_err(|e| format!("tableformer: tags: {e}"))?;
544 match cache.a.as_ref() {
545 None => docling_onnx::run(
546 &mut self.decoder,
547 ort::inputs![
548 "tags" => tags_t, "cross_k" => ck, "cross_v" => cv,
549 "cache" => &empty.0],
550 ),
551 Some(c) => docling_onnx::run(
552 &mut self.decoder,
553 ort::inputs![
554 "tags" => tags_t, "cross_k" => ck, "cross_v" => cv,
555 "cache" => c],
556 ),
557 }
558 }
559 }
560 .map_err(|e| format!("tableformer: decode: {e}"))?;
561 let (_, logits) = dout["logits"]
562 .try_extract_tensor::<f32>()
563 .map_err(|e| format!("tableformer: logits: {e}"))?;
564 let raw = argmax(logits) as i64;
565 let (_, hidden) = dout["hidden"]
566 .try_extract_tensor::<f32>()
567 .map_err(|e| format!("tableformer: hidden: {e}"))?;
568 let hidden = hidden.to_vec();
569 if self.style != DecoderStyle::Legacy {
570 cache.a = Some(
571 dout.remove("out_cache_k")
572 .ok_or_else(|| "tableformer: out_cache_k missing".to_string())?,
573 );
574 cache.b = Some(
575 dout.remove("out_cache_v")
576 .ok_or_else(|| "tableformer: out_cache_v missing".to_string())?,
577 );
578 } else {
579 cache.a = Some(
580 dout.remove("out_cache")
581 .ok_or_else(|| "tableformer: decoder output out_cache missing".to_string())?,
582 );
583 }
584 Ok((raw, hidden))
585 }
586
587 /// One step of a one-tag-per-step graph over `tags.len()` rows, through
588 /// whichever cache layout the loaded decoder has.
589 fn step_kv(
590 &mut self,
591 tags: &[i64],
592 per_layer: &[(String, DynValue)],
593 cache: &mut DecodeCache,
594 empty: &EmptyCache,
595 ) -> Result<(Vec<i64>, Vec<f32>), String> {
596 if self.style == DecoderStyle::KvLayered && self.iobinding {
597 match self.step_kv_device(tags, per_layer, cache) {
598 Ok(step) => return Ok(step),
599 // A provider that will not hand out its device memory to a
600 // binding (the device-resident path is only verified on CUDA
601 // and on CPU) fails on a decode's first step, before any
602 // state lives there: decode on with host caches, here and for
603 // every later table. A later failure is a real one.
604 Err(e) if cache.device.as_ref().is_none_or(|d| d.past == 0) => {
605 eprintln!(
606 "docling-pdf: tableformer: device-resident decode unavailable ({e}); \
607 decoding with host caches"
608 );
609 self.iobinding = false;
610 cache.device = None;
611 }
612 Err(e) => return Err(e),
613 }
614 }
615 if self.style == DecoderStyle::KvLayered {
616 self.step_kv_layered(tags, per_layer, cache)
617 } else {
618 self.step_kv_hoisted(tags, per_layer, cache, empty)
619 }
620 }
621
622 /// One `KvLayered` step through the decode's [`DeviceCache`] binding.
623 fn step_kv_device(
624 &mut self,
625 tags: &[i64],
626 per_layer: &[(String, DynValue)],
627 cache: &mut DecodeCache,
628 ) -> Result<(Vec<i64>, Vec<f32>), String> {
629 let rows = tags.len();
630 let err = |what: &str, e: ort::Error| format!("tableformer: {what}: {e}");
631 let dc = match &mut cache.device {
632 Some(dc) => dc,
633 None => {
634 let mut binding = self
635 .decoder
636 .create_binding()
637 .map_err(|e| err("io binding", e))?;
638 // The constant cross tensors: copied to the device once here,
639 // and every step of this decode reads that copy.
640 for (name, v) in per_layer {
641 binding
642 .bind_input(name.as_str(), v)
643 .map_err(|e| err(name, e))?;
644 }
645 binding
646 .synchronize_inputs()
647 .map_err(|e| err("cross inputs", e))?;
648 let alloc = self.decoder.allocator();
649 let mut empty = Vec::with_capacity(2 * N_LAYERS);
650 for j in 0..2 * N_LAYERS {
651 empty.push(
652 Tensor::<f32>::new(alloc, LayerCache::shape(j, rows, 0))
653 .map_err(|e| err("empty kv cache", e))?,
654 );
655 }
656 cache.device.insert(DeviceCache {
657 binding,
658 rows,
659 past: 0,
660 cache: Vec::new(),
661 empty,
662 device: if self.decoder_on_host {
663 ort::memory::MemoryInfo::default()
664 } else {
665 docling_onnx::device_memory_info().unwrap_or_default()
666 },
667 })
668 }
669 };
670 if dc.rows != rows || dc.past >= MAX_STEPS {
671 return Err(format!(
672 "tableformer: kv cache for {} rows at {} tokens, asked for {rows} rows one more",
673 dc.rows, dc.past
674 ));
675 }
676 let tag_t =
677 Tensor::from_array(([rows, 1usize], tags.to_vec())).map_err(|e| err("tag", e))?;
678 dc.binding
679 .bind_input("tag", &tag_t)
680 .map_err(|e| err("tag", e))?;
681 for j in 0..2 * N_LAYERS {
682 let name = LayerCache::input_name(j);
683 if dc.past == 0 {
684 dc.binding.bind_input(name.as_str(), &dc.empty[j])
685 } else {
686 dc.binding.bind_input(name.as_str(), &dc.cache[j])
687 }
688 .map_err(|e| err(&name, e))?;
689 // Re-bound every step: the output is one token longer each time,
690 // so ONNX Runtime must allocate it afresh rather than reuse the
691 // previous step's value.
692 let out = format!("out_{name}");
693 dc.binding
694 .bind_output_to_device(out.as_str(), &dc.device)
695 .map_err(|e| err(&out, e))?;
696 }
697 let host = ort::memory::MemoryInfo::default();
698 for name in ["logits", "hidden"] {
699 dc.binding
700 .bind_output_to_device(name, &host)
701 .map_err(|e| err(name, e))?;
702 }
703 // The CUDA arena is shrunk at a decode's first step only (#842's run
704 // option): shrinking after every step handed the step's working
705 // memory back to the driver and allocated it again on the next one —
706 // a cudaFree + cudaMalloc pair per token. Once per decode still
707 // returns the previous table's chunks before this one grows.
708 let shrink = if dc.past == 0 {
709 docling_onnx::run_options()
710 } else {
711 None
712 };
713 let mut dout = match shrink {
714 Some(opts) => self.decoder.run_binding_with_options(&dc.binding, opts),
715 None => self.decoder.run_binding(&dc.binding),
716 }
717 .map_err(|e| err("decode", e))?;
718 let (_, logits) = dout["logits"]
719 .try_extract_tensor::<f32>()
720 .map_err(|e| err("logits", e))?;
721 let vocab = logits.len() / rows;
722 let raws: Vec<i64> = logits
723 .chunks_exact(vocab)
724 .map(|row| argmax(row) as i64)
725 .collect();
726 let (_, hidden) = dout["hidden"]
727 .try_extract_tensor::<f32>()
728 .map_err(|e| err("hidden", e))?;
729 let hidden = hidden.to_vec();
730 let mut next = Vec::with_capacity(2 * N_LAYERS);
731 for j in 0..2 * N_LAYERS {
732 let out = format!("out_{}", LayerCache::input_name(j));
733 next.push(
734 dout.remove(out.as_str())
735 .ok_or_else(|| format!("tableformer: {out} missing"))?,
736 );
737 }
738 drop(dout);
739 dc.cache = next;
740 dc.past += 1;
741 Ok((raws, hidden))
742 }
743
744 /// One `KvLayered` step: [`Self::step_kv_hoisted`] with the caches read
745 /// from and written into the [`LayerCache`] buffers — ONNX Runtime
746 /// allocates nothing that grows with the table (#871).
747 fn step_kv_layered(
748 &mut self,
749 tags: &[i64],
750 per_layer: &[(String, DynValue)],
751 cache: &mut DecodeCache,
752 ) -> Result<(Vec<i64>, Vec<f32>), String> {
753 let rows = tags.len();
754 let lc = match &mut cache.layers {
755 Some(lc) => lc,
756 None => cache
757 .layers
758 .insert(LayerCache::new(self.decoder.allocator(), rows)?),
759 };
760 if lc.rows != rows || lc.past >= MAX_STEPS {
761 return Err(format!(
762 "tableformer: kv cache for {} rows at {} tokens, asked for {rows} rows one more",
763 lc.rows, lc.past
764 ));
765 }
766 let (past, cap) = (lc.past, lc.cap);
767 let tag_t = Tensor::from_array(([rows, 1usize], tags.to_vec()))
768 .map_err(|e| format!("tableformer: tag: {e}"))?;
769 let [first, second] = &mut lc.bufs;
770 let (src, dst) = if lc.cur == 0 {
771 (&*first, second)
772 } else {
773 (&*second, first)
774 };
775 let mut inputs: Vec<(
776 std::borrow::Cow<'_, str>,
777 ort::session::SessionInputValue<'_>,
778 )> = Vec::with_capacity(1 + 2 * N_LAYERS + per_layer.len());
779 inputs.push(("tag".into(), tag_t.into()));
780 let in_len = rows * KV_HEADS * past * KV_HEAD_DIM;
781 for j in 0..2 * N_LAYERS {
782 let name = LayerCache::input_name(j);
783 if past == 0 {
784 inputs.push((name.into(), (&lc.empty[j]).into()));
785 } else {
786 let view = TensorRef::from_array_view((
787 LayerCache::shape(j, rows, past),
788 &src[j * cap..j * cap + in_len],
789 ))
790 .map_err(|e| format!("tableformer: {name}: {e}"))?;
791 inputs.push((name.into(), view.into()));
792 }
793 }
794 for (name, v) in per_layer {
795 inputs.push((name.as_str().into(), v.into()));
796 }
797 let alloc = self.decoder.allocator();
798 let mut outputs = ort::session::OutputSelector::default();
799 for (j, region) in dst.chunks_exact_mut(cap).enumerate() {
800 // SAFETY: `dst` is borrowed mutably for this whole step and
801 // nothing else touches it; the values over it live in `opts` and
802 // `dout`, both dropped at the end of this function, before the
803 // next step reads the buffer (or the cache frees it).
804 let out = unsafe {
805 docling_onnx::borrowed_f32(region, &LayerCache::shape(j, rows, past + 1), alloc)
806 }
807 .map_err(|e| format!("tableformer: kv cache output: {e}"))?;
808 outputs = outputs.preallocate(format!("out_{}", LayerCache::input_name(j)), out);
809 }
810 let opts = docling_onnx::run_options_with_outputs(outputs)?;
811 let dout = self
812 .decoder
813 .run_with_options(inputs, &opts)
814 .map_err(|e| format!("tableformer: decode: {e}"))?;
815 let (_, logits) = dout["logits"]
816 .try_extract_tensor::<f32>()
817 .map_err(|e| format!("tableformer: logits: {e}"))?;
818 let vocab = logits.len() / rows;
819 let raws: Vec<i64> = logits
820 .chunks_exact(vocab)
821 .map(|row| argmax(row) as i64)
822 .collect();
823 let (_, hidden) = dout["hidden"]
824 .try_extract_tensor::<f32>()
825 .map_err(|e| format!("tableformer: hidden: {e}"))?;
826 let hidden = hidden.to_vec();
827 drop(dout);
828 drop(opts);
829 lc.past += 1;
830 lc.cur ^= 1;
831 Ok((raws, hidden))
832 }
833
834 /// One `KvHoisted` step over `tags.len()` rows — one table per row. `tags`
835 /// holds each row's last emitted tag, `per_layer` the cross tensors with a
836 /// matching leading batch axis (the encoder's own `[1,…]` outputs for a
837 /// single table, or [`Self::batch_cross`]'s concatenation), and the cache
838 /// grows `[N_LAYERS, rows, H, past, hd]` in lockstep. Returns each row's raw
839 /// argmax tag and the `[rows, EMBED_DIM]` hidden states, flattened.
840 fn step_kv_hoisted(
841 &mut self,
842 tags: &[i64],
843 per_layer: &[(String, DynValue)],
844 cache: &mut DecodeCache,
845 empty: &EmptyCache,
846 ) -> Result<(Vec<i64>, Vec<f32>), String> {
847 let rows = tags.len();
848 let tag_t = Tensor::from_array(([rows, 1usize], tags.to_vec()))
849 .map_err(|e| format!("tableformer: tag: {e}"))?;
850 let mut inputs: Vec<(
851 std::borrow::Cow<'_, str>,
852 ort::session::SessionInputValue<'_>,
853 )> = Vec::with_capacity(3 + per_layer.len());
854 inputs.push(("tag".into(), tag_t.into()));
855 match (cache.a.as_ref(), cache.b.as_ref()) {
856 (Some(k), Some(v)) => {
857 inputs.push(("cache_k".into(), k.into()));
858 inputs.push(("cache_v".into(), v.into()));
859 }
860 _ => {
861 inputs.push(("cache_k".into(), (&empty.0).into()));
862 inputs.push((
863 "cache_v".into(),
864 empty
865 .1
866 .as_ref()
867 .expect("kv empty cache has both halves")
868 .into(),
869 ));
870 }
871 }
872 for (name, v) in per_layer {
873 inputs.push((name.as_str().into(), v.into()));
874 }
875 let mut dout = docling_onnx::run(&mut self.decoder, inputs)
876 .map_err(|e| format!("tableformer: decode: {e}"))?;
877 let (_, logits) = dout["logits"]
878 .try_extract_tensor::<f32>()
879 .map_err(|e| format!("tableformer: logits: {e}"))?;
880 let vocab = logits.len() / rows;
881 let raws: Vec<i64> = logits
882 .chunks_exact(vocab)
883 .map(|row| argmax(row) as i64)
884 .collect();
885 let (_, hidden) = dout["hidden"]
886 .try_extract_tensor::<f32>()
887 .map_err(|e| format!("tableformer: hidden: {e}"))?;
888 let hidden = hidden.to_vec();
889 cache.a = Some(
890 dout.remove("out_cache_k")
891 .ok_or_else(|| "tableformer: out_cache_k missing".to_string())?,
892 );
893 cache.b = Some(
894 dout.remove("out_cache_v")
895 .ok_or_else(|| "tableformer: out_cache_v missing".to_string())?,
896 );
897 Ok((raws, hidden))
898 }
899
900 /// Stack the per-layer cross tensors of several encoded tables along the
901 /// batch axis (`[1,H,hd,S]` × B → `[B,H,hd,S]`, same for `cross_v`), index-
902 /// aligned with the decoder's input names. One copy per page — ~20 MB per
903 /// table, nothing next to the decode steps it lets the tables share.
904 fn batch_cross(encs: &[EncodeOut]) -> Result<Vec<(String, DynValue)>, String> {
905 let b = encs.len();
906 let mut out = Vec::with_capacity(encs[0].per_layer.len());
907 for j in 0..encs[0].per_layer.len() {
908 let name = encs[0].per_layer[j].0.clone();
909 let mut data: Vec<f32> = Vec::new();
910 let mut dims = [b, 0, 0, 0];
911 for enc in encs {
912 let (shape, v) = enc.per_layer[j]
913 .1
914 .try_extract_tensor::<f32>()
915 .map_err(|e| format!("tableformer: {name}: {e}"))?;
916 if shape.len() != 4 || shape[0] != 1 {
917 return Err(format!("tableformer: {name}: unexpected shape {shape:?}"));
918 }
919 dims[1..].copy_from_slice(&[
920 shape[1] as usize,
921 shape[2] as usize,
922 shape[3] as usize,
923 ]);
924 data.reserve(v.len() * b);
925 data.extend_from_slice(v);
926 }
927 let t = Tensor::from_array((dims, data))
928 .map_err(|e| format!("tableformer: {name}: {e}"))?;
929 out.push((name, t.into_dyn()));
930 }
931 Ok(out)
932 }
933
934 /// Decode `encs.len()` tables in lockstep: every step runs the decoder once
935 /// over all of them (a step is 49 weight-streaming GEMMs over one token per
936 /// row — B rows cost about what one does). The caches start empty for
937 /// every row and grow together, so nothing is ever padded or masked; a
938 /// table that emits `<end>` simply keeps its row (fed `END`, output
939 /// ignored) until the last one finishes. Row b of every op is exactly the
940 /// single-table computation, so each table's tokens and hidden states are
941 /// bit-identical to decoding it alone (asserted by the export script's
942 /// batching gate; the corpus snapshots pin it end-to-end).
943 fn decode_batch(&mut self, encs: &[EncodeOut]) -> Result<Vec<BboxBook>, String> {
944 let b = encs.len();
945 let cross = Self::batch_cross(encs)?;
946 let mut books: Vec<BboxBook> = (0..b).map(|_| BboxBook::new()).collect();
947 let mut active = vec![true; b];
948 let mut last = vec![START; b];
949 let mut cache = DecodeCache::default();
950 let empty = self.empty_cache(b)?;
951 crate::timing::timed("tf.decode_loop", || -> Result<(), String> {
952 // Each active table's `otsl` grows by one per step, so a shared
953 // step counter is the per-table `otsl.len() < MAX_STEPS` bound.
954 for _ in 0..MAX_STEPS {
955 if !active.iter().any(|a| *a) {
956 break;
957 }
958 let (raws, hidden) = crate::timing::timed("tf.decode_step", || {
959 self.step_kv(&last, &cross, &mut cache, &empty)
960 })?;
961 for t in 0..b {
962 if !active[t] {
963 continue;
964 }
965 let h = &hidden[t * EMBED_DIM..(t + 1) * EMBED_DIM];
966 if books[t].step(raws[t], h) {
967 last[t] = *books[t].tags.last().expect("step pushed a tag");
968 } else {
969 active[t] = false;
970 last[t] = END;
971 }
972 }
973 }
974 Ok(())
975 })?;
976 Ok(books)
977 }
978
979 /// The zero-`past` first-step cache(s) for `rows` tables, allocated through
980 /// the session allocator (ort's array constructors reject a 0-length dim;
981 /// the C API does allow it).
982 fn empty_cache(&self, rows: usize) -> Result<EmptyCache, String> {
983 let alloc = self.decoder.allocator();
984 if self.style != DecoderStyle::Legacy {
985 let mk = || {
986 Tensor::<f32>::new(alloc, [N_LAYERS, rows, KV_HEADS, 0usize, KV_HEAD_DIM])
987 .map_err(|e| format!("tableformer: empty kv cache: {e}"))
988 };
989 Ok((mk()?, Some(mk()?)))
990 } else {
991 let c = Tensor::<f32>::new(alloc, [N_LAYERS, 0usize, 1, EMBED_DIM])
992 .map_err(|e| format!("tableformer: empty cache: {e}"))?;
993 Ok((c, None))
994 }
995 }
996
997 /// Predict the OTSL structure-token sequence for a table-region image.
998 pub fn predict_otsl(&mut self, img: &RgbImage) -> Result<Vec<i64>, String> {
999 let enc = self.encode(img)?;
1000 // Structure corrections live in tf_core::correct (shared with the wasm
1001 // path); docling's line_num is never incremented, so xcel→lcel fires on
1002 // every row.
1003 let mut tags: Vec<i64> = vec![START];
1004 let mut out: Vec<i64> = Vec::new();
1005 let mut prev_ucel = false;
1006 let mut cache = DecodeCache::default();
1007 let empty = self.empty_cache(1)?;
1008 while out.len() < MAX_STEPS {
1009 let (raw, _hidden) = self.decode_step(&tags, &enc, &mut cache, &empty)?;
1010 let tag = correct(raw, prev_ucel);
1011 if tag == END {
1012 break;
1013 }
1014 out.push(tag);
1015 tags.push(tag);
1016 prev_ucel = tag == UCEL;
1017 }
1018 Ok(out)
1019 }
1020
1021 /// Full structure prediction: OTSL grid cells with per-cell boxes (in the 448
1022 /// image, normalized cxcywh). Collects per-cell decoder hidden states using
1023 /// docling's exact bbox bookkeeping (skip-after-row-break, first-lcel of a
1024 /// horizontal span), runs the bbox decoder, merges span boxes, then lays the
1025 /// cells onto the OTSL grid with row/col spans.
1026 pub fn predict_table_structure(&mut self, img: &RgbImage) -> Result<Vec<TableCell>, String> {
1027 let enc = self.encode(img)?;
1028
1029 // The autoregressive loop's bbox bookkeeping lives in tf_core::BboxBook
1030 // (shared with the wasm path); this loop only steps the decoder.
1031 let mut book = BboxBook::new();
1032 let mut cache = DecodeCache::default();
1033 let empty = self.empty_cache(1)?;
1034 crate::timing::timed("tf.decode_loop", || -> Result<(), String> {
1035 while book.otsl.len() < MAX_STEPS {
1036 let (raw, hidden) = self.decode_step(&book.tags, &enc, &mut cache, &empty)?;
1037 if !book.step(raw, &hidden) {
1038 break;
1039 }
1040 }
1041 Ok(())
1042 })?;
1043 self.finish_table(book, &enc.eo)
1044 }
1045
1046 /// The bbox stage after a table's decode loop: run the bbox decoder over
1047 /// the collected per-cell hidden states, merge span boxes, lay the cells
1048 /// onto the OTSL grid.
1049 fn finish_table(
1050 &mut self,
1051 mut book: BboxBook,
1052 eo: &DynValue,
1053 ) -> Result<Vec<TableCell>, String> {
1054 // docling#4592: a structure with no row break, or cut mid-row at the
1055 // step budget, is discarded with a warning (upstream logs at warning
1056 // level too). docling then keeps the text as one child of an empty
1057 // table; the port's empty result sends the region down the geometric
1058 // table path instead, which keeps the words *and* a grid
1059 // (docs/PDF_CONFORMANCE.md, "Degenerate TableFormer sequences").
1060 if let Some(reason) = book.degenerate_reason() {
1061 eprintln!(
1062 "docling-pdf: tableformer: discarding a predicted structure of {} tags because \
1063 {reason}; geometric table fallback",
1064 book.otsl.len()
1065 );
1066 return Ok(Vec::new());
1067 }
1068 if book.n == 0 {
1069 return Ok(Vec::new());
1070 }
1071 // One run over all cells materializes [ncells, 784, 512] fp32
1072 // intermediates (~1.6 GB each at 1,000 cells). Cells are independent
1073 // (each box/class depends on its own `tag_h` row and `enc_out`), so
1074 // running them in chunks bounds the peak without changing the output.
1075 let hiddens = std::mem::take(&mut book.hiddens);
1076 let mut boxes: Vec<[f32; 4]> = Vec::with_capacity(book.n);
1077 let mut classes: Vec<i64> = Vec::with_capacity(book.n);
1078 for rows in bbox_chunks(&hiddens) {
1079 let tag_h = Tensor::from_array(([rows.len() / EMBED_DIM, EMBED_DIM], rows.to_vec()))
1080 .map_err(|e| format!("tableformer: tag_h: {e}"))?;
1081 let bout = crate::timing::timed("tf.bbox", || {
1082 docling_onnx::run(
1083 &mut self.bbox,
1084 ort::inputs!["enc_out" => eo, "tag_h" => tag_h],
1085 )
1086 .map_err(|e| format!("tableformer: bbox: {e}"))
1087 })?;
1088 let (_, raw) = bout["boxes"]
1089 .try_extract_tensor::<f32>()
1090 .map_err(|e| format!("tableformer: boxes: {e}"))?;
1091 boxes.extend(raw.chunks_exact(4).map(|c| [c[0], c[1], c[2], c[3]]));
1092 // Per-cell class logits [n, 3] → argmax (docling's `outputs_class`).
1093 let (_, craw) = bout["classes"]
1094 .try_extract_tensor::<f32>()
1095 .map_err(|e| format!("tableformer: classes: {e}"))?;
1096 classes.extend(craw.chunks_exact(3).map(|c| argmax(c) as i64));
1097 }
1098 let (merged, merged_classes) = merge_spans(&boxes, &classes, &book.merge);
1099 Ok(build_table_cells(&book.otsl, &merged, &merged_classes))
1100 }
1101
1102 /// Predict a table region's Markdown grid: crop the region (docling's
1103 /// page→1024px box-average then bbox crop), run the structure model, then
1104 /// match the page's word cells into the predicted cells with docling's
1105 /// matching post-processor ([`crate::tf_match`]) and expand spans into a
1106 /// dense `rows × cols` grid. `region` is `(l, t, r, b)` in page points
1107 /// (top-left). Returns `None` if no structure is predicted.
1108 pub fn predict_table_rows(
1109 &mut self,
1110 page_image: &RgbImage,
1111 region: [f32; 4],
1112 words: &[TextCell],
1113 ) -> Option<crate::tf_core::TableGrid> {
1114 let page1024 = Self::page_1024(page_image);
1115 self.predict_table_rows_on(page_image.height(), &page1024, region, words)
1116 }
1117
1118 /// The page rendered at 1024 px height (cv2.INTER_AREA), the frame every
1119 /// table crop of that page is cut from. Computed once per page by the
1120 /// pipeline and shared across its tables — the resample is a full-page
1121 /// f64 box filter, 110–170 ms on the corpus pages, and it used to run
1122 /// again for every table on the page.
1123 pub fn page_1024(page_image: &RgbImage) -> RgbImage {
1124 let sf = 1024.0 / page_image.height() as f32;
1125 let pw = (page_image.width() as f32 * sf) as u32;
1126 crate::timing::timed("tableformer.inter_area", || {
1127 crate::resample::inter_area(page_image, pw, 1024)
1128 })
1129 }
1130
1131 /// [`predict_table_rows`](Self::predict_table_rows) with the page's
1132 /// 1024-px frame already built ([`page_1024`](Self::page_1024));
1133 /// `page_h` is the source page image's pixel height.
1134 pub fn predict_table_rows_on(
1135 &mut self,
1136 page_h: u32,
1137 page1024: &RgbImage,
1138 region: [f32; 4],
1139 words: &[TextCell],
1140 ) -> Option<crate::tf_core::TableGrid> {
1141 let crop = Self::crop_region(page_h, page1024, region)?;
1142 let cells = crate::timing::timed("tableformer.structure", || {
1143 self.predict_table_structure(&crop)
1144 })
1145 .ok()?;
1146 if cells.is_empty() {
1147 return None;
1148 }
1149 // The ort-free tail (word matching + grid assembly) is shared with the
1150 // browser path in tf_core.
1151 crate::tf_core::table_rows(&cells, region, words)
1152 }
1153
1154 /// Every table of a page at once: [`predict_table_rows_on`](Self::predict_table_rows_on)
1155 /// per region, except that with the dynamic-batch decoder the tables'
1156 /// decode steps are shared — each table is encoded on its own, then one
1157 /// decode loop steps all of them together ([`Self::decode_batch`]), then
1158 /// each runs its own bbox head. Per table the result is bit-identical to
1159 /// the one-at-a-time path; a page with a single table takes exactly that
1160 /// path. Should the batched run fail (an ort error), the tables are
1161 /// retried one by one so a page never loses all of its tables to one
1162 /// shared step.
1163 pub fn predict_tables_on(
1164 &mut self,
1165 page_h: u32,
1166 page1024: &RgbImage,
1167 regions: &[[f32; 4]],
1168 words: &[TextCell],
1169 ) -> Vec<Option<crate::tf_core::TableGrid>> {
1170 let mut out: Vec<Option<crate::tf_core::TableGrid>> = vec![None; regions.len()];
1171 let crops: Vec<(usize, RgbImage)> = regions
1172 .iter()
1173 .enumerate()
1174 .filter_map(|(i, r)| Self::crop_region(page_h, page1024, *r).map(|c| (i, c)))
1175 .collect();
1176 if self.batched && crops.len() > 1 {
1177 let batched = crate::timing::timed("tableformer.structure", || {
1178 self.predict_structures_batched(crops.iter().map(|(_, c)| c))
1179 });
1180 match batched {
1181 Ok(cells) => {
1182 for ((i, _), cells) in crops.iter().zip(cells) {
1183 if !cells.is_empty() {
1184 out[*i] = crate::tf_core::table_rows(&cells, regions[*i], words);
1185 }
1186 }
1187 return out;
1188 }
1189 Err(e) => docling_core::debug_log!(
1190 "docling-pdf: tableformer batched decode failed ({e}); decoding tables one by one"
1191 ),
1192 }
1193 }
1194 for (i, crop) in &crops {
1195 let cells = crate::timing::timed("tableformer.structure", || {
1196 self.predict_table_structure(crop)
1197 });
1198 if let Ok(cells) = cells {
1199 if !cells.is_empty() {
1200 out[*i] = crate::tf_core::table_rows(&cells, regions[*i], words);
1201 }
1202 }
1203 }
1204 out
1205 }
1206
1207 /// [`predict_table_structure`](Self::predict_table_structure) for several
1208 /// crops with the decode steps shared across them.
1209 fn predict_structures_batched<'a>(
1210 &mut self,
1211 crops: impl Iterator<Item = &'a RgbImage>,
1212 ) -> Result<Vec<Vec<TableCell>>, String> {
1213 let mut encs = Vec::new();
1214 for crop in crops {
1215 encs.push(self.encode(crop)?);
1216 }
1217 let books = self.decode_batch(&encs)?;
1218 books
1219 .into_iter()
1220 .zip(&encs)
1221 .map(|(book, enc)| self.finish_table(book, &enc.eo))
1222 .collect()
1223 }
1224
1225 /// Crop the table bbox out of the 1024px frame. docling's coordinate
1226 /// chain, rounding included: the cluster bbox is rounded to integer page
1227 /// points *first* (`round(cluster.bbox.l) * scale`, banker's rounding),
1228 /// scaled by 2 (its table-structure page scale), then by `1024 / <2x
1229 /// page-image height>`, and the crop indices round again. Rounding after
1230 /// scaling instead shifts some crops by a pixel — enough to change
1231 /// TableFormer's cell boxes on tall tables (redp5110's TOC). `None` for a
1232 /// region that collapses to an empty crop.
1233 fn crop_region(page_h: u32, page1024: &RgbImage, region: [f32; 4]) -> Option<RgbImage> {
1234 let k = 2.0 * 1024.0 / page_h as f64;
1235 let px = |v: f32| (v as f64).round_ties_even() * k;
1236 let x = (px(region[0]).round_ties_even()).max(0.0) as u32;
1237 let y = (px(region[1]).round_ties_even()).max(0.0) as u32;
1238 let x2 = (px(region[2]).round_ties_even() as u32).min(page1024.width());
1239 let y2 = (px(region[3]).round_ties_even() as u32).min(page1024.height());
1240 if x2 <= x || y2 <= y {
1241 return None;
1242 }
1243 Some(image::imageops::crop_imm(page1024, x, y, x2 - x, y2 - y).to_image())
1244 }
1245}
1246
1247/// Note once per process that TableFormer's ONNX graphs weren't found, so tables
1248/// fall back to geometric reconstruction. The default paths are relative
1249/// (`.models/tableformer/*.onnx`), which only resolves when the process's current
1250/// directory happens to be the repo root — a very easy miss for anything else
1251/// (an embedding app, a binding invoked from a different working directory, …),
1252/// and previously failed with no signal at all.
1253fn warn_missing_once(enc: &str, dec: &str, bbx: &str) {
1254 static WARNED: std::sync::Once = std::sync::Once::new();
1255 WARNED.call_once(|| {
1256 eprintln!(
1257 "docling.rs: TableFormer models not found (checked {enc}, {dec}, {bbx}); \
1258 tables will use geometric reconstruction instead of ML table-structure \
1259 recognition. Set DOCLING_TABLEFORMER_ENCODER / DOCLING_TABLEFORMER_DECODER \
1260 / DOCLING_TABLEFORMER_BBOX to enable it (see README.md)."
1261 );
1262 });
1263}
1264
1265/// docling's preprocessing: bilinear (cv2.INTER_LINEAR) resize the crop to 448²,
1266/// normalize `(x/255 − mean)/std`, laid out as (C, W, H) — docling transposes
1267/// (2,1,0), so width is the major spatial axis. The page→1024px box-average
1268/// (cv2.INTER_AREA) is the caller's job.
1269fn preprocess(img: &RgbImage) -> Result<Tensor<f32>, String> {
1270 Tensor::from_array(([1usize, 3, SIDE, SIDE], preprocess_input(img)))
1271 .map_err(|e| format!("tableformer: input: {e}"))
1272}
1273
1274#[cfg(test)]
1275mod bbox_chunk_tests {
1276 use super::*;
1277
1278 #[test]
1279 fn chunks_cover_all_cells_in_order_on_cell_boundaries() {
1280 for n in [1usize, 255, 256, 257, 980, 1000] {
1281 let h: Vec<f32> = (0..n * EMBED_DIM).map(|i| i as f32).collect();
1282 let chunks: Vec<&[f32]> = bbox_chunks(&h).collect();
1283 assert!(chunks.iter().all(|c| c.len() % EMBED_DIM == 0));
1284 assert!(chunks
1285 .iter()
1286 .all(|c| c.len() <= BBOX_CHUNK_CELLS * EMBED_DIM));
1287 assert_eq!(chunks.concat(), h, "n={n}");
1288 assert_eq!(chunks.len(), n.div_ceil(BBOX_CHUNK_CELLS));
1289 }
1290 }
1291}