pdf-engine 1.2.0

//! A column head set on two lines is one header row (#714).
//!
//! A statement writes `Gross Carrying` over `Amount`, `Accumulated` over
//! `Amortization`, `Weighted-` over `Average`: one head per column, wrapped
//! to the column's width. The grid read each line as a row of the table, so
//! the table opened with two or three rows of fragments over its figures,
//! and every corpus this engine is measured on writes the head as one row
//! with `Gross Carrying Amount` in its cell. TEDS charged each extra row and
//! each fragment on every such table.
//!
//! THE RULE, ON THE FINAL CELLS. The leading rows of a table that hold no
//! figure -- no digit, a bare year aside -- and no spanning cell are its
//! head; where there are two or three of them and a figure stands somewhere
//! below, they fold into one row, column by column, the fragments joined
//! with a space in reading order. A table with no figure in it at all is
//! left alone: a ratings table or a list of agreements has text in every
//! row, and its first rows are rows. One head row is left alone too: there
//! is nothing to fold.
//!
//! WHY HERE AND NOT IN THE GRID. The grid is right to read two lines as two
//! rows: that is what they are to the page, and a learned pass trained on
//! those rows reads them as it was trained. What a reader wants is the
//! table's own convention, which only the finished table shows -- the same
//! reason `crate::cell_tidy` runs on the final blocks.

// Copyright (c) 2026 Innovation Trigger B.V.
//
// PDFluent is available under two licences, at your option: the GNU AGPLv3, or
// the PDFluent Commercial Licence. See the LICENSE file in this repository --
// that file travels with the copy you received, which a URL does not.

// DISABLED IN 1.2.0 (20-10-2026). The pass is not called from the table pipeline: on the page of
// `the_paragraph_beside_a_table_is_not_its_first_column` it folds the first three data rows of a small
// table ("Parameter", "Pupils dilated", "Corneal reflex") into one header, which is wrong output, and
// its measured gain was about +0.003 TEDS. The function and its own tests stay, so that a rule which
// keeps a table's data rows apart can switch it on again by adding the four calls back to document.rs.
#![allow(dead_code)]

use crate::blocks::{Block, BlockKind};

/// The most leading rows that fold into one head.
const MAX_HEAD_ROWS: u16 = 3;

/// Fold the stacked head rows of every table in place.
pub(crate) fn merge_stacked_header_rows(blocks: &mut Vec<Block>) {
    let mut drop = vec![false; blocks.len()];
    let mut folded: Vec<(usize, usize)> = Vec::new();
    for (first, end) in tables_in(blocks) {
        let head = head_rows(&blocks[first..end]);
        if head < 2 {
            continue;
        }
        folded.push((first, end));
        // The columns the head rows use, in order; one merged cell each.
        let mut columns: Vec<u16> = Vec::new();
        for block in &blocks[first..end] {
            if let BlockKind::TableCell { row, column, .. } = block.kind {
                if row < head && !columns.contains(&column) {
                    columns.push(column);
                }
            }
        }
        columns.sort_unstable();
        for column in columns {
            let members: Vec<usize> = (first..end)
                .filter(|&i| {
                    matches!(blocks[i].kind,
                        BlockKind::TableCell { row, column: c, .. } if row < head && c == column)
                })
                .collect();
            let Some(&keep) = members.first() else {
                continue;
            };
            let mut text = String::new();
            let mut bbox = blocks[keep].bbox;
            let mut spans = Vec::new();
            let mut header = false;
            for &i in &members {
                let piece = blocks[i].text.trim();
                if !piece.is_empty() {
                    if !text.is_empty() {
                        text.push(' ');
                    }
                    text.push_str(piece);
                }
                bbox = union(bbox, blocks[i].bbox);
                spans.extend(blocks[i].spans.iter().cloned());
                if let BlockKind::TableCell { header: h, .. } = blocks[i].kind {
                    header |= h;
                }
                if i != keep {
                    drop[i] = true;
                }
            }
            let kept = &mut blocks[keep];
            kept.text = text;
            kept.bbox = bbox;
            kept.spans = spans;
            if let BlockKind::TableCell {
                row,
                row_span,
                header: h,
                ..
            } = &mut kept.kind
            {
                *row = 0;
                *row_span = 1;
                *h = header;
            }
        }
        // The rows under the head move up.
        for block in &mut blocks[first..end] {
            if let BlockKind::TableCell { row, .. } = &mut block.kind {
                if *row >= head {
                    *row -= head - 1;
                }
            }
        }
    }
    if folded.is_empty() {
        return;
    }
    // A folded table is row-major again: a column the first head row left
    // empty kept its cell from a lower row, which now stands at row 0 behind
    // the cells of row 0, and the serializer reads a coordinate that goes
    // backwards as a second table (`crate::markdown::table_run`).
    let mut order: Vec<usize> = (0..blocks.len()).collect();
    for (first, end) in folded {
        order[first..end].sort_by_key(|&i| match blocks[i].kind {
            BlockKind::TableCell { row, column, .. } => (row, column),
            _ => (u16::MAX, u16::MAX),
        });
    }
    let mut taken: Vec<Option<Block>> = blocks.drain(..).map(Some).collect();
    let mut out = Vec::with_capacity(taken.len());
    for i in order {
        if drop[i] {
            continue;
        }
        let Some(mut block) = taken[i].take() else {
            continue;
        };
        block.index = out.len();
        out.push(block);
    }
    *blocks = out;
}

/// How many leading rows of one table are a stacked head: rows without a
/// figure or a span, at least two and at most [`MAX_HEAD_ROWS`], with a
/// figure somewhere under them. Zero where the table keeps its rows.
fn head_rows(table: &[Block]) -> u16 {
    let rows = table
        .iter()
        .filter_map(|b| match b.kind {
            BlockKind::TableCell { row, .. } => Some(row),
            _ => None,
        })
        .max()
        .map_or(0, |r| r + 1);
    // The cells of each row, once, so the rows are not each a pass over
    // the table.
    let mut by_row: Vec<Vec<&Block>> = vec![Vec::new(); rows as usize];
    for b in table {
        if let BlockKind::TableCell { row, .. } = b.kind {
            by_row[row as usize].push(b);
        }
    }
    let mut head = 0u16;
    while head < rows && head < MAX_HEAD_ROWS {
        let cells = &by_row[head as usize];
        let row_is_head = cells.iter().all(|b| match b.kind {
            BlockKind::TableCell {
                row_span,
                column_span,
                ..
            } => row_span == 1 && column_span == 1 && !has_figure(&b.text),
            _ => true,
        });
        // A row with text in its stub alone is a section label (`Ameren:`),
        // which the corpora keep as a row of its own; a head names columns.
        let names_a_column = cells.iter().any(|b| match b.kind {
            BlockKind::TableCell { column, .. } => column > 0 && !b.text.trim().is_empty(),
            _ => false,
        });
        if !row_is_head || !names_a_column {
            break;
        }
        head += 1;
    }
    if head < 2 || head >= rows {
        return 0;
    }
    let figures_below = table.iter().any(|b| match b.kind {
        BlockKind::TableCell { row, .. } if row >= head => has_figure(&b.text),
        _ => false,
    });
    if figures_below {
        head
    } else {
        0
    }
}

/// Whether a cell's text carries a figure: a digit anywhere, a bare year
/// aside (`2014`, `2014,`), which a head row names as often as a body row.
fn has_figure(text: &str) -> bool {
    let trimmed = text.trim().trim_end_matches([',', '.']);
    if !trimmed.chars().any(|c| c.is_ascii_digit()) {
        return false;
    }
    let year = trimmed.len() == 4
        && trimmed.chars().all(|c| c.is_ascii_digit())
        && (trimmed.starts_with("19") || trimmed.starts_with("20"));
    !year
}

/// Every table as a half-open range of blocks: a run of cells in row-major
/// order, broken where the order restarts or the page changes.
///
/// A page change is a break of its own and not a step of the key: the key
/// `(page, row, column)` climbs across a page change, so a table that ends a
/// page and another that opens the next with nothing between them read as
/// one range until this said so, and the head fold folded both row 0s.
pub(crate) fn tables_in(blocks: &[Block]) -> Vec<(usize, usize)> {
    let mut out = Vec::new();
    let mut start: Option<usize> = None;
    let mut last: Option<(usize, u16, u16)> = None;
    for (i, block) in blocks.iter().enumerate() {
        match block.kind {
            BlockKind::TableCell { row, column, .. } => {
                let key = (block.page, row, column);
                let restarts = last.is_some_and(|seen| seen.0 != block.page || key <= seen);
                if restarts {
                    if let Some(first) = start.take() {
                        out.push((first, i));
                    }
                }
                start.get_or_insert(i);
                last = Some(key);
            }
            _ => {
                if let Some(first) = start.take() {
                    out.push((first, i));
                }
                last = None;
            }
        }
    }
    if let Some(first) = start.take() {
        out.push((first, blocks.len()));
    }
    out
}

fn union(a: [f64; 4], b: [f64; 4]) -> [f64; 4] {
    [
        a[0].min(b[0]),
        a[1].min(b[1]),
        a[2].max(b[2]),
        a[3].max(b[3]),
    ]
}

#[cfg(test)]
mod tests {
    use super::*;

    fn cell(row: u16, column: u16, text: &str) -> Block {
        Block {
            kind: BlockKind::TableCell {
                row,
                column,
                row_span: 1,
                column_span: 1,
                header: row == 0,
            },
            page: 0,
            bbox: [
                100.0 * column as f64,
                700.0 - 12.0 * row as f64,
                100.0 * column as f64 + 50.0,
                710.0 - 12.0 * row as f64,
            ],
            text: text.to_string(),
            spans: Vec::new(),
            index: 0,
            continues_previous: false,
            confidence: None,
            grid: None,
            table_origin: Default::default(),
            trace_learned: None,
        }
    }

    fn paragraph(text: &str) -> Block {
        let mut block = cell(0, 0, text);
        block.kind = BlockKind::Paragraph;
        block
    }

    fn rows_of(blocks: &[Block]) -> Vec<Vec<(u16, String)>> {
        let mut rows: Vec<Vec<(u16, String)>> = Vec::new();
        for b in blocks {
            if let BlockKind::TableCell { row, column, .. } = b.kind {
                while rows.len() <= row as usize {
                    rows.push(Vec::new());
                }
                rows[row as usize].push((column, b.text.clone()));
            }
        }
        rows
    }

    #[test]
    fn two_stacked_head_rows_fold_into_one() {
        let mut blocks = vec![
            paragraph("Intangible assets"),
            cell(0, 0, ""),
            cell(0, 1, "Gross Carrying"),
            cell(0, 2, "Accumulated"),
            cell(1, 0, ""),
            cell(1, 1, "Amount"),
            cell(1, 2, "Amortization"),
            cell(2, 0, "Lease rights"),
            cell(2, 1, "$131"),
            cell(2, 2, "$(57)"),
            cell(3, 0, "Total"),
            cell(3, 1, "$131"),
            cell(3, 2, "$(57)"),
            paragraph("after"),
        ];
        merge_stacked_header_rows(&mut blocks);
        let rows = rows_of(&blocks);
        assert_eq!(rows.len(), 3, "{rows:?}");
        assert_eq!(
            rows[0],
            vec![
                (0, String::new()),
                (1, "Gross Carrying Amount".to_string()),
                (2, "Accumulated Amortization".to_string())
            ]
        );
        assert_eq!(rows[1][0].1, "Lease rights");
        assert_eq!(rows[2][0].1, "Total");
        // The head keeps its mark, the index is counted again, prose stands.
        assert!(matches!(
            blocks[1].kind,
            BlockKind::TableCell { header: true, .. }
        ));
        assert!(blocks.iter().enumerate().all(|(i, b)| b.index == i));
        assert_eq!(blocks.len(), 11);
        assert!(matches!(blocks[10].kind, BlockKind::Paragraph));
        assert_eq!(blocks[10].text, "after");
    }

    #[test]
    fn a_column_the_first_head_row_leaves_empty_keeps_the_table_row_major() {
        // `(a)Ameren:` sits in the second head row only; after the fold it is
        // row 0, column 0 and must come before `Coal` in block order.
        let mut blocks = vec![
            cell(0, 1, "Coal"),
            cell(0, 2, "Natural"),
            cell(1, 0, "(a)Ameren:"),
            cell(1, 2, "Gas"),
            cell(2, 0, "2007"),
            cell(2, 1, "84%"),
            cell(2, 2, "2%"),
        ];
        merge_stacked_header_rows(&mut blocks);
        let coords: Vec<(u16, u16)> = blocks
            .iter()
            .filter_map(|b| match b.kind {
                BlockKind::TableCell { row, column, .. } => Some((row, column)),
                _ => None,
            })
            .collect();
        assert_eq!(coords, vec![(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)]);
        assert_eq!(blocks[0].text, "(a)Ameren:");
        assert_eq!(blocks[2].text, "Natural Gas");
        assert!(blocks.iter().enumerate().all(|(i, b)| b.index == i));
    }

    #[test]
    fn a_section_label_under_the_head_is_a_row() {
        let mut blocks = vec![
            cell(0, 1, "Coal"),
            cell(0, 2, "Nuclear"),
            cell(1, 0, "Ameren:"),
            cell(2, 0, "2007"),
            cell(2, 1, "84%"),
            cell(2, 2, "12%"),
        ];
        merge_stacked_header_rows(&mut blocks);
        assert_eq!(rows_of(&blocks).len(), 3);
    }

    #[test]
    fn two_tables_on_two_pages_with_nothing_between_fold_apart() {
        // Page 0's table ends in row 3; page 1's opens at row 0 with a
        // climbing key, so only the page tells the two apart.
        let mut blocks = vec![
            cell(0, 1, "Gross"),
            cell(1, 1, "Amount"),
            cell(2, 0, "Lease"),
            cell(2, 1, "$131"),
        ];
        let mut second = vec![
            cell(0, 1, "Net"),
            cell(1, 1, "Value"),
            cell(2, 0, "Cash"),
            cell(2, 1, "$9"),
        ];
        for b in &mut second {
            b.page = 1;
        }
        blocks.extend(second);
        merge_stacked_header_rows(&mut blocks);
        let pages: Vec<(usize, u16, String)> = blocks
            .iter()
            .filter_map(|b| match b.kind {
                BlockKind::TableCell { row, .. } => Some((b.page, row, b.text.clone())),
                _ => None,
            })
            .collect();
        assert_eq!(
            pages,
            vec![
                (0, 0, "Gross Amount".to_string()),
                (0, 1, "Lease".to_string()),
                (0, 1, "$131".to_string()),
                (1, 0, "Net Value".to_string()),
                (1, 1, "Cash".to_string()),
                (1, 1, "$9".to_string()),
            ]
        );
    }

    #[test]
    fn a_year_head_over_a_wrapped_head_folds_too() {
        let mut blocks = vec![
            cell(0, 1, "2011"),
            cell(0, 2, "2010"),
            cell(1, 1, "Weighted-"),
            cell(1, 2, "Weighted-"),
            cell(2, 1, "Average"),
            cell(2, 2, "Average"),
            cell(3, 0, "Granted"),
            cell(3, 1, "$7.34"),
            cell(3, 2, "$8.54"),
        ];
        merge_stacked_header_rows(&mut blocks);
        let rows = rows_of(&blocks);
        assert_eq!(rows.len(), 2, "{rows:?}");
        assert_eq!(rows[0][0].1, "2011 Weighted- Average");
        assert_eq!(rows[1][0].1, "Granted");
    }

    #[test]
    fn a_table_of_words_keeps_its_rows() {
        let mut blocks = vec![
            cell(0, 0, "Issuer"),
            cell(0, 1, "Moody's"),
            cell(1, 0, "Senior unsecured"),
            cell(1, 1, "Baa3"),
            cell(2, 0, "Commercial paper"),
            cell(2, 1, "P-3"),
        ];
        merge_stacked_header_rows(&mut blocks);
        assert_eq!(rows_of(&blocks).len(), 3);
        assert_eq!(blocks.len(), 6);
    }

    #[test]
    fn one_head_row_and_a_figure_in_the_second_keep_their_rows() {
        let mut blocks = vec![
            cell(0, 0, ""),
            cell(0, 1, "Amount"),
            cell(1, 0, "Balance"),
            cell(1, 1, "$169"),
            cell(2, 0, "Increases"),
            cell(2, 1, "8"),
        ];
        merge_stacked_header_rows(&mut blocks);
        assert_eq!(rows_of(&blocks).len(), 3);
    }

    #[test]
    fn a_spanning_head_cell_is_not_folded() {
        let mut blocks = vec![
            cell(0, 0, ""),
            cell(0, 1, "Years ended"),
            cell(1, 0, ""),
            cell(1, 1, "2012"),
            cell(1, 2, "2011"),
            cell(2, 0, "Revenue"),
            cell(2, 1, "153"),
            cell(2, 2, "137"),
        ];
        if let BlockKind::TableCell { column_span, .. } = &mut blocks[1].kind {
            *column_span = 2;
        }
        merge_stacked_header_rows(&mut blocks);
        assert_eq!(rows_of(&blocks).len(), 3);
    }

    #[test]
    fn at_most_three_rows_fold() {
        let mut blocks: Vec<Block> = (0..4u16).map(|r| cell(r, 1, "head")).collect();
        blocks.push(cell(4, 0, "x"));
        blocks.push(cell(4, 1, "12"));
        merge_stacked_header_rows(&mut blocks);
        let rows = rows_of(&blocks);
        assert_eq!(rows.len(), 3, "{rows:?}");
        assert_eq!(rows[0][0].1, "head head head");
        assert_eq!(rows[1][0].1, "head");
    }
}