chk_defaced 0.1.0

//! OCR-atlas escalation for PDF (feature `ocr-atlas`): render each page with **pdfium**, OCR the
//! rendering (what a human *sees*), and compare it with the PDF text layer (what a machine
//! *extracts*). A large divergence means the rendered glyphs do not match the extracted text —
//! semantic replacement / forged-font tampering (variant A3/C) that the deterministic checks cannot
//! catch.
//!
//! Needs the native OCR backend (`ocr-tesseract`), the pdfium dynamic library, and `tessdata`.

use std::collections::HashSet;
use std::path::Path;

use anyhow::{anyhow, Result};
use pdfium_render::prelude::*;

use crate::ocr::{GrayImage, OcrEngine, OcrHint};

/// Per-page comparison of extracted text vs OCR of the rendered page.
pub struct PageDivergence {
    pub page: usize,
    pub extracted: String,
    pub visual: String,
    /// Word-set Jaccard similarity in 0..=1 (1 = identical word sets).
    pub similarity: f32,
}

fn words(s: &str) -> HashSet<String> {
    s.to_lowercase()
        .split(|c: char| !c.is_alphanumeric())
        .filter(|w| w.chars().count() >= 2)
        .map(|w| w.to_string())
        .collect()
}

/// Word-set Jaccard similarity, robust to word order and OCR segmentation differences.
fn jaccard(a: &str, b: &str) -> f32 {
    let (wa, wb) = (words(a), words(b));
    if wa.is_empty() && wb.is_empty() {
        return 1.0;
    }
    let inter = wa.intersection(&wb).count() as f32;
    let union = wa.union(&wb).count() as f32;
    if union == 0.0 {
        0.0
    } else {
        inter / union
    }
}

/// Render up to `max_pages` pages at `scale`× (1.0 ≈ 72 DPI), OCR each, and compare with the page's
/// extracted text layer. `pdfium_dir` must contain the pdfium dynamic library.
pub fn ocr_vs_extracted(
    pdf: &Path,
    pdfium_dir: &Path,
    ocr: &dyn OcrEngine,
    scale: f32,
    max_pages: usize,
) -> Result<Vec<PageDivergence>> {
    let lib = Pdfium::pdfium_platform_library_name_at_path(pdfium_dir);
    let bindings =
        Pdfium::bind_to_library(&lib).map_err(|e| anyhow!("bind pdfium ({}): {e}", lib.display()))?;
    let pdfium = Pdfium::new(bindings);
    let doc = pdfium.load_pdf_from_file(pdf, None).map_err(|e| anyhow!("open pdf: {e}"))?;

    let mut out = Vec::new();
    for (i, page) in doc.pages().iter().enumerate() {
        if i >= max_pages {
            break;
        }
        let w_pt = page.width().value;
        let target_w = ((w_pt * scale).round() as i32).max(1);
        let config = PdfRenderConfig::new().set_target_width(target_w);
        let bitmap = page.render_with_config(&config).map_err(|e| anyhow!("render page: {e}"))?;
        let luma = bitmap.as_image().to_luma8();
        let (w, h) = (luma.width(), luma.height());
        let img = GrayImage::new(w, h, luma.into_raw());

        let visual = ocr.recognize(&img, OcrHint::Block)?;
        let extracted = page.text().map(|t| t.all()).unwrap_or_default();
        let similarity = jaccard(&extracted, &visual);
        out.push(PageDivergence { page: i + 1, extracted, visual, similarity });
    }
    Ok(out)
}

/// Ordered, normalized word sequence (lowercase, alphanumeric runs, length ≥ 2).
fn word_seq(s: &str) -> Vec<String> {
    s.to_lowercase()
        .split(|c: char| !c.is_alphanumeric())
        .filter(|w| w.chars().count() >= 2)
        .map(|w| w.to_string())
        .collect()
}

/// Aligned word-level mismatches between extracted and visual text (LCS diff; deletions paired with
/// insertions as substitutions). Catches **localized semantic replacement** — a few substituted words
/// in otherwise-identical text — which the page-level set similarity cannot see. NB: OCR errors also
/// surface here, so use it for investigation / as a signal, not a zero-false-positive verdict.
pub fn word_mismatches(extracted: &str, visual: &str) -> Vec<(String, String)> {
    let a = word_seq(extracted);
    let b = word_seq(visual);
    let (n, m) = (a.len(), b.len());
    if n == 0 || m == 0 {
        return Vec::new();
    }
    // LCS lengths (suffix DP).
    let mut dp = vec![vec![0u16; m + 1]; n + 1];
    for i in (0..n).rev() {
        for j in (0..m).rev() {
            dp[i][j] = if a[i] == b[j] {
                dp[i + 1][j + 1] + 1
            } else {
                dp[i + 1][j].max(dp[i][j + 1])
            };
        }
    }
    let (mut i, mut j) = (0usize, 0usize);
    let (mut dels, mut inss): (Vec<String>, Vec<String>) = (Vec::new(), Vec::new());
    let mut out = Vec::new();
    let flush = |dels: &mut Vec<String>, inss: &mut Vec<String>, out: &mut Vec<(String, String)>| {
        let k = dels.len().min(inss.len());
        for t in 0..k {
            out.push((dels[t].clone(), inss[t].clone()));
        }
        dels.clear();
        inss.clear();
    };
    while i < n && j < m {
        if a[i] == b[j] {
            flush(&mut dels, &mut inss, &mut out);
            i += 1;
            j += 1;
        } else if dp[i + 1][j] >= dp[i][j + 1] {
            dels.push(a[i].clone());
            i += 1;
        } else {
            inss.push(b[j].clone());
            j += 1;
        }
    }
    while i < n {
        dels.push(a[i].clone());
        i += 1;
    }
    while j < m {
        inss.push(b[j].clone());
        j += 1;
    }
    flush(&mut dels, &mut inss, &mut out);
    out
}

fn levenshtein(a: &[char], b: &[char]) -> usize {
    let mut prev: Vec<usize> = (0..=b.len()).collect();
    for (i, ca) in a.iter().enumerate() {
        let mut cur = vec![i + 1];
        for (j, cb) in b.iter().enumerate() {
            let cost = usize::from(ca != cb);
            cur.push((prev[j + 1] + 1).min(cur[j] + 1).min(prev[j] + cost));
        }
        prev = cur;
    }
    prev[b.len()]
}

/// Two aligned words look like a *real substitution* (not an OCR near-miss) if both are alphabetic
/// words of length ≥ 3 and their normalized edit distance is large (e.g. "delaware" vs "maryland"),
/// as opposed to OCR noise ("ai"/"al", "recipient"/"recipients").
fn is_real_substitution(e: &str, v: &str) -> bool {
    let (ec, vc): (Vec<char>, Vec<char>) = (e.chars().collect(), v.chars().collect());
    if ec.len() < 3 || vc.len() < 3 || !ec.iter().all(|c| c.is_alphabetic()) || !vc.iter().all(|c| c.is_alphabetic()) {
        return false;
    }
    let max = ec.len().max(vc.len()) as f32;
    levenshtein(&ec, &vc) as f32 / max > 0.4
}

/// Word substitutions that look deliberate (filters out OCR noise via edit distance). A non-empty
/// result on a page is a strong signal of **semantic replacement** (variant A3): the extracted word
/// differs from the rendered/visible word, both lexically valid.
pub fn significant_substitutions(extracted: &str, visual: &str) -> Vec<(String, String)> {
    word_mismatches(extracted, visual)
        .into_iter()
        .filter(|(e, v)| is_real_substitution(e, v))
        .collect()
}

/// Convenience verdict: a page diverges (likely tampered) when the rendered page has substantial text
/// (so the OCR is trustworthy) yet it barely overlaps the extracted text. This catches the case where
/// the extracted layer is garbage/PUA while the glyphs render real words.
pub fn diverges(d: &PageDivergence, threshold: f32) -> bool {
    words(&d.visual).len() >= 5 && d.similarity < threshold
}