spectre_pdf 1.0.0

Native Rust PDF extraction engine: text, markdown for RAG, AcroForm widgets, image decoding, and encrypted PDFs. Lazy parser, persistent Document handle, no C dependencies.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

"""Sanity check: do TOC titles appear at all in the rendered page text?"""
from pathlib import Path
import re
import pymupdf

PDF = Path("/mnt/c/Users/RyanJ/spectre-rs/bench-corpus/largedoc/i1040gi.pdf")


def normalize_heading(s: str) -> str:
    s = re.sub(r"[*_`]", "", s)
    s = re.sub(r"\s+", " ", s).strip().lower()
    s = re.sub(r"[\.\:\,\;\!\?\—\-]+$", "", s)
    return s


doc = pymupdf.open(PDF)
toc = doc.get_toc()
print(f"raw TOC entries: {len(toc)}")
gt = {normalize_heading(e[1]) for e in toc if e[1].strip()}
print(f"normalized unique: {len(gt)}")
print("sample TOC titles:")
for e in toc[:15]:
    print(f"  L{e[0]} p{e[2]}: {e[1]!r}")

# Build a giant haystack of every normalized line on every page.
all_lines: set[str] = set()
for page in doc:
    txt = page.get_text("text")
    for ln in txt.splitlines():
        h = normalize_heading(ln)
        if h and len(h) > 1:
            all_lines.add(h)
print(f"\nTotal unique normalized lines across doc: {len(all_lines)}")
overlap = gt & all_lines
print(f"TOC titles that appear verbatim as some line: {len(overlap)} / {len(gt)} = {len(overlap)/len(gt):.2%}")

# Show TOC titles that DON'T appear (first 15) so we can see why
missing = list(gt - all_lines)[:15]
print("\nSample TOC titles with NO verbatim match in page text:")
for m in missing:
    print(f"  {m!r}")