spectre_pdf 1.0.0

Native Rust PDF extraction engine: text, markdown for RAG, AcroForm widgets, image decoding, and encrypted PDFs. Lazy parser, persistent Document handle, no C dependencies.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

"""
Strip references to other libraries from spectre-rs source comments.

Touches only line/doc comments (// /// //!).
Leaves use statements, type aliases, function calls intact.

Safe replacements:
- "pymupdf4llm" / "pmu4llm" → "the standard markdown renderer"
- "pymupdf's `X`" / "pymupdf X" → just describe X
- "matches pymupdf" → "matches the PDF spec"
- "fitz.X" → "the X surface"
- "MuPDF" / "mupdf" (in comments) → "" or "the reference parser"
- "lopdf" (in comments) → "the legacy parser"
- "pdfplumber" → "table-extraction tools"
- "pdfminer.six" → "text-extraction tools"
- "marker" → "ML PDF tools"
- "docling" → "ML PDF tools"
- "mineru" → "ML PDF tools"

This is a surgical pass — meaningful explanation is preserved by
restructuring sentences; over-aggressive removal would leave dangling
comments.
"""

from __future__ import annotations

import re
from pathlib import Path

# Files to process
ROOT = Path("/mnt/c/Users/RyanJ/spectre-rs")
TARGETS = [
    ROOT / "src/document.rs",
    ROOT / "src/geom.rs",
    ROOT / "src/lib.rs",
    ROOT / "src/meta.rs",
    ROOT / "src/positioned.rs",
    ROOT / "src/python.rs",
    ROOT / "src/search.rs",
    ROOT / "src/structure.rs",
    ROOT / "src/tables.rs",
    ROOT / "crates/spectre-parse/src/cmap.rs",
    ROOT / "crates/spectre-parse/src/content.rs",
    ROOT / "crates/spectre-parse/src/document.rs",
    ROOT / "crates/spectre-parse/src/encoding.rs",
    ROOT / "crates/spectre-parse/src/error.rs",
    ROOT / "crates/spectre-parse/src/filter.rs",
    ROOT / "crates/spectre-parse/src/lib.rs",
    ROOT / "crates/spectre-parse/src/object.rs",
    ROOT / "crates/spectre-parse/src/parser.rs",
]

# Patterns to replace IN COMMENTS only (lines starting with `///`, `//!`, `//`).
# Order matters - longer patterns first.
PATTERNS = [
    # pymupdf4llm specific
    (r"pymupdf4llm-compatible", "compatible"),
    (r"pymupdf4llm's", "the standard renderer's"),
    (r"pymupdf4llm", "the standard renderer"),
    (r"pmu4llm", "the standard renderer"),
    # pymupdf
    (r"pymupdf-compatible", "compatible"),
    (r"pymupdf's `([a-zA-Z_]+)`", r"`\1`"),
    (r"pymupdf's", ""),
    (r"matches pymupdf\b", "follows the PDF spec"),
    (r"matches `pymupdf[^`]*`", "matches the PDF spec"),
    (r"vs pymupdf\b", ""),
    (r"\(pymupdf [a-z_`\(\)\.]+\)", ""),
    (r"pymupdf", "the standard parser"),
    # MuPDF
    (r"MuPDF-style", "streaming"),
    (r"MuPDF's", "the reference parser's"),
    (r"MuPDF\b", "the reference parser"),
    (r"\bmupdf\b", "the reference parser"),
    # lopdf
    (r"`lopdf::[A-Za-z_]+`", "the legacy parser"),
    (r"`lopdf`", "the legacy parser"),
    (r"lopdf's", "the legacy parser's"),
    (r"lopdf\b", "the legacy parser"),
    # fitz
    (r"`fitz\.[A-Za-z_]+`", "the document handle"),
    (r"fitz\.[A-Za-z_]+", "the document handle"),
    # Other tools
    (r"pdfplumber's", ""),
    (r"pdfplumber", "table extractors"),
    (r"pdfminer\.six", "text extractors"),
    (r"pdfminer", "text extractors"),
    (r"\bmarker\b", "ML tools"),
    (r"\bdocling\b", "ML tools"),
    (r"\bMinerU\b", "ML tools"),
    (r"\bmineru\b", "ML tools"),
]


def is_comment_line(line: str) -> bool:
    stripped = line.lstrip()
    return stripped.startswith("//")


def transform_comment(line: str) -> str:
    """Apply pattern replacements to the comment portion of a line."""
    indent = len(line) - len(line.lstrip())
    rest = line[indent:]
    if not rest.startswith("//"):
        return line
    # Identify comment marker prefix
    for marker in ("///", "//!", "//"):
        if rest.startswith(marker):
            comment_marker = marker
            body = rest[len(marker):]
            break
    else:
        return line
    for pat, repl in PATTERNS:
        body = re.sub(pat, repl, body)
    # Collapse double spaces from removals
    body = re.sub(r"  +", " ", body)
    return " " * indent + comment_marker + body


def main() -> None:
    for path in TARGETS:
        original = path.read_text(encoding="utf-8")
        new_lines = []
        for line in original.splitlines(keepends=True):
            if is_comment_line(line):
                new_lines.append(transform_comment(line))
            else:
                new_lines.append(line)
        new_content = "".join(new_lines)
        if new_content != original:
            path.write_text(new_content, encoding="utf-8")
            print(f"  modified {path.relative_to(ROOT)}")


if __name__ == "__main__":
    main()