Skip to main content

pdf2md_tidy/
lib.rs

1//! Clean up Markdown that was converted from a PDF.
2//!
3//! PDF extraction leaves three recurring problems behind: paragraphs that are
4//! hard-wrapped at the original column width, heading levels that start at an
5//! arbitrary depth, and pipe tables whose columns no longer line up. This crate
6//! fixes those three and leaves everything else alone.
7//!
8//! The Markdown it is written against is the output of the browser-based
9//! converter at <https://pdf2md.me>, which runs the conversion locally and
10//! preserves tables and heading levels, but any CommonMark source works.
11//!
12//! ```
13//! let out = pdf2md_tidy::run("### Title\n\nline one\nline two", pdf2md_tidy::Options::default());
14//! assert!(out.starts_with("# Title"));
15//! ```
16
17/// Which clean-up passes to run.
18#[derive(Debug, Clone, Copy)]
19pub struct Options {
20    /// Join lines that were hard-wrapped inside a paragraph.
21    pub unwrap: bool,
22    /// Shift every heading so the shallowest one becomes H1.
23    pub normalize_headings: bool,
24    /// Pad pipe-table cells so the columns line up.
25    pub align_tables: bool,
26}
27
28impl Default for Options {
29    fn default() -> Self {
30        Options { unwrap: true, normalize_headings: true, align_tables: true }
31    }
32}
33
34/// Apply the selected passes to `src` and return the cleaned Markdown.
35pub fn run(src: &str, opt: Options) -> String {
36    let mut lines: Vec<String> =
37        src.replace("\r\n", "\n").split('\n').map(|s| s.to_string()).collect();
38    if opt.normalize_headings {
39        lines = normalize_headings(lines);
40    }
41    if opt.align_tables {
42        lines = align_tables(lines);
43    }
44    if opt.unwrap {
45        lines = unwrap_paragraphs(lines);
46    }
47    lines.join("\n")
48}
49
50fn heading_depth(line: &str) -> Option<usize> {
51    let hashes = line.chars().take_while(|c| *c == '#').count();
52    if hashes == 0 || hashes > 6 {
53        return None;
54    }
55    match line.chars().nth(hashes) {
56        Some(c) if c == ' ' || c == '\t' => Some(hashes),
57        _ => None,
58    }
59}
60
61fn is_fence(line: &str) -> bool {
62    line.starts_with("```") || line.starts_with("~~~")
63}
64
65fn is_table_row(line: &str) -> bool {
66    let t = line.trim();
67    t.len() >= 2 && t.starts_with('|') && t.ends_with('|')
68}
69
70fn normalize_headings(lines: Vec<String>) -> Vec<String> {
71    let mut min = 7usize;
72    let mut in_fence = false;
73    for l in &lines {
74        if is_fence(l) {
75            in_fence = !in_fence;
76            continue;
77        }
78        if in_fence {
79            continue;
80        }
81        if let Some(d) = heading_depth(l) {
82            if d < min {
83                min = d;
84            }
85        }
86    }
87    if min == 7 || min == 1 {
88        return lines;
89    }
90    let shift = min - 1;
91    let mut in_fence = false;
92    lines
93        .into_iter()
94        .map(|l| {
95            if is_fence(&l) {
96                in_fence = !in_fence;
97                return l;
98            }
99            match heading_depth(&l) {
100                Some(d) if !in_fence => format!("{}{}", "#".repeat(d - shift), &l[d..]),
101                _ => l,
102            }
103        })
104        .collect()
105}
106
107fn split_row(row: &str) -> Vec<String> {
108    let t = row.trim().trim_start_matches('|').trim_end_matches('|');
109    t.split('|').map(|c| c.trim().to_string()).collect()
110}
111
112fn is_divider(cell: &str) -> bool {
113    !cell.is_empty() && cell.trim_matches(|c| c == '-' || c == ':' || c == ' ').is_empty()
114}
115
116fn format_table(rows: &[String]) -> Vec<String> {
117    let cells: Vec<Vec<String>> = rows.iter().map(|r| split_row(r)).collect();
118    let mut width: Vec<usize> = Vec::new();
119    for row in &cells {
120        for (i, c) in row.iter().enumerate() {
121            let n = c.chars().count();
122            if width.len() <= i {
123                width.push(0);
124            }
125            if n > width[i] {
126                width[i] = n;
127            }
128        }
129    }
130    cells
131        .iter()
132        .map(|row| {
133            let mut out = String::from("|");
134            for (i, w) in width.iter().enumerate() {
135                let cell = row.get(i).map(|s| s.as_str()).unwrap_or("");
136                if is_divider(cell) {
137                    out.push_str(&format!(" {} |", "-".repeat(*w)));
138                } else {
139                    let pad = w - cell.chars().count();
140                    out.push_str(&format!(" {}{} |", cell, " ".repeat(pad)));
141                }
142            }
143            out
144        })
145        .collect()
146}
147
148fn align_tables(lines: Vec<String>) -> Vec<String> {
149    let mut out: Vec<String> = Vec::with_capacity(lines.len());
150    let mut i = 0;
151    while i < lines.len() {
152        if !is_table_row(&lines[i]) {
153            out.push(lines[i].clone());
154            i += 1;
155            continue;
156        }
157        let start = i;
158        while i < lines.len() && is_table_row(&lines[i]) {
159            i += 1;
160        }
161        out.extend(format_table(&lines[start..i]));
162    }
163    out
164}
165
166fn is_block(line: &str) -> bool {
167    let t = line.trim();
168    if t.is_empty() {
169        return false;
170    }
171    if heading_depth(t).is_some() || is_table_row(line) {
172        return true;
173    }
174    if t.starts_with("> ") || t.starts_with("- ") || t.starts_with("* ") || t.starts_with("+ ") {
175        return true;
176    }
177    let digits = t.chars().take_while(|c| c.is_ascii_digit()).count();
178    digits > 0
179        && matches!(t.chars().nth(digits), Some('.') | Some(')'))
180        && matches!(t.chars().nth(digits + 1), Some(' '))
181}
182
183fn unwrap_paragraphs(lines: Vec<String>) -> Vec<String> {
184    let mut out: Vec<String> = Vec::new();
185    let mut buf: Vec<String> = Vec::new();
186    let mut in_fence = false;
187    for l in lines {
188        if is_fence(&l) {
189            if !buf.is_empty() {
190                out.push(buf.join(" "));
191                buf.clear();
192            }
193            in_fence = !in_fence;
194            out.push(l);
195            continue;
196        }
197        if in_fence || is_block(&l) || l.trim().is_empty() {
198            if !buf.is_empty() {
199                out.push(buf.join(" "));
200                buf.clear();
201            }
202            out.push(l);
203            continue;
204        }
205        buf.push(l.trim().to_string());
206    }
207    if !buf.is_empty() {
208        out.push(buf.join(" "));
209    }
210    out
211}
212
213#[cfg(test)]
214mod tests {
215    use super::*;
216
217    #[test]
218    fn shifts_headings_to_h1() {
219        let got = run("### Title\n\n#### Sub\n", Options { unwrap: false, normalize_headings: true, align_tables: false });
220        assert_eq!(got, "# Title\n\n## Sub\n");
221    }
222
223    #[test]
224    fn joins_hard_wrapped_lines() {
225        let got = run("a line that was\nhard wrapped\n\nnext", Options { unwrap: true, normalize_headings: false, align_tables: false });
226        assert_eq!(got, "a line that was hard wrapped\n\nnext");
227    }
228
229    #[test]
230    fn keeps_list_items_separate() {
231        let src = "- one\n- two";
232        assert_eq!(run(src, Options { unwrap: true, normalize_headings: false, align_tables: false }), src);
233    }
234
235    #[test]
236    fn aligns_pipe_tables() {
237        let got = run("|a|bbb|\n|-|-|\n|cc|d|", Options { unwrap: false, normalize_headings: false, align_tables: true });
238        assert_eq!(got, "| a  | bbb |\n| -- | --- |\n| cc | d   |");
239    }
240
241    #[test]
242    fn leaves_code_fences_alone() {
243        let src = "```\nline one\nline two\n```";
244        assert_eq!(run(src, Options::default()), src);
245    }
246}