rumdl_lib/utils/
parser_options.rs

1use pulldown_cmark::Options;
2
3/// Standard pulldown-cmark options for rumdl parsing.
4///
5/// Uses an explicit allowlist rather than `Options::all()` to prevent
6/// future pulldown-cmark releases from silently changing parse behavior.
7///
8/// Notably excludes `ENABLE_YAML_STYLE_METADATA_BLOCKS` and
9/// `ENABLE_PLUSES_DELIMITED_METADATA_BLOCKS` because rumdl handles
10/// front matter detection independently. These options cause pulldown-cmark
11/// to misinterpret `---` horizontal rules as metadata delimiters,
12/// corrupting code block detection across the entire document.
13pub fn rumdl_parser_options() -> Options {
14    let mut options = Options::empty();
15    options.insert(Options::ENABLE_TABLES);
16    options.insert(Options::ENABLE_FOOTNOTES);
17    options.insert(Options::ENABLE_STRIKETHROUGH);
18    options.insert(Options::ENABLE_TASKLISTS);
19    options.insert(Options::ENABLE_SMART_PUNCTUATION);
20    options.insert(Options::ENABLE_HEADING_ATTRIBUTES);
21    options.insert(Options::ENABLE_MATH);
22    options.insert(Options::ENABLE_GFM);
23    options.insert(Options::ENABLE_DEFINITION_LIST);
24    options.insert(Options::ENABLE_SUPERSCRIPT);
25    options.insert(Options::ENABLE_SUBSCRIPT);
26    options.insert(Options::ENABLE_WIKILINKS);
27    options
28}
29
30/// The parser options for parsing `content`.
31///
32/// [`rumdl_parser_options`] with the heading attribute extension left out of
33/// text where it corrupts the parser's tree (see `has_setext_attribute_line`).
34/// Every parse of document text takes its options from here.
35pub fn rumdl_parser_options_for(content: &str) -> Options {
36    let mut options = rumdl_parser_options();
37    if has_setext_attribute_line(content) {
38        options.remove(Options::ENABLE_HEADING_ATTRIBUTES);
39    }
40    options
41}
42
43/// Whether `content` has a line that is nothing but a `{...}` block directly
44/// followed by a setext underline, once block quote markers and indentation are
45/// set aside.
46///
47/// With heading attributes enabled, pulldown-cmark cuts the attribute block off
48/// the end of a setext heading's text. When that block is a whole line inside a
49/// block quote or list item, the cut lands on the wrong bytes: the tree is left
50/// corrupt, and the parse panics either immediately (debug builds) or when the
51/// events are read (release builds). Without the extension the same lines are an
52/// ordinary setext heading. The test over-approximates the failing shapes (it
53/// also holds at top level, where the parser copes), so that no container,
54/// indentation or tab arrangement of the failing shape is missed; the price of a
55/// false positive is that such a document reads `{...}` as text.
56fn has_setext_attribute_line(content: &str) -> bool {
57    fn trimmed(line: &str) -> &str {
58        line.trim_start_matches([' ', '\t', '\x0b', '\x0c', '>'])
59            .trim_end_matches([' ', '\t', '\x0b', '\x0c'])
60    }
61    let mut seek = 0;
62    while let Some(relative) = content[seek..].find('{') {
63        let brace = seek + relative;
64        let start = content[..brace].rfind(['\n', '\r']).map_or(0, |i| i + 1);
65        let end = brace + content[brace..].find(['\n', '\r']).unwrap_or(content.len() - brace);
66        let text = trimmed(&content[start..end]);
67        let next = if content.as_bytes().get(end) == Some(&b'\r') && content.as_bytes().get(end + 1) == Some(&b'\n') {
68            end + 2
69        } else {
70            (end + 1).min(content.len())
71        };
72        seek = next;
73        if !text.starts_with('{') || !text.ends_with('}') || next == content.len() {
74            continue;
75        }
76        let stop = next + content[next..].find(['\n', '\r']).unwrap_or(content.len() - next);
77        let underline = trimmed(&content[next..stop]);
78        if !underline.is_empty() && (underline.bytes().all(|b| b == b'=') || underline.bytes().all(|b| b == b'-')) {
79            return true;
80        }
81    }
82    false
83}
84
85/// The byte ranges of the code spans in `content` as a reader without math
86/// finds them, sorted by start.
87///
88/// rumdl reads `$...$` and `$$...$$` as math, and a backtick inside math opens
89/// no code span. CommonMark has no math, and there that backtick opens a span
90/// closing at the next backtick run of its length, inside the math or past it.
91/// A rewrite that has to keep a code span's text as written needs both
92/// readings, so this one comes from the same parse with only math left out.
93pub fn code_span_ranges_without_math(content: &str) -> Vec<(usize, usize)> {
94    if !content.contains('`') {
95        return Vec::new();
96    }
97    let mut options = rumdl_parser_options_for(content);
98    options.remove(Options::ENABLE_MATH);
99    pulldown_cmark::Parser::new_ext(content, options)
100        .into_offset_iter()
101        .filter_map(|(event, range)| {
102            matches!(event, pulldown_cmark::Event::Code(_)).then_some((range.start, range.end))
103        })
104        .collect()
105}
106
107#[cfg(test)]
108mod tests {
109    use super::*;
110    use pulldown_cmark::Parser;
111
112    fn legacy_has_setext_attribute_line(content: &str) -> bool {
113        if !content.contains('{') {
114            return false;
115        }
116        // A line ends at `\n`, `\r\n` or a lone `\r`.
117        let lines = content
118            .split('\n')
119            .flat_map(|line| line.strip_suffix('\r').unwrap_or(line).split('\r'));
120        let mut previous_is_attribute_block = false;
121        for line in lines {
122            let text = line
123                .trim_start_matches([' ', '\t', '\x0b', '\x0c', '>'])
124                .trim_end_matches([' ', '\t', '\x0b', '\x0c']);
125            let is_underline = !text.is_empty() && (text.bytes().all(|b| b == b'=') || text.bytes().all(|b| b == b'-'));
126            if previous_is_attribute_block && is_underline {
127                return true;
128            }
129            previous_is_attribute_block = text.starts_with('{') && text.ends_with('}');
130        }
131        false
132    }
133
134    /// Drive a parse to its end the way a consumer does.
135    fn parse_fully(content: &str) -> usize {
136        Parser::new_ext(content, rumdl_parser_options_for(content))
137            .into_offset_iter()
138            .count()
139    }
140
141    #[test]
142    fn parser_survives_a_setext_heading_whose_last_line_is_an_attribute_block() {
143        for content in [
144            ">o\n>  {}\n>=\n",
145            ">o\n>      {}\n>=\n",
146            ">o\n>    {}\n>-\n",
147            "- o\n   {}\n  =\n",
148            "1. o\n     {}\n   =\n",
149            ">>o\n>>      {}\n>>=\n",
150            "> - o\n>    {}\n>   =\n",
151            ">o\n>  \u{c}{}\n>=\n",
152            ">o\n>  \u{b}{}\n>=\n",
153            ">o\r\n>  {}\r\n>=\r\n",
154            ">o\r>  {}\r>=\r",
155        ] {
156            assert!(parse_fully(content) > 0, "{content:?}");
157        }
158    }
159
160    #[test]
161    fn parser_survives_every_short_combination_of_container_prefixes_and_attribute_lines() {
162        let prefixes = ["", ">", "> ", ">  ", ">    ", "- ", "   ", "\t"];
163        let bodies = ["o", "{}", "{#a}", "x {}", "=", "-", "---", "", "{ }"];
164        let lines: Vec<String> = prefixes
165            .iter()
166            .flat_map(|p| bodies.iter().map(move |b| format!("{p}{b}")))
167            .collect();
168        let mut parsed = 0usize;
169        for a in &lines {
170            for b in &lines {
171                parse_fully(&format!("{a}\n{b}\n"));
172                for c in &lines {
173                    parse_fully(&format!("{a}\n{b}\n{c}\n"));
174                    parsed += 1;
175                }
176            }
177        }
178        assert!(parsed > 100_000, "corpus too small: {parsed}");
179    }
180
181    #[test]
182    fn setext_attribute_line_detects_the_shapes_that_corrupt_the_parse() {
183        for content in [
184            ">o\n>  {}\n>=\n",
185            "- o\n   {}\n  =\n",
186            ">o\n>  \u{c}{}\n>=\n",
187            ">o\n>  {}\u{b}\n>-\n",
188            ">o\r\n>  {}\r\n>=\r\n",
189            ">o\r>  {}\r>=\r",
190            "> > o\n> >   {#a .b}\n> > ---\n",
191        ] {
192            assert!(has_setext_attribute_line(content), "{content:?}");
193            assert!(
194                !rumdl_parser_options_for(content).contains(Options::ENABLE_HEADING_ATTRIBUTES),
195                "{content:?}"
196            );
197        }
198    }
199
200    #[test]
201    fn setext_attribute_line_leaves_other_text_with_attributes_enabled() {
202        for content in [
203            "# Heading {#id}\n",
204            "Title {#id}\n=====\n",
205            ">o\n>    x{}\n>=\n",
206            "{}\n\n---\n",
207            "{\"a\": 1}\n",
208            "plain text\n",
209            "{}\nnot an underline\n",
210            "{}\n",
211        ] {
212            assert!(!has_setext_attribute_line(content), "{content:?}");
213            assert!(
214                rumdl_parser_options_for(content).contains(Options::ENABLE_HEADING_ATTRIBUTES),
215                "{content:?}"
216            );
217        }
218    }
219
220    /// Every parse of document text goes through `rumdl_parser_options_for`, so no
221    /// call site can hand the parser text it cannot survive.
222    #[test]
223    fn production_code_derives_parser_options_from_the_text_it_parses() {
224        fn rust_files(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
225            for entry in std::fs::read_dir(dir).unwrap() {
226                let path = entry.unwrap().path();
227                if path.is_dir() {
228                    rust_files(&path, out);
229                } else if path.extension().is_some_and(|ext| ext == "rs") {
230                    out.push(path);
231                }
232            }
233        }
234
235        let src = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
236        let mut files = Vec::new();
237        rust_files(&src, &mut files);
238        assert!(files.len() > 100, "source scan found only {} files", files.len());
239
240        let mut offenders = Vec::new();
241        for path in files {
242            let name = path.file_name().unwrap().to_string_lossy().into_owned();
243            if name == "parser_options.rs" || name == "tests.rs" || name.ends_with("_tests.rs") {
244                continue;
245            }
246            let source = std::fs::read_to_string(&path).unwrap();
247            let production = source.split("#[cfg(test)]").next().unwrap_or(&source);
248            for (number, line) in production.lines().enumerate() {
249                let code = line.split("//").next().unwrap_or(line);
250                if code.contains("rumdl_parser_options()") {
251                    offenders.push(format!("{}:{}", path.display(), number + 1));
252                }
253            }
254        }
255        assert!(
256            offenders.is_empty(),
257            "use rumdl_parser_options_for(<text>) instead of rumdl_parser_options() at: {offenders:#?}"
258        );
259    }
260
261    #[test]
262    fn attribute_candidates_match_the_legacy_line_walk() {
263        let mut lines = Vec::new();
264        for prefix in ["", ">", "> ", "\t", " > \t", "\x0b", "\x0c", "\u{a0}"] {
265            for body in ["{}", "{#a}", "{ }", "a{}", "{}a", "{", "}", "=", "-", "---", "===", ""] {
266                lines.push(format!("{prefix}{body}"));
267            }
268        }
269        for a in &lines {
270            for b in &lines {
271                for ending in ["\n", "\r\n", "\r"] {
272                    for source in [format!("{a}{ending}{b}"), format!("x\n{a}{ending}{b}\r\n")] {
273                        assert_eq!(
274                            has_setext_attribute_line(&source),
275                            legacy_has_setext_attribute_line(&source),
276                            "{source:?}"
277                        );
278                    }
279                }
280            }
281        }
282    }
283
284    proptest::proptest! {
285        #[test]
286        fn attribute_candidates_match_generated_line_walk(
287            pieces in proptest::collection::vec(
288                proptest::sample::select(vec!["{", "}", "\n", "\r", "\r\n", ">", " ", "\t", "\x0b", "\x0c", "é", "\u{a0}", "=", "-", "--", "===", "abc", "\"", "{}"]),
289                0..100,
290            )
291        ) {
292            let source = pieces.concat();
293            proptest::prop_assert_eq!(has_setext_attribute_line(&source), legacy_has_setext_attribute_line(&source), "{:?}", source);
294        }
295    }
296}
rumdl_lib/utils/parser_options.rs

rumdl_lib/utils/
parser_options.rs
