Skip to main content

fatou_parser/documentation/
parser.rs

1use rowan::{GreenNodeBuilder, Language as _, TextRange, TextSize};
2
3use super::syntax::{DocumentationLanguage, SyntaxKind, SyntaxNode};
4
5/// A recoverable documentation parse problem.
6#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7pub struct ParseDiagnostic {
8    pub kind: DiagnosticKind,
9    pub range: TextRange,
10}
11
12/// Kinds of malformed explicit Markdown syntax reported by the parser.
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum DiagnosticKind {
15    /// A backtick or tilde fence opened but did not close.
16    UnclosedFence,
17}
18
19/// A lossless documentation CST and its recoverable diagnostics.
20#[derive(Debug, Clone)]
21pub struct ParseOutput {
22    pub cst: SyntaxNode,
23    pub diagnostics: Vec<ParseDiagnostic>,
24}
25
26/// Parse decoded documentation text as Julia-flavored Markdown.
27pub fn parse(text: &str) -> ParseOutput {
28    assert!(
29        text.len() <= u32::MAX as usize,
30        "documentation exceeds Rowan's 4 GiB limit"
31    );
32    Parser::new(text).parse()
33}
34
35/// Reconstruct the exact parser input from a documentation CST.
36pub fn reconstruct(root: &SyntaxNode) -> String {
37    root.text().to_string()
38}
39
40#[derive(Debug, Clone, Copy)]
41struct Line {
42    start: usize,
43    content_end: usize,
44    end: usize,
45}
46
47impl Line {
48    fn content(self, text: &str) -> &str {
49        &text[self.start..self.content_end]
50    }
51
52    fn is_blank(self, text: &str) -> bool {
53        self.content(text).trim_matches([' ', '\t']).is_empty()
54    }
55}
56
57#[derive(Debug, Clone, Copy)]
58struct FenceOpen {
59    marker_start: usize,
60    marker_end: usize,
61    marker: u8,
62    length: usize,
63}
64
65struct Parser<'a> {
66    text: &'a str,
67    lines: Vec<Line>,
68    builder: GreenNodeBuilder<'static>,
69    diagnostics: Vec<ParseDiagnostic>,
70}
71
72impl<'a> Parser<'a> {
73    fn new(text: &'a str) -> Self {
74        Self {
75            text,
76            lines: lines(text),
77            builder: GreenNodeBuilder::new(),
78            diagnostics: Vec::new(),
79        }
80    }
81
82    fn parse(mut self) -> ParseOutput {
83        self.start(SyntaxKind::ROOT);
84        let mut line = 0;
85        while line < self.lines.len() {
86            if self.lines[line].is_blank(self.text) {
87                let current = self.lines[line];
88                self.token(SyntaxKind::BLANK_LINE, current.start, current.end);
89                line += 1;
90                continue;
91            }
92            line = self.parse_block(line);
93        }
94        self.finish();
95        let green = self.builder.finish();
96        ParseOutput {
97            cst: SyntaxNode::new_root(green),
98            diagnostics: self.diagnostics,
99        }
100    }
101
102    fn parse_block(&mut self, line: usize) -> usize {
103        if let Some(expression_end) = block_interpolation(self.lines[line], self.text) {
104            return self.emit_block_interpolation(line, expression_end);
105        }
106        if let Some(open) = fence_open(self.lines[line], self.text) {
107            if let Some(close) = self.find_fence_close(line + 1, open) {
108                return self.emit_fence(line, close, open);
109            }
110            self.diagnostics.push(ParseDiagnostic {
111                kind: DiagnosticKind::UnclosedFence,
112                range: range(open.marker_start, open.marker_end),
113            });
114            return self.emit_paragraph(line);
115        }
116        if atx_heading(self.lines[line], self.text).is_some() {
117            return self.emit_atx_heading(line);
118        }
119        if line + 1 < self.lines.len()
120            && contains_unescaped_pipe(self.lines[line].content(self.text))
121            && table_alignment(self.lines[line + 1].content(self.text))
122        {
123            return self.emit_table(line);
124        }
125        if block_quote_content(self.lines[line], self.text).is_some() {
126            return self.emit_block_quote(line);
127        }
128        if admonition_header(self.lines[line], self.text).is_some() {
129            return self.emit_admonition(line);
130        }
131        if footnote_header(self.lines[line], self.text).is_some() {
132            return self.emit_footnote(line);
133        }
134        if list_marker(self.lines[line], self.text).is_some() {
135            return self.emit_list(line);
136        }
137        if indentation(self.lines[line].content(self.text)) >= 4
138            || self.lines[line].content(self.text).starts_with('\t')
139        {
140            return self.emit_indented_code(line);
141        }
142        if thematic_break(self.lines[line].content(self.text)) {
143            return self.emit_simple_line(line, SyntaxKind::THEMATIC_BREAK);
144        }
145        if math_block(self.lines[line], self.text).is_some() {
146            return self.emit_math_block(line);
147        }
148        // Last before the paragraph fallback, as in Julia's flavor: an
149        // underline only makes a heading out of a line no other block claimed,
150        // so `- item` or `> quoted` above `---` stays a list or a quote
151        // followed by a thematic break.
152        if line + 1 < self.lines.len()
153            && !self.lines[line].is_blank(self.text)
154            && setext_level(self.lines[line + 1], self.text).is_some()
155        {
156            return self.emit_setext_heading(line);
157        }
158        self.emit_paragraph(line)
159    }
160
161    fn find_fence_close(&self, from: usize, open: FenceOpen) -> Option<usize> {
162        (from..self.lines.len()).find(|&line| {
163            let content = self.lines[line].content(self.text);
164            let indent = indentation(content);
165            if indent > 3 {
166                return false;
167            }
168            let bytes = content.as_bytes();
169            let mut end = indent;
170            while end < bytes.len() && bytes[end] == open.marker {
171                end += 1;
172            }
173            end - indent == open.length && content[end..].trim_matches([' ', '\t']).is_empty()
174        })
175    }
176
177    fn emit_fence(&mut self, open_line: usize, close_line: usize, open: FenceOpen) -> usize {
178        let opening = self.lines[open_line];
179        let closing = self.lines[close_line];
180        let info = self.text[open.marker_end..opening.content_end].trim();
181        let node_kind = if info == "math" {
182            SyntaxKind::MATH_BLOCK
183        } else {
184            SyntaxKind::FENCED_CODE_BLOCK
185        };
186        self.start(node_kind);
187        self.token(SyntaxKind::WHITESPACE, opening.start, open.marker_start);
188        self.token(SyntaxKind::FENCE, open.marker_start, open.marker_end);
189        self.token(
190            SyntaxKind::INFO_STRING,
191            open.marker_end,
192            opening.content_end,
193        );
194        self.token(SyntaxKind::NEWLINE, opening.content_end, opening.end);
195
196        let body_start = opening.end;
197        let body_end = closing.start;
198        self.token(
199            if node_kind == SyntaxKind::MATH_BLOCK {
200                SyntaxKind::MATH_CONTENT
201            } else {
202                SyntaxKind::CODE_CONTENT
203            },
204            body_start,
205            body_end,
206        );
207
208        let close_content = closing.content(self.text);
209        let close_indent = indentation(close_content);
210        let marker_start = closing.start + close_indent;
211        let mut marker_end = marker_start;
212        while marker_end < closing.content_end && self.text.as_bytes()[marker_end] == open.marker {
213            marker_end += 1;
214        }
215        self.token(SyntaxKind::WHITESPACE, closing.start, marker_start);
216        self.token(SyntaxKind::FENCE, marker_start, marker_end);
217        self.token(SyntaxKind::MARKER, marker_end, closing.content_end);
218        self.token(SyntaxKind::NEWLINE, closing.content_end, closing.end);
219        self.finish();
220        close_line + 1
221    }
222
223    fn emit_atx_heading(&mut self, line_index: usize) -> usize {
224        let line = self.lines[line_index];
225        let (marker_start, marker_end, content_start, content_end) =
226            atx_heading(line, self.text).expect("checked heading");
227        self.start(SyntaxKind::ATX_HEADING);
228        self.token(SyntaxKind::WHITESPACE, line.start, marker_start);
229        self.token(SyntaxKind::HEADING_MARKER, marker_start, marker_end);
230        self.token(SyntaxKind::WHITESPACE, marker_end, content_start);
231        self.parse_inlines(content_start, content_end);
232        self.token(SyntaxKind::MARKER, content_end, line.content_end);
233        self.token(SyntaxKind::NEWLINE, line.content_end, line.end);
234        self.finish();
235        line_index + 1
236    }
237
238    fn emit_setext_heading(&mut self, line_index: usize) -> usize {
239        let content = self.lines[line_index];
240        let underline = self.lines[line_index + 1];
241        self.start(SyntaxKind::SETEXT_HEADING);
242        self.emit_trimmed_inlines(content);
243        self.token(SyntaxKind::NEWLINE, content.content_end, content.end);
244        let indent = indentation(underline.content(self.text));
245        self.token(
246            SyntaxKind::WHITESPACE,
247            underline.start,
248            underline.start + indent,
249        );
250        self.token(
251            SyntaxKind::HEADING_MARKER,
252            underline.start + indent,
253            underline.content_end,
254        );
255        self.token(SyntaxKind::NEWLINE, underline.content_end, underline.end);
256        self.finish();
257        line_index + 2
258    }
259
260    fn emit_table(&mut self, line_index: usize) -> usize {
261        self.start(SyntaxKind::TABLE);
262        let mut line = line_index;
263        while line < self.lines.len()
264            && !self.lines[line].is_blank(self.text)
265            && contains_unescaped_pipe(self.lines[line].content(self.text))
266        {
267            self.emit_table_row(self.lines[line]);
268            line += 1;
269        }
270        self.finish();
271        line
272    }
273
274    fn emit_table_row(&mut self, line: Line) {
275        self.start(SyntaxKind::TABLE_ROW);
276        let mut cursor = line.start;
277        let mut cell_start = line.start;
278        while cursor < line.content_end {
279            if self.text.as_bytes()[cursor] == b'|'
280                && (cursor == line.start || self.text.as_bytes()[cursor - 1] != b'\\')
281            {
282                if cell_start < cursor {
283                    self.start(SyntaxKind::TABLE_CELL);
284                    self.parse_inlines(cell_start, cursor);
285                    self.finish();
286                }
287                self.token(SyntaxKind::MARKER, cursor, cursor + 1);
288                cursor += 1;
289                cell_start = cursor;
290            } else {
291                cursor += self.text[cursor..].chars().next().unwrap().len_utf8();
292            }
293        }
294        if cell_start < line.content_end {
295            self.start(SyntaxKind::TABLE_CELL);
296            self.parse_inlines(cell_start, line.content_end);
297            self.finish();
298        }
299        self.token(SyntaxKind::NEWLINE, line.content_end, line.end);
300        self.finish();
301    }
302
303    fn emit_block_quote(&mut self, line_index: usize) -> usize {
304        self.start(SyntaxKind::BLOCK_QUOTE);
305        let mut line_index = line_index;
306        while line_index < self.lines.len() {
307            let line = self.lines[line_index];
308            let Some((_marker_start, content_start)) = block_quote_content(line, self.text) else {
309                break;
310            };
311            if content_start == line.content_end {
312                self.token(SyntaxKind::BLANK_LINE, line.start, line.end);
313                line_index += 1;
314                continue;
315            }
316
317            if list_marker_range(self.text, content_start, line.content_end).is_some() {
318                self.start(SyntaxKind::LIST);
319                while line_index < self.lines.len() {
320                    let item_line = self.lines[line_index];
321                    let Some((quote_marker, quote_content)) =
322                        block_quote_content(item_line, self.text)
323                    else {
324                        break;
325                    };
326                    let Some((list_start, list_end, item_content)) =
327                        list_marker_range(self.text, quote_content, item_line.content_end)
328                    else {
329                        break;
330                    };
331                    self.start(SyntaxKind::LIST_ITEM);
332                    self.emit_quote_prefix(item_line, quote_marker, quote_content);
333                    self.token(SyntaxKind::WHITESPACE, quote_content, list_start);
334                    self.token(SyntaxKind::LIST_MARKER, list_start, list_end);
335                    self.token(SyntaxKind::WHITESPACE, list_end, item_content);
336                    self.parse_inlines(item_content, item_line.content_end);
337                    self.token(SyntaxKind::NEWLINE, item_line.content_end, item_line.end);
338                    self.finish();
339                    line_index += 1;
340                }
341                self.finish();
342                continue;
343            }
344
345            self.start(SyntaxKind::PARAGRAPH);
346            while line_index < self.lines.len() {
347                let paragraph_line = self.lines[line_index];
348                let Some((quote_marker, quote_content)) =
349                    block_quote_content(paragraph_line, self.text)
350                else {
351                    break;
352                };
353                if quote_content == paragraph_line.content_end
354                    || list_marker_range(self.text, quote_content, paragraph_line.content_end)
355                        .is_some()
356                {
357                    break;
358                }
359                self.emit_quote_prefix(paragraph_line, quote_marker, quote_content);
360                self.parse_inlines(quote_content, paragraph_line.content_end);
361                self.token(
362                    SyntaxKind::SOFT_BREAK,
363                    paragraph_line.content_end,
364                    paragraph_line.end,
365                );
366                line_index += 1;
367            }
368            self.finish();
369        }
370        self.finish();
371        line_index
372    }
373
374    fn emit_quote_prefix(&mut self, line: Line, marker_start: usize, content_start: usize) {
375        self.token(SyntaxKind::WHITESPACE, line.start, marker_start);
376        self.token(SyntaxKind::QUOTE_MARKER, marker_start, marker_start + 1);
377        self.token(SyntaxKind::WHITESPACE, marker_start + 1, content_start);
378    }
379
380    fn emit_admonition(&mut self, line_index: usize) -> usize {
381        let header = self.lines[line_index];
382        let (marker_start, category, title) =
383            admonition_header(header, self.text).expect("checked admonition");
384        self.start(SyntaxKind::ADMONITION);
385        self.token(SyntaxKind::WHITESPACE, header.start, marker_start);
386        self.token(SyntaxKind::MARKER, marker_start, marker_start + 3);
387        self.token(SyntaxKind::WHITESPACE, marker_start + 3, category.start);
388        self.token(
389            SyntaxKind::ADMONITION_CATEGORY,
390            category.start,
391            category.end,
392        );
393        if let Some(title) = title {
394            self.token(SyntaxKind::WHITESPACE, category.end, title.start);
395            self.token(SyntaxKind::ADMONITION_TITLE, title.start, title.end);
396            self.token(SyntaxKind::WHITESPACE, title.end, header.content_end);
397        } else {
398            self.token(SyntaxKind::WHITESPACE, category.end, header.content_end);
399        }
400        self.token(SyntaxKind::NEWLINE, header.content_end, header.end);
401        let next = self.emit_indented_container_blocks(line_index + 1);
402        self.finish();
403        next
404    }
405
406    fn emit_indented_container_blocks(&mut self, mut line_index: usize) -> usize {
407        while line_index < self.lines.len() {
408            let line = self.lines[line_index];
409            if line.is_blank(self.text) {
410                self.token(SyntaxKind::BLANK_LINE, line.start, line.end);
411                line_index += 1;
412                continue;
413            }
414            let Some(content_start) = indented_content_start(line, self.text) else {
415                break;
416            };
417            if let Some((marker_start, marker_end, _)) =
418                list_marker_range(self.text, content_start, line.content_end)
419            {
420                let ordered = self.text.as_bytes()[marker_start..marker_end]
421                    .first()
422                    .is_some_and(u8::is_ascii_digit);
423                line_index = self.emit_list_level(line_index, marker_start - line.start, ordered);
424                continue;
425            }
426
427            self.start(SyntaxKind::PARAGRAPH);
428            while line_index < self.lines.len() {
429                let paragraph_line = self.lines[line_index];
430                let Some(content_start) = indented_content_start(paragraph_line, self.text) else {
431                    break;
432                };
433                if list_marker_range(self.text, content_start, paragraph_line.content_end).is_some()
434                {
435                    break;
436                }
437                self.token(SyntaxKind::WHITESPACE, paragraph_line.start, content_start);
438                self.parse_inlines(content_start, paragraph_line.content_end);
439                self.token(
440                    SyntaxKind::SOFT_BREAK,
441                    paragraph_line.content_end,
442                    paragraph_line.end,
443                );
444                line_index += 1;
445                if line_index >= self.lines.len() || self.lines[line_index].is_blank(self.text) {
446                    break;
447                }
448            }
449            self.finish();
450        }
451        line_index
452    }
453
454    fn emit_footnote(&mut self, line_index: usize) -> usize {
455        let header = self.lines[line_index];
456        let (label_start, label_end, content_start) =
457            footnote_header(header, self.text).expect("checked footnote");
458        self.start(SyntaxKind::FOOTNOTE_DEFINITION);
459        self.token(SyntaxKind::WHITESPACE, header.start, label_start);
460        self.token(SyntaxKind::FOOTNOTE_LABEL, label_start, label_end);
461        self.token(SyntaxKind::WHITESPACE, label_end, content_start);
462        self.start(SyntaxKind::PARAGRAPH);
463        self.parse_inlines(content_start, header.content_end);
464        self.token(SyntaxKind::SOFT_BREAK, header.content_end, header.end);
465
466        let mut next = line_index + 1;
467        while next < self.lines.len() {
468            let line = self.lines[next];
469            if line.is_blank(self.text) {
470                break;
471            }
472            let Some(content_start) = indented_content_start(line, self.text) else {
473                break;
474            };
475            self.token(SyntaxKind::WHITESPACE, line.start, content_start);
476            self.parse_inlines(content_start, line.content_end);
477            self.token(SyntaxKind::SOFT_BREAK, line.content_end, line.end);
478            next += 1;
479        }
480        self.finish();
481        next = self.emit_indented_container_blocks(next);
482        self.finish();
483        next
484    }
485
486    fn emit_list(&mut self, line_index: usize) -> usize {
487        let (marker_start, marker_end, _) =
488            list_marker(self.lines[line_index], self.text).expect("checked list");
489        let base_indent = marker_start - self.lines[line_index].start;
490        let ordered = self.text.as_bytes()[marker_start..marker_end]
491            .first()
492            .is_some_and(u8::is_ascii_digit);
493        self.emit_list_level(line_index, base_indent, ordered)
494    }
495
496    fn emit_list_level(
497        &mut self,
498        mut line_index: usize,
499        base_indent: usize,
500        ordered: bool,
501    ) -> usize {
502        self.start(SyntaxKind::LIST);
503        while line_index < self.lines.len() {
504            let line = self.lines[line_index];
505            let Some((marker_start, marker_end, content_start)) = list_marker_any(line, self.text)
506            else {
507                break;
508            };
509            let indent = marker_start - line.start;
510            let item_ordered = self.text.as_bytes()[marker_start..marker_end]
511                .first()
512                .is_some_and(u8::is_ascii_digit);
513            if indent != base_indent || item_ordered != ordered {
514                break;
515            }
516            self.start(SyntaxKind::LIST_ITEM);
517            self.token(SyntaxKind::WHITESPACE, line.start, marker_start);
518            self.token(SyntaxKind::LIST_MARKER, marker_start, marker_end);
519            self.token(SyntaxKind::WHITESPACE, marker_end, content_start);
520            self.parse_inlines(content_start, line.content_end);
521            self.token(SyntaxKind::NEWLINE, line.content_end, line.end);
522            line_index += 1;
523            let item_indent = marker_end - line.start + usize::from(content_start > marker_end);
524            let mut after_blank = false;
525
526            while line_index < self.lines.len() {
527                let continuation = self.lines[line_index];
528                if continuation.is_blank(self.text) {
529                    let Some(next) = self.lines.get(line_index + 1).copied() else {
530                        break;
531                    };
532                    if next.is_blank(self.text) {
533                        break;
534                    }
535                    let continues_list = list_marker_any(next, self.text)
536                        .is_some_and(|(start, _, _)| start - next.start >= base_indent);
537                    let continues_item = indentation(next.content(self.text)) > base_indent;
538                    if !continues_list && !continues_item {
539                        break;
540                    }
541                    self.token(SyntaxKind::BLANK_LINE, continuation.start, continuation.end);
542                    line_index += 1;
543                    after_blank = true;
544                    continue;
545                }
546                if let Some((nested_start, nested_end, _)) =
547                    list_marker_any(continuation, self.text)
548                {
549                    let nested_indent = nested_start - continuation.start;
550                    let nested_ordered = self.text.as_bytes()[nested_start..nested_end]
551                        .first()
552                        .is_some_and(u8::is_ascii_digit);
553                    if nested_indent > base_indent {
554                        line_index =
555                            self.emit_list_level(line_index, nested_indent, nested_ordered);
556                        after_blank = false;
557                        continue;
558                    }
559                    break;
560                }
561                let indent = indentation(continuation.content(self.text));
562                if indent <= base_indent {
563                    break;
564                }
565                if after_blank
566                    && indented_content_start_after(continuation, self.text, item_indent).is_some()
567                {
568                    line_index = self.emit_indented_code_after(line_index, item_indent);
569                    after_blank = false;
570                    continue;
571                }
572                self.token(
573                    SyntaxKind::WHITESPACE,
574                    continuation.start,
575                    continuation.start + indent,
576                );
577                self.parse_inlines(continuation.start + indent, continuation.content_end);
578                self.token(
579                    SyntaxKind::NEWLINE,
580                    continuation.content_end,
581                    continuation.end,
582                );
583                line_index += 1;
584                after_blank = false;
585            }
586            self.finish();
587        }
588        self.finish();
589        line_index
590    }
591
592    fn emit_indented_code(&mut self, line_index: usize) -> usize {
593        self.emit_indented_code_after(line_index, 0)
594    }
595
596    fn emit_indented_code_after(&mut self, line_index: usize, prefix: usize) -> usize {
597        self.start(SyntaxKind::INDENTED_CODE_BLOCK);
598        let mut line_index = line_index;
599        while line_index < self.lines.len() {
600            let line = self.lines[line_index];
601            if line.is_blank(self.text) {
602                self.token(SyntaxKind::CODE_CONTENT, line.start, line.end);
603                line_index += 1;
604                continue;
605            }
606            let Some(content_start) = indented_content_start_after(line, self.text, prefix) else {
607                break;
608            };
609            self.token(SyntaxKind::WHITESPACE, line.start, content_start);
610            self.token(SyntaxKind::CODE_CONTENT, content_start, line.content_end);
611            self.token(SyntaxKind::NEWLINE, line.content_end, line.end);
612            line_index += 1;
613        }
614        self.finish();
615        line_index
616    }
617
618    fn emit_math_block(&mut self, line_index: usize) -> usize {
619        let line = self.lines[line_index];
620        let (open, content_start, content_end, close) =
621            math_block(line, self.text).expect("checked math block");
622        self.start(SyntaxKind::MATH_BLOCK);
623        self.token(SyntaxKind::WHITESPACE, line.start, open);
624        self.token(SyntaxKind::MARKER, open, content_start);
625        self.token(SyntaxKind::MATH_CONTENT, content_start, content_end);
626        self.token(SyntaxKind::MARKER, content_end, close);
627        self.token(SyntaxKind::WHITESPACE, close, line.content_end);
628        self.token(SyntaxKind::NEWLINE, line.content_end, line.end);
629        self.finish();
630        line_index + 1
631    }
632
633    fn emit_block_interpolation(&mut self, line_index: usize, expression_end: usize) -> usize {
634        let line = self.lines[line_index];
635        self.start(SyntaxKind::INTERPOLATION);
636        self.token(SyntaxKind::MARKER, line.start, line.start + 1);
637        self.token(SyntaxKind::TEXT, line.start + 1, expression_end);
638        self.token(SyntaxKind::WHITESPACE, expression_end, line.content_end);
639        self.token(SyntaxKind::NEWLINE, line.content_end, line.end);
640        self.finish();
641        line_index + 1
642    }
643
644    fn emit_simple_line(&mut self, line_index: usize, kind: SyntaxKind) -> usize {
645        let line = self.lines[line_index];
646        self.start(kind);
647        self.token(SyntaxKind::MARKER, line.start, line.content_end);
648        self.token(SyntaxKind::NEWLINE, line.content_end, line.end);
649        self.finish();
650        line_index + 1
651    }
652
653    fn emit_paragraph(&mut self, line_index: usize) -> usize {
654        self.start(SyntaxKind::PARAGRAPH);
655        let mut current = line_index;
656        while current < self.lines.len() && !self.lines[current].is_blank(self.text) {
657            if current > line_index && self.is_breaking_block(current) {
658                break;
659            }
660            let line = self.lines[current];
661            let hard_break = line.content_end > line.start
662                && self.text.as_bytes()[line.content_end - 1] == b'\\'
663                && line.end > line.content_end;
664            let inline_end = if hard_break {
665                line.content_end - 1
666            } else {
667                line.content_end
668            };
669            self.parse_inlines(line.start, inline_end);
670            if hard_break {
671                self.start(SyntaxKind::HARD_BREAK);
672                self.token(SyntaxKind::ESCAPE, inline_end, line.content_end);
673                self.token(SyntaxKind::NEWLINE, line.content_end, line.end);
674                self.finish();
675            } else {
676                self.token(SyntaxKind::SOFT_BREAK, line.content_end, line.end);
677            }
678            current += 1;
679        }
680        self.finish();
681        current
682    }
683
684    fn is_breaking_block(&self, line: usize) -> bool {
685        let current = self.lines[line];
686        fence_open(current, self.text).is_some()
687            || atx_heading(current, self.text).is_some()
688            || block_quote_content(current, self.text).is_some()
689            || admonition_header(current, self.text).is_some()
690            || footnote_header(current, self.text).is_some()
691            || list_marker(current, self.text).is_some()
692    }
693
694    fn emit_trimmed_inlines(&mut self, line: Line) {
695        let raw = line.content(self.text);
696        let leading = raw.len() - raw.trim_start().len();
697        let trailing = raw.trim_end().len();
698        self.token(SyntaxKind::WHITESPACE, line.start, line.start + leading);
699        self.parse_inlines(line.start + leading, line.start + trailing);
700        self.token(
701            SyntaxKind::WHITESPACE,
702            line.start + trailing,
703            line.content_end,
704        );
705    }
706
707    fn parse_inlines(&mut self, start: usize, end: usize) {
708        let mut cursor = start;
709        let mut text_start = start;
710        while cursor < end {
711            let byte = self.text.as_bytes()[cursor];
712            let trigger = matches!(byte, b'\\' | b'!' | b'[' | b'<' | b'*' | b'_' | b'`' | b'$')
713                || (byte == b'-' && cursor + 1 < end && self.text.as_bytes()[cursor + 1] == b'-');
714            if !trigger {
715                cursor += self.text[cursor..end].chars().next().unwrap().len_utf8();
716                continue;
717            }
718
719            // Inline helpers emit their nodes immediately. Flush the text that
720            // precedes a possible opener first so Rowan sees source order.
721            self.token(SyntaxKind::TEXT, text_start, cursor);
722            let opener = cursor;
723            let parsed = match self.text.as_bytes()[cursor] {
724                b'\\' => self.emit_escape(cursor, end),
725                b'!' if cursor + 1 < end && self.text.as_bytes()[cursor + 1] == b'[' => {
726                    self.emit_image(cursor, end)
727                }
728                b'[' => self.emit_link_or_footnote(cursor, end),
729                b'<' => self.emit_autolink(cursor, end),
730                b'*' | b'_' => self.emit_emphasis(cursor, end),
731                b'`' => self.emit_backticks(cursor, end),
732                b'$' => self.emit_dollar(cursor, end),
733                b'-' if cursor + 1 < end && self.text.as_bytes()[cursor + 1] == b'-' => {
734                    if cursor + 2 < end && self.text.as_bytes()[cursor + 2] == b'-' {
735                        Some((SyntaxKind::EM_DASH, cursor + 3))
736                    } else {
737                        Some((SyntaxKind::EN_DASH, cursor + 2))
738                    }
739                }
740                _ => None,
741            };
742            let Some((kind, next)) = parsed else {
743                cursor += self.text[cursor..end].chars().next().unwrap().len_utf8();
744                text_start = opener;
745                continue;
746            };
747            if matches!(
748                kind,
749                SyntaxKind::ESCAPE | SyntaxKind::EN_DASH | SyntaxKind::EM_DASH
750            ) {
751                self.token(kind, cursor, next);
752            }
753            cursor = next;
754            text_start = cursor;
755        }
756        self.token(SyntaxKind::TEXT, text_start, end);
757    }
758
759    fn emit_escape(&mut self, start: usize, end: usize) -> Option<(SyntaxKind, usize)> {
760        let next = start + 1;
761        if next >= end {
762            return None;
763        }
764        let ch = self.text[next..end].chars().next()?;
765        "\\`*_#+-.!{}[]()$"
766            .contains(ch)
767            .then_some((SyntaxKind::ESCAPE, next + ch.len_utf8()))
768    }
769
770    fn emit_image(&mut self, start: usize, end: usize) -> Option<(SyntaxKind, usize)> {
771        let label_end = find_closing(self.text, start + 2, end, b'[', b']')?;
772        let mut open = label_end + 1;
773        while open < end && self.text.as_bytes()[open].is_ascii_whitespace() {
774            open += 1;
775        }
776        if open >= end || self.text.as_bytes()[open] != b'(' {
777            return None;
778        }
779        let destination_end = find_closing(self.text, open + 1, end, b'(', b')')?;
780        self.start(SyntaxKind::IMAGE);
781        self.token(SyntaxKind::MARKER, start, start + 2);
782        self.token(SyntaxKind::TEXT, start + 2, label_end);
783        self.token(SyntaxKind::MARKER, label_end, open + 1);
784        self.token(SyntaxKind::LINK_DESTINATION, open + 1, destination_end);
785        self.token(SyntaxKind::MARKER, destination_end, destination_end + 1);
786        self.finish();
787        Some((SyntaxKind::IMAGE, destination_end + 1))
788    }
789
790    fn emit_link_or_footnote(&mut self, start: usize, end: usize) -> Option<(SyntaxKind, usize)> {
791        if start + 2 < end && self.text.as_bytes()[start + 1] == b'^' {
792            let close = self.text[start + 2..end].find(']')? + start + 2;
793            if self.text[start + 2..close]
794                .chars()
795                .all(|ch| ch.is_alphanumeric() || ch == '_')
796            {
797                self.start(SyntaxKind::FOOTNOTE_REFERENCE);
798                self.token(SyntaxKind::MARKER, start, start + 2);
799                self.token(SyntaxKind::FOOTNOTE_LABEL, start + 2, close);
800                self.token(SyntaxKind::MARKER, close, close + 1);
801                self.finish();
802                return Some((SyntaxKind::FOOTNOTE_REFERENCE, close + 1));
803            }
804        }
805
806        let label_end = find_closing(self.text, start + 1, end, b'[', b']')?;
807        let open = label_end + 1;
808        if open >= end || self.text.as_bytes()[open] != b'(' {
809            return None;
810        }
811        let destination_end = find_closing(self.text, open + 1, end, b'(', b')')?;
812        self.start(SyntaxKind::LINK);
813        self.token(SyntaxKind::MARKER, start, start + 1);
814        self.parse_inlines(start + 1, label_end);
815        self.token(SyntaxKind::MARKER, label_end, open + 1);
816        self.token(SyntaxKind::LINK_DESTINATION, open + 1, destination_end);
817        self.token(SyntaxKind::MARKER, destination_end, destination_end + 1);
818        self.finish();
819        Some((SyntaxKind::LINK, destination_end + 1))
820    }
821
822    fn emit_autolink(&mut self, start: usize, end: usize) -> Option<(SyntaxKind, usize)> {
823        let close = self.text[start + 1..end].find('>')? + start + 1;
824        let destination = &self.text[start + 1..close];
825        if !is_autolink(destination) {
826            return None;
827        }
828        self.start(SyntaxKind::AUTOLINK);
829        self.token(SyntaxKind::MARKER, start, start + 1);
830        self.token(SyntaxKind::LINK_DESTINATION, start + 1, close);
831        self.token(SyntaxKind::MARKER, close, close + 1);
832        self.finish();
833        Some((SyntaxKind::AUTOLINK, close + 1))
834    }
835
836    fn emit_emphasis(&mut self, start: usize, end: usize) -> Option<(SyntaxKind, usize)> {
837        let byte = self.text.as_bytes()[start];
838        let length = if start + 1 < end && self.text.as_bytes()[start + 1] == byte {
839            2
840        } else {
841            1
842        };
843        let delimiter = &self.text[start..start + length];
844        let close = self.text[start + length..end].find(delimiter)? + start + length;
845        if close == start + length {
846            return None;
847        }
848        self.start(if length == 2 {
849            SyntaxKind::STRONG
850        } else {
851            SyntaxKind::EMPHASIS
852        });
853        self.token(SyntaxKind::MARKER, start, start + length);
854        self.parse_inlines(start + length, close);
855        self.token(SyntaxKind::MARKER, close, close + length);
856        self.finish();
857        Some((SyntaxKind::EMPHASIS, close + length))
858    }
859
860    fn emit_backticks(&mut self, start: usize, end: usize) -> Option<(SyntaxKind, usize)> {
861        let mut open_end = start;
862        while open_end < end && self.text.as_bytes()[open_end] == b'`' {
863            open_end += 1;
864        }
865        let delimiter = &self.text[start..open_end];
866        let close = self.text[open_end..end].find(delimiter)? + open_end;
867        let kind = if delimiter.len().is_multiple_of(2) {
868            SyntaxKind::INLINE_MATH
869        } else {
870            SyntaxKind::INLINE_CODE
871        };
872        self.start(kind);
873        self.token(SyntaxKind::MARKER, start, open_end);
874        self.token(
875            if kind == SyntaxKind::INLINE_MATH {
876                SyntaxKind::MATH_CONTENT
877            } else {
878                SyntaxKind::CODE_CONTENT
879            },
880            open_end,
881            close,
882        );
883        self.token(SyntaxKind::MARKER, close, close + delimiter.len());
884        self.finish();
885        Some((kind, close + delimiter.len()))
886    }
887
888    fn emit_dollar(&mut self, start: usize, end: usize) -> Option<(SyntaxKind, usize)> {
889        if let Some(relative) = self.text[start + 1..end].find('$') {
890            let close = start + 1 + relative;
891            self.start(SyntaxKind::INLINE_MATH);
892            self.token(SyntaxKind::MARKER, start, start + 1);
893            self.token(SyntaxKind::MATH_CONTENT, start + 1, close);
894            self.token(SyntaxKind::MARKER, close, close + 1);
895            self.finish();
896            return Some((SyntaxKind::INLINE_MATH, close + 1));
897        }
898        let expression_end = interpolation_end(self.text, start, end)?;
899        self.start(SyntaxKind::INTERPOLATION);
900        self.token(SyntaxKind::MARKER, start, start + 1);
901        self.token(SyntaxKind::TEXT, start + 1, expression_end);
902        self.finish();
903        Some((SyntaxKind::INTERPOLATION, expression_end))
904    }
905
906    fn start(&mut self, kind: SyntaxKind) {
907        self.builder
908            .start_node(DocumentationLanguage::kind_to_raw(kind));
909    }
910
911    fn finish(&mut self) {
912        self.builder.finish_node();
913    }
914
915    fn token(&mut self, kind: SyntaxKind, start: usize, end: usize) {
916        if start < end {
917            self.builder.token(
918                DocumentationLanguage::kind_to_raw(kind),
919                &self.text[start..end],
920            );
921        }
922    }
923}
924
925fn lines(text: &str) -> Vec<Line> {
926    let mut out = Vec::new();
927    let mut start = 0;
928    while start < text.len() {
929        let Some(relative) = text[start..].find('\n') else {
930            out.push(Line {
931                start,
932                content_end: text.len(),
933                end: text.len(),
934            });
935            break;
936        };
937        let newline = start + relative;
938        let content_end = if newline > start && text.as_bytes()[newline - 1] == b'\r' {
939            newline - 1
940        } else {
941            newline
942        };
943        out.push(Line {
944            start,
945            content_end,
946            end: newline + 1,
947        });
948        start = newline + 1;
949    }
950    out
951}
952
953fn indentation(text: &str) -> usize {
954    text.as_bytes()
955        .iter()
956        .take_while(|&&byte| byte == b' ')
957        .count()
958}
959
960fn indented_content_start(line: Line, text: &str) -> Option<usize> {
961    indented_content_start_after(line, text, 0)
962}
963
964fn indented_content_start_after(line: Line, text: &str, prefix: usize) -> Option<usize> {
965    let content = line.content(text);
966    if content.len() < prefix
967        || !content.as_bytes()[..prefix]
968            .iter()
969            .all(|&byte| byte == b' ')
970    {
971        return None;
972    }
973    let tail = &content[prefix..];
974    if tail.starts_with('\t') {
975        Some(line.start + prefix + 1)
976    } else {
977        (indentation(tail) >= 4).then_some(line.start + prefix + 4)
978    }
979}
980
981fn fence_open(line: Line, text: &str) -> Option<FenceOpen> {
982    let content = line.content(text);
983    let indent = indentation(content);
984    if indent > 3 {
985        return None;
986    }
987    let marker = *content.as_bytes().get(indent)?;
988    if marker != b'`' && marker != b'~' {
989        return None;
990    }
991    let mut end = indent;
992    while end < content.len() && content.as_bytes()[end] == marker {
993        end += 1;
994    }
995    if end - indent < 3 || content.as_bytes()[end..].contains(&marker) {
996        return None;
997    }
998    Some(FenceOpen {
999        marker_start: line.start + indent,
1000        marker_end: line.start + end,
1001        marker,
1002        length: end - indent,
1003    })
1004}
1005
1006fn atx_heading(line: Line, text: &str) -> Option<(usize, usize, usize, usize)> {
1007    let content = line.content(text);
1008    let indent = indentation(content);
1009    if indent > 3 {
1010        return None;
1011    }
1012    let bytes = content.as_bytes();
1013    let mut marker_end = indent;
1014    while marker_end < bytes.len() && bytes[marker_end] == b'#' {
1015        marker_end += 1;
1016    }
1017    let level = marker_end - indent;
1018    if !(1..=6).contains(&level)
1019        || (marker_end < bytes.len() && !bytes[marker_end].is_ascii_whitespace())
1020    {
1021        return None;
1022    }
1023    let mut content_start = marker_end;
1024    while content_start < bytes.len() && matches!(bytes[content_start], b' ' | b'\t') {
1025        content_start += 1;
1026    }
1027    let mut content_end = content.len();
1028    while content_end > content_start && matches!(bytes[content_end - 1], b' ' | b'\t') {
1029        content_end -= 1;
1030    }
1031    let trailing_end = content_end;
1032    while content_end > content_start && bytes[content_end - 1] == b'#' {
1033        content_end -= 1;
1034    }
1035    if content_end < trailing_end
1036        && content_end > content_start
1037        && bytes[content_end - 1].is_ascii_whitespace()
1038    {
1039        while content_end > content_start && bytes[content_end - 1].is_ascii_whitespace() {
1040            content_end -= 1;
1041        }
1042    } else {
1043        content_end = trailing_end;
1044    }
1045    Some((
1046        line.start + indent,
1047        line.start + marker_end,
1048        line.start + content_start,
1049        line.start + content_end,
1050    ))
1051}
1052
1053fn setext_level(line: Line, text: &str) -> Option<u8> {
1054    let trimmed = line.content(text).trim_matches([' ', '\t']);
1055    if trimmed.len() < 3 {
1056        return None;
1057    }
1058    let marker = trimmed.as_bytes()[0];
1059    if !matches!(marker, b'=' | b'-') || !trimmed.bytes().all(|byte| byte == marker) {
1060        return None;
1061    }
1062    Some(if marker == b'=' { 1 } else { 2 })
1063}
1064
1065fn block_quote_content(line: Line, text: &str) -> Option<(usize, usize)> {
1066    let content = line.content(text);
1067    let indent = indentation(content);
1068    if indent > 3 || content.as_bytes().get(indent) != Some(&b'>') {
1069        return None;
1070    }
1071    let marker = line.start + indent;
1072    let content = if text.as_bytes().get(marker + 1) == Some(&b' ') {
1073        marker + 2
1074    } else {
1075        marker + 1
1076    };
1077    Some((marker, content))
1078}
1079
1080fn admonition_header(
1081    line: Line,
1082    text: &str,
1083) -> Option<(
1084    usize,
1085    std::ops::Range<usize>,
1086    Option<std::ops::Range<usize>>,
1087)> {
1088    let content = line.content(text);
1089    let indent = indentation(content);
1090    if indent > 3 || !content[indent..].starts_with("!!! ") {
1091        return None;
1092    }
1093    let marker_start = line.start + indent;
1094    let rest_start = marker_start + 4;
1095    let rest = &text[rest_start..line.content_end];
1096    let category_len = rest
1097        .bytes()
1098        .take_while(|byte| byte.is_ascii_lowercase())
1099        .count();
1100    if category_len == 0 {
1101        return None;
1102    }
1103    let category = rest_start..rest_start + category_len;
1104    let tail = text[category.end..line.content_end].trim();
1105    let title = if tail.is_empty() {
1106        None
1107    } else if tail.starts_with('"') && tail.ends_with('"') && tail.len() >= 2 {
1108        // End at the closing quote: trailing spaces inside the token would
1109        // survive `Admonition::title`'s quote trimming.
1110        let offset = text[category.end..line.content_end].find('"')?;
1111        Some(category.end + offset..category.end + offset + tail.len())
1112    } else {
1113        return None;
1114    };
1115    Some((marker_start, category, title))
1116}
1117
1118fn footnote_header(line: Line, text: &str) -> Option<(usize, usize, usize)> {
1119    let content = line.content(text);
1120    let indent = indentation(content);
1121    if indent > 3 || !content[indent..].starts_with("[^") {
1122        return None;
1123    }
1124    let label_start = line.start + indent;
1125    let close = text[label_start + 2..line.content_end].find("]:")? + label_start + 2;
1126    if !text[label_start + 2..close]
1127        .chars()
1128        .all(|ch| ch.is_alphanumeric() || ch == '_')
1129    {
1130        return None;
1131    }
1132    let label_end = close + 2;
1133    let mut content_start = label_end;
1134    while content_start < line.content_end && text.as_bytes()[content_start].is_ascii_whitespace() {
1135        content_start += 1;
1136    }
1137    Some((label_start, label_end, content_start))
1138}
1139
1140fn list_marker(line: Line, text: &str) -> Option<(usize, usize, usize)> {
1141    let marker = list_marker_any(line, text)?;
1142    (marker.0 - line.start <= 3).then_some(marker)
1143}
1144
1145fn list_marker_any(line: Line, text: &str) -> Option<(usize, usize, usize)> {
1146    list_marker_range(text, line.start, line.content_end)
1147}
1148
1149fn list_marker_range(text: &str, start: usize, end_offset: usize) -> Option<(usize, usize, usize)> {
1150    let content = &text[start..end_offset];
1151    let indent = indentation(content);
1152    let bytes = content.as_bytes();
1153    let mut end = indent;
1154    if matches!(bytes.get(end), Some(b'*' | b'+' | b'-')) {
1155        end += 1;
1156    } else {
1157        while end < bytes.len() && bytes[end].is_ascii_digit() {
1158            end += 1;
1159        }
1160        if end == indent || !matches!(bytes.get(end), Some(b'.' | b')')) {
1161            return None;
1162        }
1163        end += 1;
1164    }
1165    if end < bytes.len() && !bytes[end].is_ascii_whitespace() {
1166        return None;
1167    }
1168    let mut content_start = end;
1169    while content_start < bytes.len() && bytes[content_start].is_ascii_whitespace() {
1170        content_start += 1;
1171    }
1172    Some((start + indent, start + end, start + content_start))
1173}
1174
1175fn thematic_break(content: &str) -> bool {
1176    let mut marker = None;
1177    let mut count = 0;
1178    for byte in content.bytes() {
1179        if byte.is_ascii_whitespace() {
1180            continue;
1181        }
1182        if !matches!(byte, b'*' | b'-') {
1183            return false;
1184        }
1185        if marker.is_some_and(|current| current != byte) {
1186            return false;
1187        }
1188        marker = Some(byte);
1189        count += 1;
1190    }
1191    count >= 3
1192}
1193
1194fn math_block(line: Line, text: &str) -> Option<(usize, usize, usize, usize)> {
1195    let raw = line.content(text);
1196    let leading = raw.len() - raw.trim_start().len();
1197    let trailing = raw.trim_end().len();
1198    let trimmed = &raw[leading..trailing];
1199    if trimmed.len() < 2 || !trimmed.starts_with('$') || !trimmed.ends_with('$') {
1200        return None;
1201    }
1202    let open = line.start + leading;
1203    let close = line.start + trailing;
1204    Some((open, open + 1, close - 1, close))
1205}
1206
1207fn block_interpolation(line: Line, text: &str) -> Option<usize> {
1208    if text.as_bytes().get(line.start) != Some(&b'$') {
1209        return None;
1210    }
1211    let end = interpolation_end(text, line.start, line.content_end)?;
1212    text[end..line.content_end]
1213        .trim_matches([' ', '\t'])
1214        .is_empty()
1215        .then_some(end)
1216}
1217
1218fn contains_unescaped_pipe(text: &str) -> bool {
1219    text.as_bytes()
1220        .iter()
1221        .enumerate()
1222        .any(|(index, &byte)| byte == b'|' && (index == 0 || text.as_bytes()[index - 1] != b'\\'))
1223}
1224
1225fn table_alignment(text: &str) -> bool {
1226    let trimmed = text.trim();
1227    let trimmed = trimmed.strip_prefix('|').unwrap_or(trimmed);
1228    let trimmed = trimmed.strip_suffix('|').unwrap_or(trimmed);
1229    let mut cells = 0;
1230    for cell in trimmed.split('|') {
1231        let cell = cell.trim();
1232        if cell.len() < 3 || !cell.bytes().all(|byte| matches!(byte, b'-' | b':')) {
1233            return false;
1234        }
1235        cells += 1;
1236    }
1237    cells > 0
1238}
1239
1240fn find_closing(text: &str, start: usize, end: usize, open: u8, close: u8) -> Option<usize> {
1241    let mut depth = 0usize;
1242    let mut cursor = start;
1243    while cursor < end {
1244        let byte = text.as_bytes()[cursor];
1245        if byte == b'\\' {
1246            cursor += 1;
1247            if cursor < end {
1248                cursor += text[cursor..end].chars().next()?.len_utf8();
1249            }
1250            continue;
1251        }
1252        if byte == open {
1253            depth += 1;
1254        } else if byte == close {
1255            if depth == 0 {
1256                return Some(cursor);
1257            }
1258            depth -= 1;
1259        }
1260        cursor += text[cursor..end].chars().next()?.len_utf8();
1261    }
1262    None
1263}
1264
1265fn interpolation_end(text: &str, start: usize, end: usize) -> Option<usize> {
1266    let first = start + 1;
1267    if first >= end || text.as_bytes()[first].is_ascii_whitespace() {
1268        return None;
1269    }
1270    if text.as_bytes()[first] == b'(' {
1271        return find_closing(text, first + 1, end, b'(', b')').map(|close| close + 1);
1272    }
1273    let mut cursor = first;
1274    let mut saw_name = false;
1275    while cursor < end {
1276        let ch = text[cursor..end].chars().next()?;
1277        if ch.is_alphanumeric() || ch == '_' || ch == '!' || ch == '?' || !ch.is_ascii() {
1278            saw_name = true;
1279            cursor += ch.len_utf8();
1280        } else if ch == '.' {
1281            let after_dot = cursor + 1;
1282            let Some(next) = text[after_dot..end].chars().next() else {
1283                break;
1284            };
1285            if next.is_alphanumeric() || next == '_' || !next.is_ascii() {
1286                cursor = after_dot;
1287            } else {
1288                break;
1289            }
1290        } else {
1291            break;
1292        }
1293    }
1294    if saw_name && cursor < end && text.as_bytes()[cursor] == b'(' {
1295        cursor = find_closing(text, cursor + 1, end, b'(', b')')? + 1;
1296    }
1297    saw_name.then_some(cursor)
1298}
1299
1300fn is_autolink(text: &str) -> bool {
1301    text.contains("://")
1302        || text.starts_with("mailto:")
1303        || (text.contains('@') && !text.chars().any(char::is_whitespace))
1304}
1305
1306fn range(start: usize, end: usize) -> TextRange {
1307    TextRange::new(TextSize::from(start as u32), TextSize::from(end as u32))
1308}