markdown/
parser.rs

1use collections::{BTreeMap, HashMap, HashSet};
2use gpui::SharedString;
3use linkify::LinkFinder;
4pub use pulldown_cmark::TagEnd as MarkdownTagEnd;
5use pulldown_cmark::{
6    Alignment, CowStr, HeadingLevel, LinkType, MetadataBlockKind, Options, Parser,
7};
8use std::{ops::Range, sync::Arc};
9use util::markdown::generate_heading_slug;
10
11use crate::path_range::PathWithRange;
12
13// `html` is nested under `parser` (not a crate-level module) because only
14// `parser.rs` uses it — `html_rendering.rs` depends on `element.rs`/
15// `builder.rs` instead. See `parser/html.rs` for details.
16pub(crate) mod html;
17
18pub const PARSE_OPTIONS: Options = Options::ENABLE_TABLES
19    .union(Options::ENABLE_FOOTNOTES)
20    .union(Options::ENABLE_STRIKETHROUGH)
21    .union(Options::ENABLE_TASKLISTS)
22    .union(Options::ENABLE_SMART_PUNCTUATION)
23    .union(Options::ENABLE_HEADING_ATTRIBUTES)
24    .union(Options::ENABLE_PLUSES_DELIMITED_METADATA_BLOCKS)
25    .union(Options::ENABLE_OLD_FOOTNOTES)
26    .union(Options::ENABLE_GFM)
27    .union(Options::ENABLE_SUPERSCRIPT)
28    .union(Options::ENABLE_SUBSCRIPT);
29
30#[derive(Default)]
31struct ParseState {
32    events: Vec<(Range<usize>, MarkdownEvent)>,
33    root_block_starts: Vec<usize>,
34    depth: usize,
35}
36
37#[derive(Debug, Default)]
38#[cfg_attr(test, derive(PartialEq))]
39pub(crate) struct ParsedMarkdownData {
40    pub events: Vec<(Range<usize>, MarkdownEvent)>,
41    pub language_names: HashSet<SharedString>,
42    pub language_paths: HashSet<Arc<str>>,
43    pub root_block_starts: Vec<usize>,
44    pub html_blocks: BTreeMap<usize, html::html_parser::ParsedHtmlBlock>,
45    pub metadata_blocks: BTreeMap<usize, ParsedMetadataBlock>,
46    pub heading_slugs: HashMap<SharedString, usize>,
47    pub footnote_definitions: HashMap<SharedString, usize>,
48}
49
50#[derive(Clone, Debug, PartialEq)]
51pub(crate) struct ParsedMetadataBlock {
52    pub content_range: Range<usize>,
53    pub rows: Option<Vec<MetadataRow>>,
54}
55
56#[derive(Clone, Debug, PartialEq)]
57pub(crate) struct MetadataRow {
58    pub key: Range<usize>,
59    pub value: Range<usize>,
60}
61
62impl ParseState {
63    fn push_event(&mut self, range: Range<usize>, event: MarkdownEvent) {
64        match &event {
65            MarkdownEvent::Start(_) => {
66                if self.depth == 0 {
67                    self.root_block_starts.push(range.start);
68                    self.events.push((range.clone(), MarkdownEvent::RootStart));
69                }
70                self.depth += 1;
71                self.events.push((range, event));
72            }
73            MarkdownEvent::End(_) => {
74                self.events.push((range.clone(), event));
75                if self.depth > 0 {
76                    self.depth -= 1;
77                    if self.depth == 0 {
78                        let root_block_index = self.root_block_starts.len() - 1;
79                        self.events
80                            .push((range, MarkdownEvent::RootEnd(root_block_index)));
81                    }
82                }
83            }
84            MarkdownEvent::Rule => {
85                if self.depth == 0 && !range.is_empty() {
86                    self.root_block_starts.push(range.start);
87                    let root_block_index = self.root_block_starts.len() - 1;
88                    self.events.push((range.clone(), MarkdownEvent::RootStart));
89                    self.events.push((range.clone(), event));
90                    self.events
91                        .push((range, MarkdownEvent::RootEnd(root_block_index)));
92                } else {
93                    self.events.push((range, event));
94                }
95            }
96            _ => {
97                self.events.push((range, event));
98            }
99        }
100    }
101}
102
103const MAX_DUPLICATE_HEADING_SLUGS: usize = 128;
104
105fn build_heading_slugs(
106    source: &str,
107    events: &[(Range<usize>, MarkdownEvent)],
108) -> HashMap<SharedString, usize> {
109    let mut slugs = HashMap::default();
110    let mut slug_counts: HashMap<String, usize> = HashMap::default();
111    let mut inside_heading = false;
112    let mut heading_text = String::new();
113    let mut heading_source_start: Option<usize> = None;
114
115    for (range, event) in events {
116        match event {
117            MarkdownEvent::Start(MarkdownTag::Heading { .. }) => {
118                inside_heading = true;
119                heading_text.clear();
120                heading_source_start = None;
121            }
122            MarkdownEvent::End(MarkdownTagEnd::Heading(_)) => {
123                if inside_heading {
124                    let source_offset = heading_source_start.unwrap_or(range.start);
125                    let base_slug = generate_heading_slug(&heading_text);
126                    let count = slug_counts.entry(base_slug.clone()).or_insert(0);
127                    let mut slug = if *count == 0 {
128                        base_slug.clone()
129                    } else {
130                        format!("{base_slug}-{count}")
131                    };
132                    *count += 1;
133                    while slugs.contains_key(slug.as_str()) {
134                        let Some(count) = slug_counts.get_mut(&base_slug) else {
135                            slug.clear();
136                            break;
137                        };
138                        if *count >= MAX_DUPLICATE_HEADING_SLUGS {
139                            slug.clear();
140                            break;
141                        }
142                        slug = format!("{base_slug}-{count}");
143                        *count += 1;
144                    }
145                    if !slug.is_empty() {
146                        slugs.insert(SharedString::from(slug), source_offset);
147                    }
148                    inside_heading = false;
149                }
150            }
151            MarkdownEvent::Text | MarkdownEvent::Code if inside_heading => {
152                if heading_source_start.is_none() {
153                    heading_source_start = Some(range.start);
154                }
155                heading_text.push_str(&source[range.clone()]);
156            }
157            MarkdownEvent::SubstitutedCode(substituted) if inside_heading => {
158                if heading_source_start.is_none() {
159                    heading_source_start = Some(range.start);
160                }
161                heading_text.push_str(substituted);
162            }
163            MarkdownEvent::SubstitutedText(substituted) if inside_heading => {
164                if heading_source_start.is_none() {
165                    heading_source_start = Some(range.start);
166                }
167                heading_text.push_str(substituted);
168            }
169            _ => {}
170        }
171    }
172
173    slugs
174}
175
176fn parse_metadata_table_rows(source: &str, source_range: Range<usize>) -> Option<Vec<MetadataRow>> {
177    let mut rows = Vec::new();
178    let mut line_start = source_range.start;
179
180    for line in source[source_range].split_inclusive('\n') {
181        let line_end = line_start + line.len();
182        let content_end = line_start + line.trim_end_matches(['\r', '\n']).len();
183        let content_range = line_start..content_end;
184        let line_text = &source[content_range.clone()];
185
186        if line_text.is_empty()
187            || line_text
188                .chars()
189                .next()
190                .is_some_and(|character| character.is_whitespace())
191        {
192            return None;
193        }
194
195        let delimiter = line_text.find(':')?;
196        let key = trim_metadata_range(source, content_range.start..content_range.start + delimiter);
197        let value = trim_metadata_range(
198            source,
199            content_range.start + delimiter + 1..content_range.end,
200        );
201        if key.is_empty() || value.is_empty() {
202            return None;
203        }
204
205        rows.push(MetadataRow { key, value });
206        line_start = line_end;
207    }
208
209    if rows.is_empty() { None } else { Some(rows) }
210}
211
212fn trim_metadata_range(source: &str, range: Range<usize>) -> Range<usize> {
213    let text = &source[range.clone()];
214    let start_offset = text.len() - text.trim_start().len();
215    let end_offset = text.trim_end().len();
216    let start = range.start + start_offset;
217    let end = (range.start + end_offset).max(start);
218    start..end
219}
220
221fn is_br_tag(html: &str) -> bool {
222    let Some(inner) = html
223        .trim()
224        .strip_prefix('<')
225        .and_then(|s| s.strip_suffix('>'))
226    else {
227        return false;
228    };
229    let inner = inner.strip_suffix('/').unwrap_or(inner);
230    inner
231        .split_ascii_whitespace()
232        .next()
233        .is_some_and(|name| name.eq_ignore_ascii_case("br"))
234}
235
236pub(crate) fn parse_markdown_with_options(
237    text: &str,
238    parse_html: bool,
239    parse_heading_slugs: bool,
240    parse_metadata_blocks: bool,
241) -> ParsedMarkdownData {
242    let mut state = ParseState::default();
243    let mut language_names = HashSet::default();
244    let mut language_paths = HashSet::default();
245    let mut html_blocks = BTreeMap::default();
246    let mut metadata_blocks = BTreeMap::default();
247    let mut within_link = false;
248    let mut within_code_block = false;
249    let mut within_metadata = false;
250    let mut within_table = false;
251    let mut current_metadata_block_start = None;
252    let mut metadata_block_content_range: Option<Range<usize>> = None;
253    let parse_options = if parse_metadata_blocks {
254        PARSE_OPTIONS.union(Options::ENABLE_YAML_STYLE_METADATA_BLOCKS)
255    } else {
256        PARSE_OPTIONS
257    };
258    let mut parser = Parser::new_ext(text, parse_options)
259        .into_offset_iter()
260        .peekable();
261    while let Some((pulldown_event, range)) = parser.next() {
262        if within_metadata && !parse_metadata_blocks {
263            if let pulldown_cmark::Event::End(pulldown_cmark::TagEnd::MetadataBlock(_)) =
264                pulldown_event
265            {
266                within_metadata = false;
267                current_metadata_block_start = None;
268                metadata_block_content_range = None;
269            }
270            continue;
271        }
272        match pulldown_event {
273            pulldown_cmark::Event::Start(tag) => {
274                if let pulldown_cmark::Tag::HtmlBlock = &tag {
275                    state.push_event(range.clone(), MarkdownEvent::Start(MarkdownTag::HtmlBlock));
276
277                    if parse_html {
278                        if let Some(block) =
279                            html::html_parser::parse_html_block(&text[range.clone()], range.clone())
280                        {
281                            html_blocks.insert(range.start, block);
282
283                            while let Some((event, end_range)) = parser.next() {
284                                if let pulldown_cmark::Event::End(
285                                    pulldown_cmark::TagEnd::HtmlBlock,
286                                ) = event
287                                {
288                                    state.push_event(
289                                        end_range,
290                                        MarkdownEvent::End(MarkdownTagEnd::HtmlBlock),
291                                    );
292                                    break;
293                                }
294                            }
295                        }
296                    }
297                    continue;
298                }
299
300                let tag = match tag {
301                    pulldown_cmark::Tag::Link {
302                        link_type,
303                        dest_url,
304                        title,
305                        id,
306                    } => {
307                        within_link = true;
308                        MarkdownTag::Link {
309                            link_type,
310                            dest_url: SharedString::from(dest_url.into_string()),
311                            title: SharedString::from(title.into_string()),
312                            id: SharedString::from(id.into_string()),
313                        }
314                    }
315                    pulldown_cmark::Tag::MetadataBlock(kind) => {
316                        within_metadata = true;
317                        current_metadata_block_start = Some(range.start);
318                        metadata_block_content_range = None;
319                        if !parse_metadata_blocks {
320                            continue;
321                        }
322                        MarkdownTag::MetadataBlock(kind)
323                    }
324                    pulldown_cmark::Tag::CodeBlock(pulldown_cmark::CodeBlockKind::Indented) => {
325                        within_code_block = true;
326                        MarkdownTag::CodeBlock {
327                            kind: CodeBlockKind::Indented,
328                            metadata: CodeBlockMetadata {
329                                content_range: range.clone(),
330                                line_count: 1,
331                                is_fenced_closed: false,
332                            },
333                        }
334                    }
335                    pulldown_cmark::Tag::CodeBlock(pulldown_cmark::CodeBlockKind::Fenced(
336                        ref info,
337                    )) => {
338                        within_code_block = true;
339                        let content_range = extract_code_block_content_range(&text[range.clone()]);
340                        let content_range =
341                            content_range.start + range.start..content_range.end + range.start;
342
343                        // Valid to use bytes since multi-byte UTF-8 doesn't use ASCII chars.
344                        let line_count = text[content_range.clone()]
345                            .bytes()
346                            .filter(|c| *c == b'\n')
347                            .count();
348                        let is_fenced_closed = {
349                            let code_block_source = &text[range.clone()];
350                            code_block_source
351                                .trim_end()
352                                .lines()
353                                .last()
354                                .is_some_and(|line| {
355                                    let trimmed = line.trim_start();
356                                    trimmed.len() >= 3 && trimmed.chars().all(|c| c == '`')
357                                })
358                        };
359
360                        let metadata = CodeBlockMetadata {
361                            content_range,
362                            line_count,
363                            is_fenced_closed,
364                        };
365
366                        let info = info.trim();
367                        let kind = if info.is_empty() {
368                            CodeBlockKind::Fenced
369                            // Languages should never contain a slash, and PathRanges always should.
370                            // (Models are told to specify them relative to a workspace root.)
371                        } else if info.contains('/') {
372                            let path_range = PathWithRange::new(info);
373                            language_paths.insert(path_range.path.clone());
374                            CodeBlockKind::FencedSrc(path_range)
375                        } else {
376                            let language = SharedString::from(info.to_string());
377                            language_names.insert(language.clone());
378                            CodeBlockKind::FencedLang(language)
379                        };
380
381                        MarkdownTag::CodeBlock { kind, metadata }
382                    }
383                    pulldown_cmark::Tag::Paragraph => MarkdownTag::Paragraph,
384                    pulldown_cmark::Tag::Heading {
385                        level,
386                        id,
387                        classes,
388                        attrs,
389                    } => {
390                        let id = id.map(|id| SharedString::from(id.into_string()));
391                        let classes = classes
392                            .into_iter()
393                            .map(|c| SharedString::from(c.into_string()))
394                            .collect();
395                        let attrs = attrs
396                            .into_iter()
397                            .map(|(key, value)| {
398                                (
399                                    SharedString::from(key.into_string()),
400                                    value.map(|v| SharedString::from(v.into_string())),
401                                )
402                            })
403                            .collect();
404                        MarkdownTag::Heading {
405                            level,
406                            id,
407                            classes,
408                            attrs,
409                        }
410                    }
411                    pulldown_cmark::Tag::BlockQuote(kind) => MarkdownTag::BlockQuote(kind),
412                    pulldown_cmark::Tag::List(start_number) => MarkdownTag::List(start_number),
413                    pulldown_cmark::Tag::Item => MarkdownTag::Item,
414                    pulldown_cmark::Tag::FootnoteDefinition(label) => {
415                        MarkdownTag::FootnoteDefinition(SharedString::from(label.to_string()))
416                    }
417                    pulldown_cmark::Tag::Table(alignments) => {
418                        within_table = true;
419                        MarkdownTag::Table(alignments)
420                    }
421                    pulldown_cmark::Tag::TableHead => MarkdownTag::TableHead,
422                    pulldown_cmark::Tag::TableRow => MarkdownTag::TableRow,
423                    pulldown_cmark::Tag::TableCell => MarkdownTag::TableCell,
424                    pulldown_cmark::Tag::Emphasis => MarkdownTag::Emphasis,
425                    pulldown_cmark::Tag::Strong => MarkdownTag::Strong,
426                    pulldown_cmark::Tag::Strikethrough => MarkdownTag::Strikethrough,
427                    pulldown_cmark::Tag::Superscript => MarkdownTag::Superscript,
428                    pulldown_cmark::Tag::Subscript => MarkdownTag::Subscript,
429                    pulldown_cmark::Tag::Image {
430                        link_type,
431                        dest_url,
432                        title,
433                        id,
434                    } => MarkdownTag::Image {
435                        link_type,
436                        dest_url: SharedString::from(dest_url.into_string()),
437                        title: SharedString::from(title.into_string()),
438                        id: SharedString::from(id.into_string()),
439                    },
440                    pulldown_cmark::Tag::HtmlBlock => MarkdownTag::HtmlBlock, // this is handled above separately
441                    pulldown_cmark::Tag::DefinitionList => MarkdownTag::DefinitionList,
442                    pulldown_cmark::Tag::DefinitionListTitle => MarkdownTag::DefinitionListTitle,
443                    pulldown_cmark::Tag::DefinitionListDefinition => {
444                        MarkdownTag::DefinitionListDefinition
445                    }
446                };
447                state.push_event(range, MarkdownEvent::Start(tag))
448            }
449            pulldown_cmark::Event::End(tag) => {
450                if let pulldown_cmark::TagEnd::Link = tag {
451                    within_link = false;
452                } else if let pulldown_cmark::TagEnd::CodeBlock = tag {
453                    within_code_block = false;
454                } else if let pulldown_cmark::TagEnd::MetadataBlock(_) = tag {
455                    within_metadata = false;
456                    let block_start = current_metadata_block_start.take();
457                    let content_range = metadata_block_content_range.take();
458                    if parse_metadata_blocks
459                        && let (Some(block_start), Some(content_range)) =
460                            (block_start, content_range)
461                    {
462                        metadata_blocks.insert(
463                            block_start,
464                            ParsedMetadataBlock {
465                                rows: parse_metadata_table_rows(text, content_range.clone()),
466                                content_range,
467                            },
468                        );
469                    }
470                    if !parse_metadata_blocks {
471                        continue;
472                    }
473                } else if let pulldown_cmark::TagEnd::Table = tag {
474                    within_table = false;
475                }
476                state.push_event(range, MarkdownEvent::End(tag));
477            }
478            pulldown_cmark::Event::Text(parsed) => {
479                fn event_for(
480                    text: &str,
481                    range: Range<usize>,
482                    str: &str,
483                ) -> (Range<usize>, MarkdownEvent) {
484                    if str == &text[range.clone()] {
485                        (range, MarkdownEvent::Text)
486                    } else {
487                        (range, MarkdownEvent::SubstitutedText(str.to_owned()))
488                    }
489                }
490
491                if within_metadata {
492                    match &mut metadata_block_content_range {
493                        Some(content_range) => {
494                            content_range.start = content_range.start.min(range.start);
495                            content_range.end = content_range.end.max(range.end);
496                        }
497                        None => metadata_block_content_range = Some(range.clone()),
498                    }
499                    state.push_event(range, MarkdownEvent::Text);
500                    continue;
501                }
502
503                if within_code_block {
504                    let (range, event) = event_for(text, range, &parsed);
505                    state.push_event(range, event);
506                    continue;
507                }
508
509                #[derive(Debug)]
510                struct TextRange<'a> {
511                    source_range: Range<usize>,
512                    merged_range: Range<usize>,
513                    parsed: CowStr<'a>,
514                }
515
516                let mut last_len = parsed.len();
517                let mut ranges = vec![TextRange {
518                    source_range: range.clone(),
519                    merged_range: 0..last_len,
520                    parsed,
521                }];
522
523                while match parser.peek() {
524                    Some((pulldown_cmark::Event::Text(_), _)) => true,
525                    Some((pulldown_cmark::Event::InlineHtml(html), _)) => {
526                        parse_html && !is_br_tag(html)
527                    }
528                    _ => false,
529                } {
530                    let Some((next_event, next_range)) = parser.next() else {
531                        unreachable!()
532                    };
533                    let next_text = match next_event {
534                        pulldown_cmark::Event::Text(next_event) => next_event,
535                        pulldown_cmark::Event::InlineHtml(_) => CowStr::Borrowed(""),
536                        _ => unreachable!(),
537                    };
538                    let next_len = last_len + next_text.len();
539                    ranges.push(TextRange {
540                        source_range: next_range.clone(),
541                        merged_range: last_len..next_len,
542                        parsed: next_text,
543                    });
544                    last_len = next_len;
545                }
546
547                let mut merged_text =
548                    String::with_capacity(ranges.last().unwrap().merged_range.end);
549                for range in &ranges {
550                    merged_text.push_str(&range.parsed);
551                }
552
553                let mut ranges = ranges.into_iter().peekable();
554
555                if !within_link && !within_code_block {
556                    let mut finder = LinkFinder::new();
557                    finder.kinds(&[linkify::LinkKind::Url]);
558
559                    // Find links in the merged text
560                    for link in finder.links(&merged_text) {
561                        let link_start_in_merged = link.start();
562                        let link_end_in_merged = link.end();
563
564                        while ranges
565                            .peek()
566                            .is_some_and(|range| range.merged_range.end <= link_start_in_merged)
567                        {
568                            let range = ranges.next().unwrap();
569                            let (range, event) = event_for(text, range.source_range, &range.parsed);
570                            state.push_event(range, event);
571                        }
572
573                        let Some(range) = ranges.peek_mut() else {
574                            continue;
575                        };
576                        let prefix_len = link_start_in_merged - range.merged_range.start;
577                        if prefix_len > 0 {
578                            let (head, tail) = range.parsed.split_at(prefix_len);
579                            let (event_range, event) = event_for(
580                                text,
581                                range.source_range.start..range.source_range.start + prefix_len,
582                                head,
583                            );
584                            state.push_event(event_range, event);
585                            range.parsed = CowStr::Boxed(tail.into());
586                            range.merged_range.start += prefix_len;
587                            range.source_range.start += prefix_len;
588                        }
589
590                        let link_start_in_source = range.source_range.start;
591                        let mut link_end_in_source = range.source_range.end;
592                        let mut link_events = Vec::new();
593
594                        while ranges
595                            .peek()
596                            .is_some_and(|range| range.merged_range.end <= link_end_in_merged)
597                        {
598                            let range = ranges.next().unwrap();
599                            link_end_in_source = range.source_range.end;
600                            link_events.push(event_for(text, range.source_range, &range.parsed));
601                        }
602
603                        if let Some(range) = ranges.peek_mut() {
604                            let prefix_len = link_end_in_merged - range.merged_range.start;
605                            if prefix_len > 0 {
606                                let (head, tail) = range.parsed.split_at(prefix_len);
607                                link_events.push(event_for(
608                                    text,
609                                    range.source_range.start..range.source_range.start + prefix_len,
610                                    head,
611                                ));
612                                range.parsed = CowStr::Boxed(tail.into());
613                                range.merged_range.start += prefix_len;
614                                range.source_range.start += prefix_len;
615                                link_end_in_source = range.source_range.start;
616                            }
617                        }
618                        let link_range = link_start_in_source..link_end_in_source;
619
620                        state.push_event(
621                            link_range.clone(),
622                            MarkdownEvent::Start(MarkdownTag::Link {
623                                link_type: LinkType::Autolink,
624                                dest_url: SharedString::from(link.as_str().to_string()),
625                                title: SharedString::default(),
626                                id: SharedString::default(),
627                            }),
628                        );
629                        for (range, event) in link_events {
630                            state.push_event(range, event);
631                        }
632                        state.push_event(
633                            link_range.clone(),
634                            MarkdownEvent::End(MarkdownTagEnd::Link),
635                        );
636                    }
637                }
638
639                for range in ranges {
640                    let (range, event) = event_for(text, range.source_range, &range.parsed);
641                    state.push_event(range, event);
642                }
643            }
644            pulldown_cmark::Event::Code(parsed) => {
645                let content_range = extract_code_content_range(&text[range.clone()]);
646                let content_range =
647                    content_range.start + range.start..content_range.end + range.start;
648                let source = &text[content_range.clone()];
649                let event = if within_table && source.contains(r"\|") {
650                    MarkdownEvent::SubstitutedCode(parsed.to_string())
651                } else {
652                    MarkdownEvent::Code
653                };
654                state.push_event(content_range, event)
655            }
656            pulldown_cmark::Event::Html(_) => state.push_event(range, MarkdownEvent::Html),
657            pulldown_cmark::Event::InlineHtml(html) => {
658                if parse_html && is_br_tag(&html) {
659                    state.push_event(range, MarkdownEvent::HardBreak)
660                } else {
661                    state.push_event(range, MarkdownEvent::InlineHtml)
662                }
663            }
664            pulldown_cmark::Event::FootnoteReference(label) => state.push_event(
665                range,
666                MarkdownEvent::FootnoteReference(SharedString::from(label.to_string())),
667            ),
668            pulldown_cmark::Event::SoftBreak => state.push_event(range, MarkdownEvent::SoftBreak),
669            pulldown_cmark::Event::HardBreak => state.push_event(range, MarkdownEvent::HardBreak),
670            pulldown_cmark::Event::Rule => state.push_event(range, MarkdownEvent::Rule),
671            pulldown_cmark::Event::TaskListMarker(checked) => {
672                state.push_event(range, MarkdownEvent::TaskListMarker(checked))
673            }
674            pulldown_cmark::Event::InlineMath(_) | pulldown_cmark::Event::DisplayMath(_) => {}
675        }
676    }
677
678    let heading_slugs = if parse_heading_slugs {
679        build_heading_slugs(text, &state.events)
680    } else {
681        HashMap::default()
682    };
683    let footnote_definitions = build_footnote_definitions(&state.events);
684
685    ParsedMarkdownData {
686        events: state.events,
687        language_names,
688        language_paths,
689        root_block_starts: state.root_block_starts,
690        html_blocks,
691        metadata_blocks,
692        heading_slugs,
693        footnote_definitions,
694    }
695}
696
697fn build_footnote_definitions(
698    events: &[(Range<usize>, MarkdownEvent)],
699) -> HashMap<SharedString, usize> {
700    let mut definitions = HashMap::default();
701    let mut current_label: Option<SharedString> = None;
702
703    for (range, event) in events {
704        match event {
705            MarkdownEvent::Start(MarkdownTag::FootnoteDefinition(label)) => {
706                current_label = Some(label.clone());
707            }
708            MarkdownEvent::End(MarkdownTagEnd::FootnoteDefinition) => {
709                current_label = None;
710            }
711            MarkdownEvent::Text if current_label.is_some() => {
712                if let Some(label) = current_label.take() {
713                    definitions.entry(label).or_insert(range.start);
714                }
715            }
716            _ => {}
717        }
718    }
719
720    definitions
721}
722
723pub fn parse_links_only(text: &str) -> Vec<(Range<usize>, MarkdownEvent)> {
724    let mut events = Vec::new();
725    let mut finder = LinkFinder::new();
726    finder.kinds(&[linkify::LinkKind::Url]);
727    let mut text_range = Range {
728        start: 0,
729        end: text.len(),
730    };
731    for link in finder.links(text) {
732        let link_range = link.start()..link.end();
733
734        if link_range.start > text_range.start {
735            events.push((text_range.start..link_range.start, MarkdownEvent::Text));
736        }
737
738        events.push((
739            link_range.clone(),
740            MarkdownEvent::Start(MarkdownTag::Link {
741                link_type: LinkType::Autolink,
742                dest_url: SharedString::from(link.as_str().to_string()),
743                title: SharedString::default(),
744                id: SharedString::default(),
745            }),
746        ));
747        events.push((link_range.clone(), MarkdownEvent::Text));
748        events.push((link_range.clone(), MarkdownEvent::End(MarkdownTagEnd::Link)));
749
750        text_range.start = link_range.end;
751    }
752
753    if text_range.end > text_range.start {
754        events.push((text_range, MarkdownEvent::Text));
755    }
756
757    events
758}
759
760/// A static-lifetime equivalent of pulldown_cmark::Event so we can cache the
761/// parse result for rendering without resorting to unsafe lifetime coercion.
762#[derive(Clone, Debug, PartialEq)]
763pub enum MarkdownEvent {
764    /// Start of a tagged element. Events that are yielded after this event
765    /// and before its corresponding `End` event are inside this element.
766    /// Start and end events are guaranteed to be balanced.
767    Start(MarkdownTag),
768    /// End of a tagged element.
769    End(MarkdownTagEnd),
770    /// Text that uses the associated range from the markdown source.
771    Text,
772    /// Text that differs from the markdown source - typically due to substitution of HTML entities
773    /// and smart punctuation.
774    SubstitutedText(String),
775    /// An inline code node.
776    Code,
777    /// An inline code node that differs from the markdown source due to escape decoding.
778    SubstitutedCode(String),
779    /// An HTML node.
780    Html,
781    /// An inline HTML node.
782    InlineHtml,
783    /// A reference to a footnote with given label, which may or may not be defined
784    /// by an event with a `Tag::FootnoteDefinition` tag. Definitions and references to them may
785    /// occur in any order.
786    FootnoteReference(SharedString),
787    /// A soft line break.
788    SoftBreak,
789    /// A hard line break.
790    HardBreak,
791    /// A horizontal ruler.
792    Rule,
793    /// A task list marker, rendered as a checkbox in HTML. Contains a true when it is checked.
794    TaskListMarker(bool),
795    /// Start of a root-level block (a top-level structural element like a paragraph, heading, list, etc.).
796    RootStart,
797    /// End of a root-level block. Contains the root block index.
798    RootEnd(usize),
799}
800
801/// Tags for elements that can contain other elements.
802#[derive(Clone, Debug, PartialEq)]
803pub enum MarkdownTag {
804    /// A paragraph of text and other inline elements.
805    Paragraph,
806
807    /// A heading, with optional identifier, classes and custom attributes.
808    /// The identifier is prefixed with `#` and the last one in the attributes
809    /// list is chosen, classes are prefixed with `.` and custom attributes
810    /// have no prefix and can optionally have a value (`myattr` o `myattr=myvalue`).
811    Heading {
812        level: HeadingLevel,
813        id: Option<SharedString>,
814        classes: Vec<SharedString>,
815        /// The first item of the tuple is the attr and second one the value.
816        attrs: Vec<(SharedString, Option<SharedString>)>,
817    },
818
819    BlockQuote(Option<pulldown_cmark::BlockQuoteKind>),
820
821    /// A code block.
822    CodeBlock {
823        kind: CodeBlockKind,
824        metadata: CodeBlockMetadata,
825    },
826
827    /// A HTML block.
828    HtmlBlock,
829
830    /// A list. If the list is ordered the field indicates the number of the first item.
831    /// Contains only list items.
832    List(Option<u64>),
833
834    /// A list item.
835    Item,
836
837    /// A footnote definition. The value contained is the footnote's label by which it can
838    /// be referred to.
839    FootnoteDefinition(SharedString),
840
841    /// A table. Contains a vector describing the text-alignment for each of its columns.
842    Table(Vec<Alignment>),
843
844    /// A table header. Contains only `TableCell`s. Note that the table body starts immediately
845    /// after the closure of the `TableHead` tag. There is no `TableBody` tag.
846    TableHead,
847
848    /// A table row. Is used both for header rows as body rows. Contains only `TableCell`s.
849    TableRow,
850    TableCell,
851
852    // span-level tags
853    Emphasis,
854    Strong,
855    Strikethrough,
856    Superscript,
857    Subscript,
858
859    /// A link.
860    Link {
861        link_type: LinkType,
862        dest_url: SharedString,
863        title: SharedString,
864        /// Identifier of reference links, e.g. `world` in the link `[hello][world]`.
865        id: SharedString,
866    },
867
868    /// An image. The first field is the link type, the second the destination URL and the third is a title,
869    /// the fourth is the link identifier.
870    Image {
871        link_type: LinkType,
872        dest_url: SharedString,
873        title: SharedString,
874        /// Identifier of reference links, e.g. `world` in the link `[hello][world]`.
875        id: SharedString,
876    },
877
878    /// A metadata block.
879    MetadataBlock(MetadataBlockKind),
880
881    DefinitionList,
882    DefinitionListTitle,
883    DefinitionListDefinition,
884}
885
886#[derive(Clone, Debug, PartialEq)]
887pub enum CodeBlockKind {
888    Indented,
889    /// "Fenced" means "surrounded by triple backticks."
890    /// There can optionally be either a language after the backticks (like in traditional Markdown)
891    /// or, if an agent is specifying a path for a source location in the project, it can be a PathRange,
892    /// e.g. ```path/to/foo.rs#L123-456 instead of ```rust
893    Fenced,
894    FencedLang(SharedString),
895    FencedSrc(PathWithRange),
896}
897
898#[derive(Default, Clone, Debug, PartialEq)]
899pub struct CodeBlockMetadata {
900    pub content_range: Range<usize>,
901    pub line_count: usize,
902    pub is_fenced_closed: bool,
903}
904
905fn extract_code_content_range(text: &str) -> Range<usize> {
906    let text_len = text.len();
907    if text_len == 0 {
908        return 0..0;
909    }
910
911    let start_ticks = text.chars().take_while(|&c| c == '`').count();
912
913    if start_ticks == 0 || start_ticks > text_len {
914        return 0..text_len;
915    }
916
917    let end_ticks = text.chars().rev().take_while(|&c| c == '`').count();
918
919    if end_ticks != start_ticks || text_len < start_ticks + end_ticks {
920        return 0..text_len;
921    }
922
923    start_ticks..text_len - end_ticks
924}
925
926pub(crate) fn extract_code_block_content_range(text: &str) -> Range<usize> {
927    let mut range = 0..text.len();
928    if text.starts_with("```") {
929        range.start += 3;
930
931        if let Some(newline_ix) = text[range.clone()].find('\n') {
932            range.start += newline_ix + 1;
933        }
934    }
935
936    if !range.is_empty() && text.ends_with("```") {
937        range.end -= 3;
938    }
939    if range.start > range.end {
940        range.end = range.start;
941    }
942    range
943}
944
945#[cfg(test)]
946mod tests {
947    use super::MarkdownEvent::*;
948    use super::MarkdownTag::*;
949    use super::*;
950
951    const CONDITIONAL_OPTIONS: Options = Options::ENABLE_YAML_STYLE_METADATA_BLOCKS;
952    const UNWANTED_OPTIONS: Options = Options::ENABLE_MATH
953        .union(Options::ENABLE_DEFINITION_LIST)
954        .union(Options::ENABLE_WIKILINKS);
955
956    #[test]
957    fn all_options_considered() {
958        // The purpose of this is to fail when new options are added to pulldown_cmark, so that they
959        // can be evaluated for inclusion.
960        assert_eq!(
961            PARSE_OPTIONS
962                .union(CONDITIONAL_OPTIONS)
963                .union(UNWANTED_OPTIONS),
964            Options::all()
965        );
966    }
967
968    #[test]
969    fn wanted_and_unwanted_options_disjoint() {
970        assert_eq!(
971            PARSE_OPTIONS
972                .union(CONDITIONAL_OPTIONS)
973                .intersection(UNWANTED_OPTIONS),
974            Options::empty()
975        );
976    }
977
978    #[test]
979    fn test_yaml_style_metadata_block() {
980        assert_eq!(
981            parse_markdown_with_options("---\ntitle: Post\n---\n# Heading", false, false, true),
982            ParsedMarkdownData {
983                events: vec![
984                    (0..19, RootStart),
985                    (0..19, Start(MetadataBlock(MetadataBlockKind::YamlStyle))),
986                    (4..16, Text),
987                    (
988                        0..19,
989                        End(MarkdownTagEnd::MetadataBlock(MetadataBlockKind::YamlStyle))
990                    ),
991                    (0..19, RootEnd(0)),
992                    (20..29, RootStart),
993                    (
994                        20..29,
995                        Start(Heading {
996                            level: HeadingLevel::H1,
997                            id: None,
998                            classes: Vec::new(),
999                            attrs: Vec::new(),
1000                        })
1001                    ),
1002                    (22..29, Text),
1003                    (20..29, End(MarkdownTagEnd::Heading(HeadingLevel::H1))),
1004                    (20..29, RootEnd(1)),
1005                ],
1006                root_block_starts: vec![0, 20],
1007                metadata_blocks: BTreeMap::from_iter([(
1008                    0,
1009                    ParsedMetadataBlock {
1010                        content_range: 4..16,
1011                        rows: Some(vec![MetadataRow {
1012                            key: 4..9,
1013                            value: 11..15,
1014                        }]),
1015                    },
1016                )]),
1017                ..Default::default()
1018            }
1019        )
1020    }
1021
1022    #[test]
1023    fn test_metadata_block_text_is_verbatim() {
1024        let parsed =
1025            parse_markdown_with_options("---\nurl: https://zed.dev\n---\nBody", false, false, true);
1026        assert!(
1027            parsed
1028                .events
1029                .iter()
1030                .all(|(_, event)| !matches!(event, Start(Link { .. })))
1031        );
1032    }
1033
1034    #[test]
1035    fn test_metadata_blocks_store_table_rows() {
1036        let parsed = parse_markdown_with_options(
1037            "---\ntitle: Post\nauthor: Zed\n---\nBody",
1038            false,
1039            false,
1040            true,
1041        );
1042
1043        assert_eq!(
1044            parsed.metadata_blocks,
1045            BTreeMap::from_iter([(
1046                0,
1047                ParsedMetadataBlock {
1048                    content_range: 4..28,
1049                    rows: Some(vec![
1050                        MetadataRow {
1051                            key: 4..9,
1052                            value: 11..15,
1053                        },
1054                        MetadataRow {
1055                            key: 16..22,
1056                            value: 24..27,
1057                        },
1058                    ]),
1059                },
1060            )])
1061        );
1062    }
1063
1064    #[test]
1065    fn test_metadata_blocks_store_fallback_for_nested_yaml() {
1066        let parsed =
1067            parse_markdown_with_options("---\ntags:\n  - zed\n---\nBody", false, false, true);
1068
1069        assert_eq!(
1070            parsed.metadata_blocks,
1071            BTreeMap::from_iter([(
1072                0,
1073                ParsedMetadataBlock {
1074                    content_range: 4..18,
1075                    rows: None,
1076                },
1077            )])
1078        );
1079    }
1080
1081    #[test]
1082    fn test_metadata_table_rows_parse_simple_colon_pairs() {
1083        let source = "title: Post\nauthor: Zed\n";
1084        let Some(rows) = parse_metadata_table_rows(source, 0..source.len()) else {
1085            panic!("expected metadata rows");
1086        };
1087        let pairs = rows
1088            .into_iter()
1089            .map(|row| (&source[row.key], &source[row.value]))
1090            .collect::<Vec<_>>();
1091
1092        assert_eq!(pairs, vec![("title", "Post"), ("author", "Zed")]);
1093    }
1094
1095    #[test]
1096    fn test_metadata_table_rows_reject_non_simple_colon_pairs() {
1097        for source in [
1098            "tags:\n  - zed\n",
1099            "title = Post\n",
1100            "title:\n",
1101            "title:   \n",
1102            ": Post\n",
1103            " title: Post\n",
1104            "\n",
1105        ] {
1106            assert!(parse_metadata_table_rows(source, 0..source.len()).is_none());
1107        }
1108    }
1109
1110    #[test]
1111    fn test_trim_metadata_range_returns_valid_empty_range() {
1112        let source = "key:   \n";
1113        let trimmed = trim_metadata_range(source, 4..7);
1114
1115        assert_eq!(trimmed, 7..7);
1116        assert!(source[trimmed].is_empty());
1117    }
1118
1119    #[test]
1120    fn test_html_comments() {
1121        assert_eq!(
1122            parse_markdown_with_options(
1123                "  <!--\nrdoc-file=string.c\n-->\nReturns",
1124                false,
1125                false,
1126                false
1127            ),
1128            ParsedMarkdownData {
1129                events: vec![
1130                    (2..30, RootStart),
1131                    (2..30, Start(HtmlBlock)),
1132                    (2..2, SubstitutedText("  ".into())),
1133                    (2..7, Html),
1134                    (7..26, Html),
1135                    (26..30, Html),
1136                    (2..30, End(MarkdownTagEnd::HtmlBlock)),
1137                    (2..30, RootEnd(0)),
1138                    (30..37, RootStart),
1139                    (30..37, Start(Paragraph)),
1140                    (30..37, Text),
1141                    (30..37, End(MarkdownTagEnd::Paragraph)),
1142                    (30..37, RootEnd(1)),
1143                ],
1144                root_block_starts: vec![2, 30],
1145                ..Default::default()
1146            }
1147        )
1148    }
1149
1150    #[test]
1151    fn test_plain_urls_and_escaped_text() {
1152        assert_eq!(
1153            parse_markdown_with_options(
1154                "&nbsp;&nbsp; https://some.url some \\`&#9658;\\` text",
1155                false,
1156                false,
1157                false,
1158            ),
1159            ParsedMarkdownData {
1160                events: vec![
1161                    (0..51, RootStart),
1162                    (0..51, Start(Paragraph)),
1163                    (0..6, SubstitutedText("\u{a0}".into())),
1164                    (6..12, SubstitutedText("\u{a0}".into())),
1165                    (12..13, Text),
1166                    (
1167                        13..29,
1168                        Start(Link {
1169                            link_type: LinkType::Autolink,
1170                            dest_url: "https://some.url".into(),
1171                            title: "".into(),
1172                            id: "".into(),
1173                        })
1174                    ),
1175                    (13..29, Text),
1176                    (13..29, End(MarkdownTagEnd::Link)),
1177                    (29..35, Text),
1178                    (36..37, Text), // Escaped backtick
1179                    (37..44, SubstitutedText("►".into())),
1180                    (45..46, Text), // Escaped backtick
1181                    (46..51, Text),
1182                    (0..51, End(MarkdownTagEnd::Paragraph)),
1183                    (0..51, RootEnd(0)),
1184                ],
1185                root_block_starts: vec![0],
1186                ..Default::default()
1187            }
1188        );
1189    }
1190
1191    #[test]
1192    fn test_incomplete_link() {
1193        assert_eq!(
1194            parse_markdown_with_options(
1195                "You can use the [GitHub Search API](https://docs.github.com/en",
1196                false,
1197                false,
1198                false,
1199            )
1200            .events,
1201            vec![
1202                (0..62, RootStart),
1203                (0..62, Start(Paragraph)),
1204                (0..16, Text),
1205                (16..17, Text),
1206                (17..34, Text),
1207                (34..35, Text),
1208                (35..36, Text),
1209                (
1210                    36..62,
1211                    Start(Link {
1212                        link_type: LinkType::Autolink,
1213                        dest_url: "https://docs.github.com/en".into(),
1214                        title: "".into(),
1215                        id: "".into()
1216                    })
1217                ),
1218                (36..62, Text),
1219                (36..62, End(MarkdownTagEnd::Link)),
1220                (0..62, End(MarkdownTagEnd::Paragraph)),
1221                (0..62, RootEnd(0)),
1222            ],
1223        );
1224    }
1225
1226    #[test]
1227    fn test_smart_punctuation() {
1228        assert_eq!(
1229            parse_markdown_with_options(
1230                "-- --- ... \"double quoted\" 'single quoted' ----------",
1231                false,
1232                false,
1233                false,
1234            ),
1235            ParsedMarkdownData {
1236                events: vec![
1237                    (0..53, RootStart),
1238                    (0..53, Start(Paragraph)),
1239                    (0..2, SubstitutedText("–".into())),
1240                    (2..3, Text),
1241                    (3..6, SubstitutedText("—".into())),
1242                    (6..7, Text),
1243                    (7..10, SubstitutedText("…".into())),
1244                    (10..11, Text),
1245                    (11..12, SubstitutedText("\u{201c}".into())),
1246                    (12..25, Text),
1247                    (25..26, SubstitutedText("\u{201d}".into())),
1248                    (26..27, Text),
1249                    (27..28, SubstitutedText("\u{2018}".into())),
1250                    (28..41, Text),
1251                    (41..42, SubstitutedText("\u{2019}".into())),
1252                    (42..43, Text),
1253                    (43..53, SubstitutedText("–––––".into())),
1254                    (0..53, End(MarkdownTagEnd::Paragraph)),
1255                    (0..53, RootEnd(0)),
1256                ],
1257                root_block_starts: vec![0],
1258                ..Default::default()
1259            }
1260        )
1261    }
1262
1263    #[test]
1264    fn test_code_block_metadata() {
1265        assert_eq!(
1266            parse_markdown_with_options(
1267                "```rust\nfn main() {\n let a = 1;\n}\n```",
1268                false,
1269                false,
1270                false
1271            ),
1272            ParsedMarkdownData {
1273                events: vec![
1274                    (0..37, RootStart),
1275                    (
1276                        0..37,
1277                        Start(CodeBlock {
1278                            kind: CodeBlockKind::FencedLang("rust".into()),
1279                            metadata: CodeBlockMetadata {
1280                                content_range: 8..34,
1281                                line_count: 3,
1282                                is_fenced_closed: true,
1283                            }
1284                        })
1285                    ),
1286                    (8..34, Text),
1287                    (0..37, End(MarkdownTagEnd::CodeBlock)),
1288                    (0..37, RootEnd(0)),
1289                ],
1290                language_names: {
1291                    let mut h = HashSet::default();
1292                    h.insert("rust".into());
1293                    h
1294                },
1295                root_block_starts: vec![0],
1296                ..Default::default()
1297            }
1298        );
1299        assert_eq!(
1300            parse_markdown_with_options("    fn main() {}", false, false, false),
1301            ParsedMarkdownData {
1302                events: vec![
1303                    (4..16, RootStart),
1304                    (
1305                        4..16,
1306                        Start(CodeBlock {
1307                            kind: CodeBlockKind::Indented,
1308                            metadata: CodeBlockMetadata {
1309                                content_range: 4..16,
1310                                line_count: 1,
1311                                is_fenced_closed: false,
1312                            }
1313                        })
1314                    ),
1315                    (4..16, Text),
1316                    (4..16, End(MarkdownTagEnd::CodeBlock)),
1317                    (4..16, RootEnd(0)),
1318                ],
1319                root_block_starts: vec![4],
1320                ..Default::default()
1321            }
1322        );
1323    }
1324
1325    fn assert_code_block_does_not_emit_links(markdown: &str) {
1326        let parsed = parse_markdown_with_options(markdown, false, false, false);
1327        let mut code_block_depth = 0;
1328        let mut code_block_count = 0;
1329        let mut saw_text_inside_code_block = false;
1330
1331        for (_, event) in &parsed.events {
1332            match event {
1333                Start(CodeBlock { .. }) => {
1334                    code_block_depth += 1;
1335                    code_block_count += 1;
1336                }
1337                End(MarkdownTagEnd::CodeBlock) => {
1338                    assert!(
1339                        code_block_depth > 0,
1340                        "encountered a code block end without a matching start"
1341                    );
1342                    code_block_depth -= 1;
1343                }
1344                Start(Link { .. }) | End(MarkdownTagEnd::Link) => {
1345                    assert_eq!(
1346                        code_block_depth, 0,
1347                        "code blocks should not emit link events"
1348                    );
1349                }
1350                Text | SubstitutedText(_) if code_block_depth > 0 => {
1351                    saw_text_inside_code_block = true;
1352                }
1353                _ => {}
1354            }
1355        }
1356
1357        assert_eq!(code_block_count, 1, "expected exactly one code block");
1358        assert_eq!(code_block_depth, 0, "unterminated code block");
1359        assert!(
1360            saw_text_inside_code_block,
1361            "expected text inside the code block"
1362        );
1363    }
1364
1365    #[test]
1366    fn test_code_blocks_do_not_autolink_urls() {
1367        assert_code_block_does_not_emit_links("```txt\nhttps://example.com\n```");
1368        assert_code_block_does_not_emit_links("    https://example.com");
1369        assert_code_block_does_not_emit_links(
1370            "```txt\r\nhttps:/\\/example.com\r\nhttps://example&#46;com\r\n```",
1371        );
1372        assert_code_block_does_not_emit_links(
1373            "    https:/\\/example.com\r\n    https://example&#46;com",
1374        );
1375    }
1376
1377    #[test]
1378    fn test_metadata_blocks_are_root_blocks() {
1379        assert_eq!(
1380            parse_markdown_with_options(
1381                "+++\ntitle = \"Example\"\n+++\n\nParagraph",
1382                false,
1383                false,
1384                true
1385            ),
1386            ParsedMarkdownData {
1387                events: vec![
1388                    (0..25, RootStart),
1389                    (0..25, Start(MetadataBlock(MetadataBlockKind::PlusesStyle))),
1390                    (4..22, Text),
1391                    (
1392                        0..25,
1393                        End(MarkdownTagEnd::MetadataBlock(
1394                            MetadataBlockKind::PlusesStyle
1395                        ))
1396                    ),
1397                    (0..25, RootEnd(0)),
1398                    (27..36, RootStart),
1399                    (27..36, Start(Paragraph)),
1400                    (27..36, Text),
1401                    (27..36, End(MarkdownTagEnd::Paragraph)),
1402                    (27..36, RootEnd(1)),
1403                ],
1404                root_block_starts: vec![0, 27],
1405                metadata_blocks: BTreeMap::from_iter([(
1406                    0,
1407                    ParsedMetadataBlock {
1408                        content_range: 4..22,
1409                        rows: None,
1410                    },
1411                )]),
1412                ..Default::default()
1413            }
1414        );
1415    }
1416
1417    #[test]
1418    fn test_metadata_blocks_are_omitted_by_default() {
1419        assert_eq!(
1420            parse_markdown_with_options(
1421                "+++\ntitle = \"Example\"\n+++\n\nParagraph",
1422                false,
1423                false,
1424                false
1425            ),
1426            ParsedMarkdownData {
1427                events: vec![
1428                    (27..36, RootStart),
1429                    (27..36, Start(Paragraph)),
1430                    (27..36, Text),
1431                    (27..36, End(MarkdownTagEnd::Paragraph)),
1432                    (27..36, RootEnd(0)),
1433                ],
1434                root_block_starts: vec![27],
1435                ..Default::default()
1436            }
1437        );
1438    }
1439
1440    #[test]
1441    fn test_table_checkboxes_remain_text_in_cells() {
1442        let markdown = "\
1443| Done | Task    |
1444|------|---------|
1445| [x]  | Fix bug |
1446| [ ]  | Add feature |";
1447        let parsed = parse_markdown_with_options(markdown, false, false, false);
1448
1449        let mut in_table = false;
1450        let mut saw_task_list_marker = false;
1451        let mut cell_texts = Vec::new();
1452        let mut current_cell = String::new();
1453
1454        for (range, event) in &parsed.events {
1455            match event {
1456                Start(Table(_)) => in_table = true,
1457                End(MarkdownTagEnd::Table) => in_table = false,
1458                Start(TableCell) => current_cell.clear(),
1459                End(MarkdownTagEnd::TableCell) => {
1460                    if in_table {
1461                        cell_texts.push(current_cell.clone());
1462                    }
1463                }
1464                Text if in_table => current_cell.push_str(&markdown[range.clone()]),
1465                TaskListMarker(_) if in_table => saw_task_list_marker = true,
1466                _ => {}
1467            }
1468        }
1469
1470        let checkbox_cells: Vec<&str> = cell_texts
1471            .iter()
1472            .map(|cell| cell.trim())
1473            .filter(|cell| *cell == "[x]" || *cell == "[X]" || *cell == "[ ]")
1474            .collect();
1475
1476        assert!(
1477            !saw_task_list_marker,
1478            "Table checkboxes should remain text, not task-list markers"
1479        );
1480        assert_eq!(checkbox_cells, vec!["[x]", "[ ]"]);
1481    }
1482
1483    #[test]
1484    fn test_extract_code_content_range() {
1485        let input = "```let x = 5;```";
1486        assert_eq!(extract_code_content_range(input), 3..13);
1487
1488        let input = "``let x = 5;``";
1489        assert_eq!(extract_code_content_range(input), 2..12);
1490
1491        let input = "`let x = 5;`";
1492        assert_eq!(extract_code_content_range(input), 1..11);
1493
1494        let input = "plain text";
1495        assert_eq!(extract_code_content_range(input), 0..10);
1496
1497        let input = "``let x = 5;`";
1498        assert_eq!(extract_code_content_range(input), 0..13);
1499    }
1500
1501    #[test]
1502    fn test_inline_code_substitutes_escaped_pipes() {
1503        let markdown = r"| Pattern |
1504| --- |
1505| `a\|b` |";
1506        let parsed = parse_markdown_with_options(markdown, false, false, false);
1507        let code_range = {
1508            let start = markdown.find(r"a\|b").expect("inline code source");
1509            start..start + r"a\|b".len()
1510        };
1511
1512        assert!(
1513            parsed
1514                .events
1515                .iter()
1516                .any(|(range, event)| range == &code_range
1517                    && event == &SubstitutedCode("a|b".into())),
1518            "expected escaped pipe in table inline code to render as decoded inline code: {:?}",
1519            parsed.events
1520        );
1521    }
1522
1523    #[test]
1524    fn test_inline_code_keeps_escaped_pipes_outside_tables() {
1525        let markdown = r"`a\|b`";
1526        let parsed = parse_markdown_with_options(markdown, false, false, false);
1527
1528        assert!(
1529            parsed
1530                .events
1531                .iter()
1532                .any(|(range, event)| range == &(1..5) && event == &Code),
1533            "expected escaped pipe outside a table to remain normal inline code: {:?}",
1534            parsed.events
1535        );
1536    }
1537
1538    #[test]
1539    fn test_extract_code_block_content_range() {
1540        let input = "```rust\nlet x = 5;\n```";
1541        assert_eq!(extract_code_block_content_range(input), 8..19);
1542
1543        let input = "plain text";
1544        assert_eq!(extract_code_block_content_range(input), 0..10);
1545
1546        let input = "```python\nprint('hello')\nprint('world')\n```";
1547        assert_eq!(extract_code_block_content_range(input), 10..40);
1548
1549        // Malformed input
1550        let input = "`````";
1551        assert_eq!(extract_code_block_content_range(input), 3..3);
1552    }
1553
1554    #[test]
1555    fn test_footnotes() {
1556        let parsed = parse_markdown_with_options(
1557            "Text with a footnote[^1] and some more text.\n\n[^1]: This is the footnote content.",
1558            false,
1559            false,
1560            false,
1561        );
1562        assert_eq!(
1563            parsed.events,
1564            vec![
1565                (0..45, RootStart),
1566                (0..45, Start(Paragraph)),
1567                (0..20, Text),
1568                (20..24, FootnoteReference("1".into())),
1569                (24..44, Text),
1570                (0..45, End(MarkdownTagEnd::Paragraph)),
1571                (0..45, RootEnd(0)),
1572                (46..81, RootStart),
1573                (46..81, Start(FootnoteDefinition("1".into()))),
1574                (52..81, Start(Paragraph)),
1575                (52..81, Text),
1576                (52..81, End(MarkdownTagEnd::Paragraph)),
1577                (46..81, End(MarkdownTagEnd::FootnoteDefinition)),
1578                (46..81, RootEnd(1)),
1579            ]
1580        );
1581        assert_eq!(parsed.footnote_definitions.len(), 1);
1582        assert_eq!(parsed.footnote_definitions.get("1").copied(), Some(52));
1583    }
1584
1585    #[test]
1586    fn test_footnote_definitions_multiple() {
1587        let parsed = parse_markdown_with_options(
1588            "Text[^a] and[^b].\n\n[^a]: First.\n\n[^b]: Second.",
1589            false,
1590            false,
1591            false,
1592        );
1593        assert_eq!(parsed.footnote_definitions.len(), 2);
1594        assert!(parsed.footnote_definitions.contains_key("a"));
1595        assert!(parsed.footnote_definitions.contains_key("b"));
1596    }
1597
1598    #[test]
1599    fn test_links_split_across_fragments() {
1600        // This test verifies that links split across multiple text fragments due to escaping or other issues
1601        // are correctly detected and processed
1602        // Note: In real usage, pulldown_cmark creates separate text events for the escaped character
1603        // We're verifying our parser can handle this correctly
1604        assert_eq!(
1605            parse_markdown_with_options(
1606                "https:/\\/example.com is equivalent to https://example&#46;com!",
1607                false,
1608                false,
1609                false,
1610            )
1611            .events,
1612            vec![
1613                (0..62, RootStart),
1614                (0..62, Start(Paragraph)),
1615                (
1616                    0..20,
1617                    Start(Link {
1618                        link_type: LinkType::Autolink,
1619                        dest_url: "https://example.com".into(),
1620                        title: "".into(),
1621                        id: "".into()
1622                    })
1623                ),
1624                (0..7, Text),
1625                (8..20, Text),
1626                (0..20, End(MarkdownTagEnd::Link)),
1627                (20..38, Text),
1628                (
1629                    38..61,
1630                    Start(Link {
1631                        link_type: LinkType::Autolink,
1632                        dest_url: "https://example.com".into(),
1633                        title: "".into(),
1634                        id: "".into()
1635                    })
1636                ),
1637                (38..53, Text),
1638                (53..58, SubstitutedText(".".into())),
1639                (58..61, Text),
1640                (38..61, End(MarkdownTagEnd::Link)),
1641                (61..62, Text),
1642                (0..62, End(MarkdownTagEnd::Paragraph)),
1643                (0..62, RootEnd(0)),
1644            ],
1645        );
1646
1647        assert_eq!(
1648            parse_markdown_with_options(
1649                "Visit https://example.com/cat\\/é&#8205;☕ for coffee!",
1650                false,
1651                false,
1652                false,
1653            )
1654            .events,
1655            [
1656                (0..55, RootStart),
1657                (0..55, Start(Paragraph)),
1658                (0..6, Text),
1659                (
1660                    6..43,
1661                    Start(Link {
1662                        link_type: LinkType::Autolink,
1663                        dest_url: "https://example.com/cat/é\u{200d}☕".into(),
1664                        title: "".into(),
1665                        id: "".into()
1666                    })
1667                ),
1668                (6..29, Text),
1669                (30..33, Text),
1670                (33..40, SubstitutedText("\u{200d}".into())),
1671                (40..43, Text),
1672                (6..43, End(MarkdownTagEnd::Link)),
1673                (43..55, Text),
1674                (0..55, End(MarkdownTagEnd::Paragraph)),
1675                (0..55, RootEnd(0)),
1676            ]
1677        );
1678    }
1679
1680    #[test]
1681    fn test_heading_slugs() {
1682        let parsed = parse_markdown_with_options(
1683            "# Hello World\n\n## Code `block`\n\n### Third Level\n\n#### Fourth Level\n\n## Hello World",
1684            false,
1685            true,
1686            false,
1687        );
1688        assert_eq!(parsed.heading_slugs.len(), 5);
1689        assert!(parsed.heading_slugs.contains_key("hello-world"));
1690        assert!(parsed.heading_slugs.contains_key("code-block"));
1691        assert!(parsed.heading_slugs.contains_key("third-level"));
1692        assert!(parsed.heading_slugs.contains_key("fourth-level"));
1693        assert!(parsed.heading_slugs.contains_key("hello-world-1"));
1694    }
1695
1696    #[test]
1697    fn test_heading_source_index_for_slug() {
1698        let parsed = parse_markdown_with_options(
1699            "# Duplicate\n\nText\n\n## Duplicate\n\nMore text",
1700            false,
1701            true,
1702            false,
1703        );
1704        let first = parsed.heading_slugs.get("duplicate").copied();
1705        let second = parsed.heading_slugs.get("duplicate-1").copied();
1706        assert!(first.is_some());
1707        assert!(second.is_some());
1708        assert!(first.expect("first slug missing") < second.expect("second slug missing"));
1709    }
1710
1711    #[test]
1712    fn test_heading_slug_collision_with_dedup_suffix() {
1713        let parsed = parse_markdown_with_options("# Foo\n\n## Foo\n\n## Foo 1", false, true, false);
1714        assert_eq!(parsed.heading_slugs.len(), 3);
1715        assert!(parsed.heading_slugs.contains_key("foo"));
1716        assert!(parsed.heading_slugs.contains_key("foo-1"));
1717        assert!(parsed.heading_slugs.contains_key("foo-1-1"));
1718    }
1719
1720    #[test]
1721    fn test_gfm_alert_block_quote_kinds() {
1722        use pulldown_cmark::BlockQuoteKind;
1723
1724        let markdown = "\n> [!NOTE]\n> A note.\n\n> [!TIP]\n> A tip.\n\n> [!IMPORTANT]\n> Important.\n\n> [!WARNING]\n> A warning.\n\n> [!CAUTION]\n> A caution.\n\n> Plain quote.\n";
1725        let parsed = parse_markdown_with_options(markdown, false, false, false);
1726
1727        let block_quote_kinds: Vec<_> = parsed
1728            .events
1729            .iter()
1730            .filter_map(|(_, event)| match event {
1731                Start(BlockQuote(kind)) => Some(*kind),
1732                _ => None,
1733            })
1734            .collect();
1735
1736        assert_eq!(
1737            block_quote_kinds,
1738            vec![
1739                Some(BlockQuoteKind::Note),
1740                Some(BlockQuoteKind::Tip),
1741                Some(BlockQuoteKind::Important),
1742                Some(BlockQuoteKind::Warning),
1743                Some(BlockQuoteKind::Caution),
1744                None,
1745            ]
1746        );
1747    }
1748
1749    #[test]
1750    fn test_br_tag_emits_hard_break() {
1751        for input in [
1752            "hello<br>world",
1753            "hello<br/>world",
1754            "hello<br />world",
1755            "hello<br >world",
1756            "hello<BR>world",
1757            "hello<br class=\"x\">world",
1758            "hello<br class=\"x\"/>world",
1759        ] {
1760            let parsed = parse_markdown_with_options(input, true, false, false);
1761            let has_hard_break = parsed
1762                .events
1763                .iter()
1764                .any(|(_, event)| matches!(event, MarkdownEvent::HardBreak));
1765            let has_empty_substituted_text = parsed.events.iter().any(|(_, event)| {
1766                matches!(event, MarkdownEvent::SubstitutedText(text) if text.is_empty())
1767            });
1768            assert!(has_hard_break, "<br> in \"{input}\" should emit HardBreak");
1769            assert!(
1770                !has_empty_substituted_text,
1771                "<br> in \"{input}\" should not produce empty SubstitutedText"
1772            );
1773        }
1774    }
1775
1776    #[test]
1777    fn test_br_tag_not_a_hard_break_without_parse_html() {
1778        for input in ["hello<br>world", "hello<br/>world", "hello<br />world"] {
1779            let parsed = parse_markdown_with_options(input, false, false, false);
1780            let has_hard_break = parsed
1781                .events
1782                .iter()
1783                .any(|(_, event)| matches!(event, MarkdownEvent::HardBreak));
1784            let has_inline_html = parsed
1785                .events
1786                .iter()
1787                .any(|(_, event)| matches!(event, MarkdownEvent::InlineHtml));
1788            assert!(
1789                !has_hard_break,
1790                "<br> in \"{input}\" should not emit HardBreak when parse_html is disabled"
1791            );
1792            assert!(
1793                has_inline_html,
1794                "<br> in \"{input}\" should be preserved as InlineHtml when parse_html is disabled"
1795            );
1796        }
1797    }
1798
1799    #[test]
1800    fn test_br_prefixed_tag_is_not_a_hard_break() {
1801        for input in ["a<break>b", "a<brick>b", "a<b>bold</b>c"] {
1802            let parsed = parse_markdown_with_options(input, true, false, false);
1803            let has_hard_break = parsed
1804                .events
1805                .iter()
1806                .any(|(_, event)| matches!(event, MarkdownEvent::HardBreak));
1807            assert!(
1808                !has_hard_break,
1809                "\"{input}\" should not be treated as a <br> hard break"
1810            );
1811        }
1812    }
1813
1814    #[test]
1815    fn test_unrecognized_inline_html_preserved_as_inline_html() {
1816        for input in ["a<span>b</span>c", "a<em>b</em>c", "a<strong>b</strong>c"] {
1817            let parsed = parse_markdown_with_options(input, false, false, false);
1818            let has_inline_html = parsed
1819                .events
1820                .iter()
1821                .any(|(_, event)| matches!(event, MarkdownEvent::InlineHtml));
1822            let has_hard_break = parsed
1823                .events
1824                .iter()
1825                .any(|(_, event)| matches!(event, MarkdownEvent::HardBreak));
1826            assert!(
1827                has_inline_html,
1828                "unrecognized inline HTML \"{input}\" should emit InlineHtml"
1829            );
1830            assert!(
1831                !has_hard_break,
1832                "unrecognized inline HTML \"{input}\" should not emit HardBreak"
1833            );
1834        }
1835    }
1836}
markdown/parser.rs

markdown/
parser.rs
