cairo_lang_doc/
parser.rs

1use std::fmt;
2use std::ops::Range;
3
4use cairo_lang_debug::DebugWithDb;
5use cairo_lang_filesystem::span::{TextOffset, TextSpan, TextWidth};
6use itertools::Itertools;
7use pulldown_cmark::{
8    Alignment, BrokenLink, CodeBlockKind, Event, HeadingLevel, LinkType, Options,
9    Parser as MarkdownParser, Tag, TagEnd,
10};
11
12use crate::db::DocGroup;
13
14#[derive(PartialEq, Eq, Hash, Debug, Clone)]
15pub struct MarkdownLink {
16    /// The span of the whole link, including the label, the destination URL and the delimiters.
17    pub link_span: TextSpan,
18    /// Where the link leads to. Not present when the label could not be resolved.
19    pub dest_span: Option<TextSpan>,
20    /// The underlying content of `dest_span`, if present.
21    pub dest_text: Option<String>,
22}
23
24/// Token representing a link to another item inside the documentation.
25#[derive(Debug, PartialEq, Clone, Eq, salsa::Update)]
26pub struct CommentLinkToken {
27    /// A link part that's inside "[]" brackets.
28    pub label: String,
29    /// A link part that's inside "()" brackets, right after the label.
30    pub path: Option<String>,
31    /// The link.
32    pub md_link: MarkdownLink,
33}
34
35/// Generic type for a comment token. It's either plain content or a link.
36/// Notice that the Content token type can store much more than just one word.
37#[derive(Debug, PartialEq, Clone, Eq, salsa::Update)]
38pub enum DocumentationCommentToken {
39    /// Token with plain documentation content.
40    Content(String),
41    /// Link token.
42    Link(CommentLinkToken),
43}
44
45impl DocumentationCommentToken {
46    /// Checks if string representation of [`DocumentationCommentToken`] ends with newline.
47    pub fn ends_with_newline(self) -> bool {
48        match self {
49            DocumentationCommentToken::Content(content) => content.ends_with('\n'),
50            DocumentationCommentToken::Link(link_token) => link_token.label.ends_with('\n'),
51        }
52    }
53}
54
55/// Helper struct for formatting possibly nested Markdown lists.
56struct DocCommentListItem {
57    /// Ordered list item separator.
58    delimiter: Option<u64>,
59    /// Flag for an ordered list.
60    is_ordered_list: bool,
61}
62
63struct PendingLink {
64    label: String,
65    path: Option<String>,
66    link_start: usize,
67    link_type: LinkType,
68    destination: String,
69    label_range: Option<Range<usize>>,
70}
71
72/// Parses documentation comment content into a vector of [DocumentationCommentToken]s, keeping
73/// the order in which they were present in the content.
74///
75/// We look for 3 link patterns (ignore the backslash):
76/// "\[label\](path)", "\[path\]" or "\[`path`\]".
77pub fn parse_documentation_comment(documentation_comment: &str) -> Vec<DocumentationCommentToken> {
78    let mut tokens = Vec::new();
79    let mut current_link: Option<PendingLink> = None;
80    let mut is_indented_code_block = false;
81    let mut replacer = |broken_link: BrokenLink<'_>| {
82        if matches!(broken_link.link_type, LinkType::ShortcutUnknown | LinkType::Shortcut) {
83            return Some((broken_link.reference.to_string().into(), "".into()));
84        }
85        None
86    };
87
88    let mut options = Options::empty();
89    options.insert(Options::ENABLE_TABLES);
90    let parser = MarkdownParser::new_with_broken_link_callback(
91        documentation_comment,
92        options,
93        Some(&mut replacer),
94    );
95
96    let mut list_nesting: Vec<DocCommentListItem> = Vec::new();
97    let write_list_item_prefix =
98        |list_nesting: &mut Vec<DocCommentListItem>,
99         tokens: &mut Vec<DocumentationCommentToken>| {
100            if !list_nesting.is_empty() {
101                let indent = "    ".repeat(list_nesting.len() - 1);
102                let list_nesting = list_nesting.last_mut().unwrap();
103
104                tokens.push(DocumentationCommentToken::Content(if list_nesting.is_ordered_list {
105                    let delimiter = list_nesting.delimiter.unwrap_or(0);
106                    list_nesting.delimiter = Some(delimiter + 1);
107                    format!("{indent}{delimiter}. ",)
108                } else {
109                    format!("{indent}- ")
110                }));
111            }
112        };
113    let mut prefix_list_item = false;
114    let mut last_two_events = [None, None];
115    let mut table_alignment: Vec<Alignment> = Vec::new();
116
117    for (event, range) in parser.into_offset_iter() {
118        match &event {
119            Event::Text(text) => {
120                if prefix_list_item {
121                    write_list_item_prefix(&mut list_nesting, &mut tokens);
122                    prefix_list_item = false;
123                }
124                if let Some(link) = current_link.as_mut() {
125                    link.label.push_str(text.as_ref());
126                    link.label_range = Some(range.clone());
127                } else {
128                    let text = {
129                        if is_indented_code_block {
130                            format!("    {text}")
131                        } else {
132                            text.to_string()
133                        }
134                    };
135                    tokens.push(DocumentationCommentToken::Content(text));
136                }
137            }
138            Event::Code(code) => {
139                if prefix_list_item {
140                    write_list_item_prefix(&mut list_nesting, &mut tokens);
141                    prefix_list_item = false;
142                }
143                let complete_code = format!("`{code}`");
144                if let Some(link) = current_link.as_mut() {
145                    link.label.push_str(&complete_code);
146                    link.label_range = Some(range.clone());
147                } else {
148                    tokens.push(DocumentationCommentToken::Content(complete_code));
149                }
150            }
151            Event::Start(tag_start) => match tag_start {
152                Tag::Heading { level, .. } => {
153                    if let Some(last_token) = tokens.last_mut()
154                        && !last_token.clone().ends_with_newline()
155                    {
156                        tokens.push(DocumentationCommentToken::Content("\n".to_string()));
157                    }
158                    tokens.push(DocumentationCommentToken::Content(format!(
159                        "{} ",
160                        heading_level_to_markdown(*level)
161                    )));
162                }
163                Tag::List(list_type) => {
164                    if !list_nesting.is_empty() {
165                        tokens.push(DocumentationCommentToken::Content("\n".to_string()));
166                    }
167                    list_nesting.push(DocCommentListItem {
168                        delimiter: *list_type,
169                        is_ordered_list: list_type.is_some(),
170                    });
171                }
172                Tag::CodeBlock(kind) => match kind {
173                    CodeBlockKind::Fenced(language) => {
174                        if language.trim().is_empty() {
175                            tokens.push(DocumentationCommentToken::Content(String::from(
176                                "```cairo\n",
177                            )));
178                        } else {
179                            tokens.push(DocumentationCommentToken::Content(format!(
180                                "```{language}\n"
181                            )));
182                        }
183                    }
184                    CodeBlockKind::Indented => {
185                        tokens.push(DocumentationCommentToken::Content("\n".to_string()));
186                        is_indented_code_block = true;
187                    }
188                },
189                Tag::Link { link_type, dest_url, .. } => {
190                    let path = match *link_type {
191                        LinkType::ShortcutUnknown | LinkType::Shortcut => None,
192                        _ => Some(dest_url.clone().into_string()),
193                    };
194                    current_link = Some(PendingLink {
195                        label: String::new(),
196                        path,
197                        link_start: range.start,
198                        link_type: *link_type,
199                        destination: dest_url.clone().into_string(),
200                        label_range: None,
201                    });
202                }
203                Tag::Paragraph | Tag::TableRow => {
204                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
205                }
206                Tag::Item => {
207                    prefix_list_item = true;
208                }
209                Tag::Table(alignment) => {
210                    table_alignment = alignment.clone();
211                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
212                }
213                Tag::TableCell => {
214                    tokens.push(DocumentationCommentToken::Content("|".to_string()));
215                }
216                Tag::Strong => {
217                    tokens.push(DocumentationCommentToken::Content("**".to_string()));
218                }
219                Tag::Emphasis => {
220                    tokens.push(DocumentationCommentToken::Content("_".to_string()));
221                }
222                _ => {}
223            },
224            Event::End(tag_end) => match tag_end {
225                TagEnd::Heading(_) | TagEnd::Table => {
226                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
227                }
228                TagEnd::List(_) => {
229                    list_nesting.pop();
230                }
231                TagEnd::Item
232                    if !matches!(last_two_events[0], Some(Event::End(_)))
233                        | !matches!(last_two_events[1], Some(Event::End(_))) =>
234                {
235                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
236                }
237                TagEnd::TableHead => {
238                    tokens.push(DocumentationCommentToken::Content(format!(
239                        "|\n|{}|",
240                        table_alignment
241                            .iter()
242                            .map(|a| {
243                                let (left, right) = get_alignment_markers(a);
244                                format!("{left}---{right}")
245                            })
246                            .join("|")
247                    )));
248                    table_alignment.clear();
249                }
250                TagEnd::CodeBlock => {
251                    if !is_indented_code_block {
252                        tokens.push(DocumentationCommentToken::Content("```\n".to_string()));
253                    }
254                    is_indented_code_block = false;
255                }
256                TagEnd::Link => {
257                    if let Some(link) = current_link {
258                        let link_span = span_from_relative_range(
259                            documentation_comment,
260                            link.link_start..range.end,
261                        );
262                        let (dest_span, dest_text) = link
263                            .label_range
264                            .as_ref()
265                            .and_then(|label_range| {
266                                location_from_link_fields(
267                                    documentation_comment,
268                                    link.link_type,
269                                    &link.destination,
270                                    label_range,
271                                )
272                            })
273                            .map(|(dest_range, dest_text)| {
274                                (
275                                    Some(span_from_relative_range(
276                                        documentation_comment,
277                                        dest_range,
278                                    )),
279                                    Some(dest_text),
280                                )
281                            })
282                            .unwrap_or((None, None));
283                        let md_link = MarkdownLink { link_span, dest_span, dest_text };
284                        tokens.push(DocumentationCommentToken::Link(CommentLinkToken {
285                            label: link.label,
286                            path: link.path,
287                            md_link,
288                        }));
289                    }
290                    current_link = None;
291                }
292                TagEnd::TableRow => {
293                    tokens.push(DocumentationCommentToken::Content("|".to_string()));
294                }
295                TagEnd::Strong => {
296                    tokens.push(DocumentationCommentToken::Content("**".to_string()));
297                }
298                TagEnd::Emphasis => {
299                    tokens.push(DocumentationCommentToken::Content("_".to_string()));
300                }
301                TagEnd::Paragraph => {
302                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
303                }
304                _ => {}
305            },
306            Event::SoftBreak => {
307                tokens.push(DocumentationCommentToken::Content("\n".to_string()));
308            }
309            Event::Rule => {
310                tokens.push(DocumentationCommentToken::Content("___\n".to_string()));
311            }
312            _ => {}
313        }
314        last_two_events = [last_two_events[1].clone(), Some(event)];
315    }
316
317    if let Some(DocumentationCommentToken::Content(token)) = tokens.first()
318        && token == "\n"
319    {
320        tokens.remove(0);
321    }
322    if let Some(DocumentationCommentToken::Content(token)) = tokens.last_mut() {
323        *token = token.trim_end().to_string();
324        if token.is_empty() {
325            tokens.pop();
326        }
327    }
328
329    tokens
330}
331
332impl fmt::Display for CommentLinkToken {
333    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
334        match self.path.clone() {
335            Some(path) => write!(f, "[{}]({})", self.label, path),
336            None => write!(f, "[{}]", self.label),
337        }
338    }
339}
340
341impl fmt::Display for DocumentationCommentToken {
342    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
343        match self {
344            DocumentationCommentToken::Content(content) => {
345                write!(f, "{content}")
346            }
347            DocumentationCommentToken::Link(link_token) => {
348                write!(f, "{link_token}")
349            }
350        }
351    }
352}
353
354impl<'db> DebugWithDb<'db> for CommentLinkToken {
355    type Db = dyn DocGroup;
356    fn fmt(&self, f: &mut fmt::Formatter<'_>, _db: &Self::Db) -> fmt::Result {
357        f.debug_struct("CommentLinkToken")
358            .field("label", &self.label)
359            .field("path", &self.path)
360            .field("md_link", &self.md_link)
361            .finish()
362    }
363}
364
365/// Converts a byte range within the string into a `TextSpan` relative to the string start.
366fn span_from_relative_range(content: &str, range: Range<usize>) -> TextSpan {
367    let start = TextOffset::START.add_width(TextWidth::at(content, range.start));
368    let end = TextOffset::START.add_width(TextWidth::at(content, range.end));
369    TextSpan::new(start, end)
370}
371
372/// Extracts a location link span and normalized destination text for the given link fields.
373fn location_from_link_fields(
374    content: &str,
375    link_type: LinkType,
376    destination: &str,
377    label_range: &Range<usize>,
378) -> Option<(Range<usize>, String)> {
379    let (destination_normalized, backticked) = normalize_location_text(destination)?;
380
381    let range = match link_type {
382        LinkType::Inline => find_inline_destination_range(content, label_range.end, destination),
383        LinkType::Collapsed
384        | LinkType::CollapsedUnknown
385        | LinkType::Shortcut
386        | LinkType::ShortcutUnknown => label_range.clone(),
387        _ => return None,
388    };
389    Some((trim_backtick_range(range, backticked), destination_normalized))
390}
391
392/// Returns true when the string looks like a location path (letters, digits, '_' or ':').
393fn is_location_string(value: &str) -> bool {
394    !value.is_empty() && value.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == ':')
395}
396
397/// Normalizes the link destination and reports whether it was backticked.
398fn normalize_location_text(value: &str) -> Option<(String, bool)> {
399    let (value, backticked) = strip_backticks(value);
400    is_location_string(value).then(|| (value.to_string(), backticked))
401}
402
403/// Strips backticks around a string if present and reports whether a pair was removed.
404fn strip_backticks(value: &str) -> (&str, bool) {
405    let value = value.trim();
406    if let Some(stripped) = value.strip_prefix('`').and_then(|rest| rest.strip_suffix('`')) {
407        (stripped, true)
408    } else {
409        (value, false)
410    }
411}
412
413/// Trims the range by one on each end when a backticked span is expected.
414fn trim_backtick_range(range: Range<usize>, backticked: bool) -> Range<usize> {
415    if backticked { (range.start + 1)..(range.end - 1) } else { range }
416}
417
418/// Computes the range for an inline destination that follows a label.
419fn find_inline_destination_range(
420    content: &str,
421    label_last_end: usize,
422    destination: &str,
423) -> Range<usize> {
424    // Finds the actual `](` boundary after the label, or uses the label's end if not found.
425    // This handles cases where the label ends in markup (e.g. `[**bold**](path)`),
426    // so the `](` that opens the destination is not necessarily adjacent to it.
427    let destination_start =
428        label_last_end + content[label_last_end..].find("](").unwrap_or_default() + 2;
429    destination_start..(destination_start + destination.len())
430}
431
432/// Maps `HeadingLevel` to the correct markdown marker.
433fn heading_level_to_markdown(heading_level: HeadingLevel) -> String {
434    let heading_char: String = String::from("#");
435    match heading_level {
436        HeadingLevel::H1 => heading_char,
437        HeadingLevel::H2 => heading_char.repeat(2),
438        HeadingLevel::H3 => heading_char.repeat(3),
439        HeadingLevel::H4 => heading_char.repeat(4),
440        HeadingLevel::H5 => heading_char.repeat(5),
441        HeadingLevel::H6 => heading_char.repeat(6),
442    }
443}
444
445/// Maps [`Alignment`] to the correct markdown markers.
446fn get_alignment_markers(alignment: &Alignment) -> (String, String) {
447    let (left, right) = match alignment {
448        Alignment::None => ("", ""),
449        Alignment::Left => (":", ""),
450        Alignment::Right => ("", ":"),
451        Alignment::Center => (":", ":"),
452    };
453    (left.to_string(), right.to_string())
454}
cairo_lang_doc/parser.rs

cairo_lang_doc/
parser.rs
