cairo_lang_doc/
parser.rs

1use std::fmt;
2use std::ops::Range;
3
4use cairo_lang_debug::DebugWithDb;
5use cairo_lang_filesystem::span::{TextOffset, TextSpan, TextWidth};
6use itertools::Itertools;
7use pulldown_cmark::{
8    Alignment, BrokenLink, CodeBlockKind, Event, HeadingLevel, LinkType, Options,
9    Parser as MarkdownParser, Tag, TagEnd,
10};
11
12use crate::db::DocGroup;
13
14#[derive(PartialEq, Eq, Hash, Debug, Clone)]
15pub struct MarkdownLink {
16    /// The span of the whole link, including the label, the destination URL and the delimiters.
17    pub link_span: TextSpan,
18    /// Where the link leads to. Not present when the label could not be resolved.
19    pub dest_span: Option<TextSpan>,
20    /// The underlying content of `dest_span`, if present.
21    pub dest_text: Option<String>,
22}
23
24/// Token representing a link to another item inside the documentation.
25#[derive(Debug, PartialEq, Clone, Eq, salsa::Update)]
26pub struct CommentLinkToken {
27    /// A link part that's inside "[]" brackets.
28    pub label: String,
29    /// A link part that's inside "()" brackets, right after the label.
30    pub path: Option<String>,
31    /// The link.
32    pub md_link: MarkdownLink,
33}
34
35/// Generic type for a comment token. It's either plain content or a link.
36/// Notice that the Content token type can store much more than just one word.
37#[derive(Debug, PartialEq, Clone, Eq, salsa::Update)]
38pub enum DocumentationCommentToken {
39    /// Token with plain documentation content.
40    Content(String),
41    /// Link token.
42    Link(CommentLinkToken),
43}
44
45impl DocumentationCommentToken {
46    /// Checks if string representation of [`DocumentationCommentToken`] ends with newline.
47    pub fn ends_with_newline(self) -> bool {
48        match self {
49            DocumentationCommentToken::Content(content) => content.ends_with('\n'),
50            DocumentationCommentToken::Link(link_token) => link_token.label.ends_with('\n'),
51        }
52    }
53}
54
55/// Helper struct for formatting possibly nested Markdown lists.
56struct DocCommentListItem {
57    /// Ordered list item separator.
58    delimiter: Option<u64>,
59    /// Flag for an ordered list.
60    is_ordered_list: bool,
61}
62
63struct PendingLink {
64    label: String,
65    path: Option<String>,
66    link_start: usize,
67    link_type: LinkType,
68    destination: String,
69    label_range: Option<Range<usize>>,
70}
71
72/// Parses documentation comment content into a vector of [DocumentationCommentToken]s, keeping
73/// the order in which they were present in the content.
74///
75/// We look for 3 link patterns (ignore the backslash):
76/// "\[label\](path)", "\[path\]" or "\[`path`\]".
77pub fn parse_documentation_comment(documentation_comment: &str) -> Vec<DocumentationCommentToken> {
78    let mut tokens = Vec::new();
79    let mut current_link: Option<PendingLink> = None;
80    let mut is_indented_code_block = false;
81    let mut replacer = |broken_link: BrokenLink<'_>| {
82        if matches!(broken_link.link_type, LinkType::ShortcutUnknown | LinkType::Shortcut) {
83            return Some((broken_link.reference.to_string().into(), "".into()));
84        }
85        None
86    };
87
88    let mut options = Options::empty();
89    options.insert(Options::ENABLE_TABLES);
90    let parser = MarkdownParser::new_with_broken_link_callback(
91        documentation_comment,
92        options,
93        Some(&mut replacer),
94    );
95
96    let mut list_nesting: Vec<DocCommentListItem> = Vec::new();
97    let write_list_item_prefix =
98        |list_nesting: &mut Vec<DocCommentListItem>,
99         tokens: &mut Vec<DocumentationCommentToken>| {
100            if !list_nesting.is_empty() {
101                let indent = "    ".repeat(list_nesting.len() - 1);
102                let list_nesting = list_nesting.last_mut().unwrap();
103
104                tokens.push(DocumentationCommentToken::Content(if list_nesting.is_ordered_list {
105                    let delimiter = list_nesting.delimiter.unwrap_or(0);
106                    list_nesting.delimiter = Some(delimiter + 1);
107                    format!("{indent}{delimiter}. ",)
108                } else {
109                    format!("{indent}- ")
110                }));
111            }
112        };
113    let mut prefix_list_item = false;
114    let mut last_two_events = [None, None];
115    let mut table_alignment: Vec<Alignment> = Vec::new();
116
117    for (event, range) in parser.into_offset_iter() {
118        match &event {
119            Event::Text(text) => {
120                if prefix_list_item {
121                    write_list_item_prefix(&mut list_nesting, &mut tokens);
122                    prefix_list_item = false;
123                }
124                if let Some(link) = current_link.as_mut() {
125                    link.label.push_str(text.as_ref());
126                    link.label_range = Some(range.clone());
127                } else {
128                    let text = {
129                        if is_indented_code_block {
130                            format!("    {text}")
131                        } else {
132                            text.to_string()
133                        }
134                    };
135                    tokens.push(DocumentationCommentToken::Content(text));
136                }
137            }
138            Event::Code(code) => {
139                if prefix_list_item {
140                    write_list_item_prefix(&mut list_nesting, &mut tokens);
141                    prefix_list_item = false;
142                }
143                let complete_code = format!("`{code}`");
144                if let Some(link) = current_link.as_mut() {
145                    link.label.push_str(&complete_code);
146                    link.label_range = Some(range.clone());
147                } else {
148                    tokens.push(DocumentationCommentToken::Content(complete_code));
149                }
150            }
151            Event::Start(tag_start) => match tag_start {
152                Tag::Heading { level, .. } => {
153                    if let Some(last_token) = tokens.last_mut()
154                        && !last_token.clone().ends_with_newline()
155                    {
156                        tokens.push(DocumentationCommentToken::Content("\n".to_string()));
157                    }
158                    tokens.push(DocumentationCommentToken::Content(format!(
159                        "{} ",
160                        heading_level_to_markdown(*level)
161                    )));
162                }
163                Tag::List(list_type) => {
164                    if !list_nesting.is_empty() {
165                        tokens.push(DocumentationCommentToken::Content("\n".to_string()));
166                    }
167                    list_nesting.push(DocCommentListItem {
168                        delimiter: *list_type,
169                        is_ordered_list: list_type.is_some(),
170                    });
171                }
172                Tag::CodeBlock(kind) => match kind {
173                    CodeBlockKind::Fenced(language) => {
174                        if language.trim().is_empty() {
175                            tokens.push(DocumentationCommentToken::Content(String::from(
176                                "```cairo\n",
177                            )));
178                        } else {
179                            tokens.push(DocumentationCommentToken::Content(format!(
180                                "```{language}\n"
181                            )));
182                        }
183                    }
184                    CodeBlockKind::Indented => {
185                        tokens.push(DocumentationCommentToken::Content("\n".to_string()));
186                        is_indented_code_block = true;
187                    }
188                },
189                Tag::Link { link_type, dest_url, .. } => {
190                    let path = match *link_type {
191                        LinkType::ShortcutUnknown | LinkType::Shortcut => None,
192                        _ => Some(dest_url.clone().into_string()),
193                    };
194                    current_link = Some(PendingLink {
195                        label: String::new(),
196                        path,
197                        link_start: range.start,
198                        link_type: *link_type,
199                        destination: dest_url.clone().into_string(),
200                        label_range: None,
201                    });
202                }
203                Tag::Paragraph | Tag::TableRow => {
204                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
205                }
206                Tag::Item => {
207                    prefix_list_item = true;
208                }
209                Tag::Table(alignment) => {
210                    table_alignment = alignment.clone();
211                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
212                }
213                Tag::TableCell => {
214                    tokens.push(DocumentationCommentToken::Content("|".to_string()));
215                }
216                Tag::Strong => {
217                    tokens.push(DocumentationCommentToken::Content("**".to_string()));
218                }
219                Tag::Emphasis => {
220                    tokens.push(DocumentationCommentToken::Content("_".to_string()));
221                }
222                _ => {}
223            },
224            Event::End(tag_end) => match tag_end {
225                TagEnd::Heading(_) | TagEnd::Table => {
226                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
227                }
228                TagEnd::List(_) => {
229                    list_nesting.pop();
230                }
231                TagEnd::Item
232                    if !matches!(last_two_events[0], Some(Event::End(_)))
233                        | !matches!(last_two_events[1], Some(Event::End(_))) =>
234                {
235                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
236                }
237                TagEnd::TableHead => {
238                    tokens.push(DocumentationCommentToken::Content(format!(
239                        "|\n|{}|",
240                        table_alignment
241                            .iter()
242                            .map(|a| {
243                                let (left, right) = get_alignment_markers(a);
244                                format!("{left}---{right}")
245                            })
246                            .join("|")
247                    )));
248                    table_alignment.clear();
249                }
250                TagEnd::CodeBlock => {
251                    if !is_indented_code_block {
252                        tokens.push(DocumentationCommentToken::Content("```\n".to_string()));
253                    }
254                    is_indented_code_block = false;
255                }
256                TagEnd::Link => {
257                    if let Some(link) = current_link {
258                        let link_span = span_from_relative_range(
259                            documentation_comment,
260                            link.link_start..range.end,
261                        );
262                        let (dest_span, dest_text) = link
263                            .label_range
264                            .as_ref()
265                            .and_then(|label_range| {
266                                location_from_link_fields(
267                                    link.link_type,
268                                    &link.destination,
269                                    label_range,
270                                )
271                            })
272                            .map(|(dest_range, dest_text)| {
273                                (
274                                    Some(span_from_relative_range(
275                                        documentation_comment,
276                                        dest_range,
277                                    )),
278                                    Some(dest_text),
279                                )
280                            })
281                            .unwrap_or((None, None));
282                        let md_link = MarkdownLink { link_span, dest_span, dest_text };
283                        tokens.push(DocumentationCommentToken::Link(CommentLinkToken {
284                            label: link.label,
285                            path: link.path,
286                            md_link,
287                        }));
288                    }
289                    current_link = None;
290                }
291                TagEnd::TableRow => {
292                    tokens.push(DocumentationCommentToken::Content("|".to_string()));
293                }
294                TagEnd::Strong => {
295                    tokens.push(DocumentationCommentToken::Content("**".to_string()));
296                }
297                TagEnd::Emphasis => {
298                    tokens.push(DocumentationCommentToken::Content("_".to_string()));
299                }
300                TagEnd::Paragraph => {
301                    tokens.push(DocumentationCommentToken::Content("\n".to_string()));
302                }
303                _ => {}
304            },
305            Event::SoftBreak => {
306                tokens.push(DocumentationCommentToken::Content("\n".to_string()));
307            }
308            Event::Rule => {
309                tokens.push(DocumentationCommentToken::Content("___\n".to_string()));
310            }
311            _ => {}
312        }
313        last_two_events = [last_two_events[1].clone(), Some(event)];
314    }
315
316    if let Some(DocumentationCommentToken::Content(token)) = tokens.first()
317        && token == "\n"
318    {
319        tokens.remove(0);
320    }
321    if let Some(DocumentationCommentToken::Content(token)) = tokens.last_mut() {
322        *token = token.trim_end().to_string();
323        if token.is_empty() {
324            tokens.pop();
325        }
326    }
327
328    tokens
329}
330
331impl fmt::Display for CommentLinkToken {
332    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
333        match self.path.clone() {
334            Some(path) => write!(f, "[{}]({})", self.label, path),
335            None => write!(f, "[{}]", self.label),
336        }
337    }
338}
339
340impl fmt::Display for DocumentationCommentToken {
341    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
342        match self {
343            DocumentationCommentToken::Content(content) => {
344                write!(f, "{content}")
345            }
346            DocumentationCommentToken::Link(link_token) => {
347                write!(f, "{link_token}")
348            }
349        }
350    }
351}
352
353impl<'db> DebugWithDb<'db> for CommentLinkToken {
354    type Db = dyn DocGroup;
355    fn fmt(&self, f: &mut fmt::Formatter<'_>, _db: &Self::Db) -> fmt::Result {
356        f.debug_struct("CommentLinkToken")
357            .field("label", &self.label)
358            .field("path", &self.path)
359            .field("md_link", &self.md_link)
360            .finish()
361    }
362}
363
364/// Converts a byte range within the string into a `TextSpan` relative to the string start.
365fn span_from_relative_range(content: &str, range: Range<usize>) -> TextSpan {
366    let start = TextOffset::START.add_width(TextWidth::at(content, range.start));
367    let end = TextOffset::START.add_width(TextWidth::at(content, range.end));
368    TextSpan::new(start, end)
369}
370
371/// Extracts a location link span and normalized destination text for the given link fields.
372fn location_from_link_fields(
373    link_type: LinkType,
374    destination: &str,
375    label_range: &Range<usize>,
376) -> Option<(Range<usize>, String)> {
377    let (destination_normalized, backticked) = normalize_location_text(destination)?;
378
379    match link_type {
380        LinkType::Inline => {
381            let range = find_inline_destination_range(label_range.end, destination);
382            Some((range, destination_normalized))
383        }
384        LinkType::Collapsed
385        | LinkType::CollapsedUnknown
386        | LinkType::Shortcut
387        | LinkType::ShortcutUnknown => Some((label_range.clone(), destination_normalized)),
388        _ => None,
389    }
390    .map(|(range, text)| (trim_backtick_range(range.clone(), backticked), text))
391}
392
393/// Returns true when the string looks like a location path (letters, digits, '_' or ':').
394fn is_location_string(value: &str) -> bool {
395    !value.is_empty() && value.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == ':')
396}
397
398/// Normalizes the link destination and reports whether it was backticked.
399fn normalize_location_text(value: &str) -> Option<(String, bool)> {
400    let (value, backticked) = strip_backticks(value);
401    is_location_string(value).then(|| (value.to_string(), backticked))
402}
403
404/// Strips backticks around a string if present and reports whether a pair was removed.
405fn strip_backticks(value: &str) -> (&str, bool) {
406    let value = value.trim();
407    if let Some(stripped) = value.strip_prefix('`').and_then(|rest| rest.strip_suffix('`')) {
408        (stripped, true)
409    } else {
410        (value, false)
411    }
412}
413
414/// Trims the range by one on each end when a backticked span is expected.
415fn trim_backtick_range(range: Range<usize>, backticked: bool) -> Range<usize> {
416    if backticked { (range.start + 1)..(range.end - 1) } else { range }
417}
418
419/// Computes the range for an inline destination that follows a label.
420fn find_inline_destination_range(label_last_end: usize, destination: &str) -> Range<usize> {
421    let destination_start = label_last_end + 2;
422    let destination_end = destination_start + destination.len();
423    destination_start..destination_end
424}
425
426/// Maps `HeadingLevel` to the correct markdown marker.
427fn heading_level_to_markdown(heading_level: HeadingLevel) -> String {
428    let heading_char: String = String::from("#");
429    match heading_level {
430        HeadingLevel::H1 => heading_char,
431        HeadingLevel::H2 => heading_char.repeat(2),
432        HeadingLevel::H3 => heading_char.repeat(3),
433        HeadingLevel::H4 => heading_char.repeat(4),
434        HeadingLevel::H5 => heading_char.repeat(5),
435        HeadingLevel::H6 => heading_char.repeat(6),
436    }
437}
438
439/// Maps [`Alignment`] to the correct markdown markers.
440fn get_alignment_markers(alignment: &Alignment) -> (String, String) {
441    let (left, right) = match alignment {
442        Alignment::None => ("", ""),
443        Alignment::Left => (":", ""),
444        Alignment::Right => ("", ":"),
445        Alignment::Center => (":", ":"),
446    };
447    (left.to_string(), right.to_string())
448}
cairo_lang_doc/parser.rs

cairo_lang_doc/
parser.rs
