mant_core/tldr/
parser.rs

1//! Parses the constrained tldr-pages Markdown dialect into the shared AST.
2
3use std::{error::Error, fmt};
4
5use mant_ast::{TldrCommandPart, TldrDocument, TldrExample, TldrOrigin};
6
7use crate::text_safety::mask_terminal_controls;
8
9/// Source identity attached to a parsed tldr page.
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct TldrPageLocation {
12    pub platform: String,
13    pub language: String,
14    pub source_path: String,
15}
16
17/// A tldr page lacks the minimum structure required by the contract.
18#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum TldrParseError {
20    MissingCommandHeading,
21}
22
23impl fmt::Display for TldrParseError {
24    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
25        match self {
26            Self::MissingCommandHeading => {
27                formatter.write_str("tldr page is missing its command heading")
28            }
29        }
30    }
31}
32
33impl Error for TldrParseError {}
34
35/// Parse the tldr placeholder extension and choose the long option variant.
36#[must_use]
37pub fn parse_tldr_command(command: &str) -> Vec<TldrCommandPart> {
38    let mut parts = Vec::new();
39    let mut cursor = 0;
40
41    while cursor < command.len() {
42        let remainder = &command[cursor..];
43        if let Some(escaped) = remainder.strip_prefix(r"\{\{")
44            && let Some(close) = escaped.find(r"\}\}")
45        {
46            push_part(
47                &mut parts,
48                PartKind::Text,
49                format!("{{{{{}}}}}", &escaped[..close]),
50            );
51            cursor += 4 + close + 4;
52            continue;
53        }
54
55        if let Some(placeholder) = remainder.strip_prefix("{{")
56            && let Some(close) = placeholder.find("}}")
57        {
58            let value =
59                resolve_option_placeholder(&placeholder[..close]).unwrap_or(&placeholder[..close]);
60            push_part(&mut parts, PartKind::Placeholder, value.to_owned());
61            cursor += 2 + close + 2;
62            continue;
63        }
64
65        let Some(character) = remainder.chars().next() else {
66            break;
67        };
68        push_part(&mut parts, PartKind::Text, character.to_string());
69        cursor += character.len_utf8();
70    }
71
72    parts
73}
74
75/// Parse one tldr Markdown page without performing any I/O.
76///
77/// # Errors
78///
79/// Returns [`TldrParseError::MissingCommandHeading`] when no `# command`
80/// heading is present.
81pub fn parse_tldr_page(
82    markdown: &str,
83    location: TldrPageLocation,
84) -> Result<TldrDocument, TldrParseError> {
85    let sanitized = mask_terminal_controls(markdown).0;
86    let markdown = sanitized.as_deref().unwrap_or(markdown);
87    let normalized = markdown.replace("\r\n", "\n").replace('\r', "\n");
88    let mut title = String::new();
89    let mut description = Vec::new();
90    let mut more_information = None;
91    let mut examples = Vec::new();
92    let mut pending_page_description = None;
93    let mut pending_example_description = None;
94
95    for line in normalized.lines() {
96        let trimmed = line.trim();
97        if trimmed.is_empty() {
98            flush_description_paragraph(&mut pending_page_description, &mut description);
99            continue;
100        }
101
102        if title.is_empty()
103            && let Some(heading) = trimmed.strip_prefix("# ")
104        {
105            flush_description_paragraph(&mut pending_page_description, &mut description);
106            title = flatten_markdown(heading);
107            continue;
108        }
109
110        if let Some(quote) = trimmed.strip_prefix('>') {
111            let quote = flatten_markdown(quote);
112            if let Some(value) = strip_prefix_ascii_case(&quote, "More information:") {
113                flush_description_paragraph(&mut pending_page_description, &mut description);
114                let value = value.trim();
115                if !value.is_empty() {
116                    more_information = Some(value.to_owned());
117                }
118            } else if quote.is_empty() {
119                flush_description_paragraph(&mut pending_page_description, &mut description);
120            } else {
121                append_soft_line(&mut pending_page_description, quote);
122            }
123            continue;
124        }
125
126        flush_description_paragraph(&mut pending_page_description, &mut description);
127
128        if let Some(item) = trimmed.strip_prefix("- ") {
129            flush_pending(&mut pending_example_description, &mut examples);
130            if let Some((example_description, command)) = extract_trailing_code(item) {
131                examples.push(make_example(example_description, command));
132            } else {
133                let value = flatten_markdown(item);
134                if !value.is_empty() {
135                    pending_example_description = Some(value);
136                }
137            }
138            continue;
139        }
140
141        if let Some(command) = standalone_code(trimmed)
142            && let Some(example_description) = pending_example_description.take()
143        {
144            examples.push(make_example(example_description, command.to_owned()));
145            continue;
146        }
147
148        if pending_example_description.is_some()
149            && line.chars().next().is_some_and(char::is_whitespace)
150        {
151            append_soft_line(&mut pending_example_description, flatten_markdown(trimmed));
152        }
153    }
154
155    flush_description_paragraph(&mut pending_page_description, &mut description);
156    flush_pending(&mut pending_example_description, &mut examples);
157    if title.is_empty() {
158        return Err(TldrParseError::MissingCommandHeading);
159    }
160
161    Ok(TldrDocument {
162        title,
163        description,
164        more_information,
165        examples,
166        platform: location.platform,
167        language: location.language,
168        source_path: location.source_path,
169        origin: TldrOrigin::TldrPages,
170    })
171}
172
173#[derive(Debug, Clone, Copy, PartialEq, Eq)]
174enum PartKind {
175    Text,
176    Placeholder,
177}
178
179fn push_part(parts: &mut Vec<TldrCommandPart>, kind: PartKind, value: String) {
180    if value.is_empty() {
181        return;
182    }
183    match (parts.last_mut(), kind) {
184        (Some(TldrCommandPart::Text { value: previous }), PartKind::Text)
185        | (Some(TldrCommandPart::Placeholder { value: previous }), PartKind::Placeholder) => {
186            previous.push_str(&value);
187        }
188        (_, PartKind::Text) => parts.push(TldrCommandPart::Text { value }),
189        (_, PartKind::Placeholder) => parts.push(TldrCommandPart::Placeholder { value }),
190    }
191}
192
193fn resolve_option_placeholder(value: &str) -> Option<&str> {
194    let choices = value.strip_prefix('[')?.strip_suffix(']')?;
195    let (_, long) = choices.split_once('|')?;
196    (!long.is_empty()).then_some(long)
197}
198
199fn flush_pending(pending: &mut Option<String>, examples: &mut Vec<TldrExample>) {
200    if let Some(description) = pending.take() {
201        examples.push(make_example(description, String::new()));
202    }
203}
204
205fn append_soft_line(paragraph: &mut Option<String>, line: String) {
206    if line.is_empty() {
207        return;
208    }
209    if let Some(paragraph) = paragraph {
210        paragraph.push(' ');
211        paragraph.push_str(&line);
212    } else {
213        *paragraph = Some(line);
214    }
215}
216
217fn flush_description_paragraph(pending: &mut Option<String>, paragraphs: &mut Vec<String>) {
218    if let Some(paragraph) = pending.take() {
219        paragraphs.push(paragraph);
220    }
221}
222
223fn make_example(mut description: String, command: String) -> TldrExample {
224    let description_len = description
225        .trim_end()
226        .trim_end_matches(':')
227        .trim_end()
228        .len();
229    description.truncate(description_len);
230    TldrExample {
231        description,
232        command_parts: parse_tldr_command(&command),
233        command,
234    }
235}
236
237fn extract_trailing_code(value: &str) -> Option<(String, String)> {
238    let trimmed = value.trim_end();
239    let close = trimmed.strip_suffix('`')?;
240    let open = close.rfind('`')?;
241    let command = &close[open + 1..];
242    if command.is_empty() || command.contains('`') {
243        return None;
244    }
245    let description = flatten_markdown(close[..open].trim_end().trim_end_matches(':'));
246    Some((description, command.to_owned()))
247}
248
249fn standalone_code(value: &str) -> Option<&str> {
250    value.strip_prefix('`')?.strip_suffix('`')
251}
252
253fn strip_prefix_ascii_case<'a>(value: &'a str, prefix: &str) -> Option<&'a str> {
254    let candidate = value.get(..prefix.len())?;
255    candidate
256        .eq_ignore_ascii_case(prefix)
257        .then(|| &value[prefix.len()..])
258}
259
260fn flatten_markdown(value: &str) -> String {
261    let mut flattened = flatten_links(value);
262    for marker in ["**", "__", "*", "_"] {
263        flattened = strip_paired_marker(&flattened, marker);
264    }
265    flattened = flattened.replace(['`', '<', '>'], "");
266    flattened.split_whitespace().collect::<Vec<_>>().join(" ")
267}
268
269fn flatten_links(value: &str) -> String {
270    let mut flattened = String::new();
271    let mut remainder = value;
272    while let Some(open) = remainder.find('[') {
273        flattened.push_str(&remainder[..open]);
274        let after_open = &remainder[open + 1..];
275        let Some(label_end) = after_open.find("](") else {
276            flattened.push_str(&remainder[open..]);
277            return flattened;
278        };
279        let after_target_open = &after_open[label_end + 2..];
280        let Some(target_end) = after_target_open.find(')') else {
281            flattened.push_str(&remainder[open..]);
282            return flattened;
283        };
284        flattened.push_str(&after_open[..label_end]);
285        remainder = &after_target_open[target_end + 1..];
286    }
287    flattened.push_str(remainder);
288    flattened
289}
290
291fn strip_paired_marker(value: &str, marker: &str) -> String {
292    let mut stripped = String::new();
293    let mut remainder = value;
294    while let Some(open) = remainder.find(marker) {
295        let after_open = &remainder[open + marker.len()..];
296        let Some(close) = after_open.find(marker) else {
297            break;
298        };
299        stripped.push_str(&remainder[..open]);
300        stripped.push_str(&after_open[..close]);
301        remainder = &after_open[close + marker.len()..];
302    }
303    stripped.push_str(remainder);
304    stripped
305}
306
307#[cfg(test)]
308mod tests {
309    use mant_ast::TldrCommandPart;
310
311    use super::{TldrPageLocation, TldrParseError, parse_tldr_command, parse_tldr_page};
312
313    const PAGE: &str = r"# tar
314
315> Archiving utility.
316> More information: <https://www.gnu.org/software/tar>.
317
318- Create an archive:
319  `tar {{[-c|--create]}} {{path/to/archive.tar}} {{path/to/file}}`
320
321- Extract an archive: `tar --extract --file {{path/to/archive.tar}}`
322";
323
324    fn location() -> TldrPageLocation {
325        TldrPageLocation {
326            platform: "linux".to_owned(),
327            language: "en".to_owned(),
328            source_path: "/cache/pages/linux/tar.md".to_owned(),
329        }
330    }
331
332    #[test]
333    fn parses_examples_markup_and_long_option_placeholders() {
334        let page = parse_tldr_page(PAGE, location()).expect("valid tldr page");
335
336        assert_eq!(page.title, "tar");
337        assert_eq!(page.description, ["Archiving utility."]);
338        assert_eq!(
339            page.more_information.as_deref(),
340            Some("https://www.gnu.org/software/tar.")
341        );
342        assert_eq!(page.examples.len(), 2);
343        assert_eq!(
344            page.examples[0].command,
345            "tar {{[-c|--create]}} {{path/to/archive.tar}} {{path/to/file}}"
346        );
347        assert_eq!(
348            page.examples[0].command_parts,
349            [
350                TldrCommandPart::Text {
351                    value: "tar ".to_owned()
352                },
353                TldrCommandPart::Placeholder {
354                    value: "--create".to_owned()
355                },
356                TldrCommandPart::Text {
357                    value: " ".to_owned()
358                },
359                TldrCommandPart::Placeholder {
360                    value: "path/to/archive.tar".to_owned()
361                },
362                TldrCommandPart::Text {
363                    value: " ".to_owned()
364                },
365                TldrCommandPart::Placeholder {
366                    value: "path/to/file".to_owned()
367                },
368            ]
369        );
370    }
371
372    #[test]
373    fn preserves_escaped_braces_and_unicode_text() {
374        assert_eq!(
375            parse_tldr_command(r"echo \{\{不是占位符\}\} {{值}}"),
376            [
377                TldrCommandPart::Text {
378                    value: "echo {{不是占位符}} ".to_owned()
379                },
380                TldrCommandPart::Placeholder {
381                    value: "值".to_owned()
382                },
383            ]
384        );
385    }
386
387    #[test]
388    fn accepts_inline_examples_and_flattens_description_markup() {
389        let page = parse_tldr_page(
390            "# demo\n> Use **demo** with [docs](https://example.test).\n- Run it: `demo _x_`\n",
391            location(),
392        )
393        .expect("valid tldr page");
394
395        assert_eq!(page.description, ["Use demo with docs."]);
396        assert_eq!(page.examples[0].description, "Run it");
397        assert_eq!(page.examples[0].command, "demo _x_");
398    }
399
400    #[test]
401    fn commonmark_soft_breaks_do_not_become_rendered_line_breaks() {
402        let page = parse_tldr_page(
403            "# demo\n\n> A description wrapped in the source\n> remains one rendered paragraph.\n>\n> A distinct paragraph remains distinct.\n> More information: <https://example.test/demo>.\n\n- Run a command whose explanation is\n  wrapped only for source readability:\n\n  `demo --long-option value`\n",
404            location(),
405        )
406        .expect("valid source-wrapped tldr page");
407
408        assert_eq!(
409            page.description,
410            [
411                "A description wrapped in the source remains one rendered paragraph.",
412                "A distinct paragraph remains distinct.",
413            ]
414        );
415        assert_eq!(
416            page.more_information.as_deref(),
417            Some("https://example.test/demo.")
418        );
419        assert_eq!(
420            page.examples[0].description,
421            "Run a command whose explanation is wrapped only for source readability"
422        );
423        assert_eq!(page.examples[0].command, "demo --long-option value");
424    }
425
426    #[test]
427    fn masks_terminal_control_characters_before_parsing() {
428        let page = parse_tldr_page(
429            "# de\u{1b}[2Jmo\n> safe\u{85} description\n- Run: `demo\u{7}`\n",
430            location(),
431        )
432        .expect("valid tldr page");
433
434        assert_eq!(page.title, "de [2Jmo");
435        assert_eq!(page.description, ["safe description"]);
436        assert_eq!(page.examples[0].command, "demo ");
437    }
438
439    #[test]
440    fn retains_an_example_description_when_its_command_is_missing() {
441        let page =
442            parse_tldr_page("# demo\n- Explain only:\n", location()).expect("valid tldr page");
443        assert_eq!(page.examples[0].description, "Explain only");
444        assert!(page.examples[0].command.is_empty());
445    }
446
447    #[test]
448    fn rejects_a_page_without_a_command_heading() {
449        assert_eq!(
450            parse_tldr_page("> description only", location()),
451            Err(TldrParseError::MissingCommandHeading)
452        );
453    }
454}
mant_core/tldr/parser.rs

mant_core/tldr/
parser.rs
