mant_engine/tldr/
parser.rs

1//! Parses the constrained tldr-pages Markdown dialect into the shared IR.
2
3use std::{error::Error, fmt};
4
5use mant_ir::{TldrCommandPart, TldrDocument, TldrExample, TldrOrigin};
6
7use crate::text_safety::mask_terminal_controls;
8
9/// Source identity attached to a parsed tldr page.
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct TldrPageLocation {
12    /// tldr platform bucket containing the page.
13    pub platform: String,
14    /// tldr language directory containing the page.
15    pub language: String,
16    /// Stable caller-facing source path.
17    pub source_path: String,
18}
19
20/// A tldr page lacks the minimum structure required by the contract.
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub enum TldrParseError {
23    /// No top-level `# command` heading was present.
24    MissingCommandHeading,
25}
26
27impl fmt::Display for TldrParseError {
28    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
29        match self {
30            Self::MissingCommandHeading => {
31                formatter.write_str("tldr page is missing its command heading")
32            }
33        }
34    }
35}
36
37impl Error for TldrParseError {}
38
39/// Parse the tldr placeholder extension and choose the long option variant.
40#[must_use]
41pub fn parse_tldr_command(command: &str) -> Vec<TldrCommandPart> {
42    let mut parts = Vec::new();
43    let mut cursor = 0;
44
45    while cursor < command.len() {
46        let remainder = &command[cursor..];
47        if let Some(escaped) = remainder.strip_prefix(r"\{\{")
48            && let Some(close) = escaped.find(r"\}\}")
49        {
50            push_part(
51                &mut parts,
52                PartKind::Text,
53                format!("{{{{{}}}}}", &escaped[..close]),
54            );
55            cursor += 4 + close + 4;
56            continue;
57        }
58
59        if let Some(placeholder) = remainder.strip_prefix("{{")
60            && let Some(close) = placeholder.find("}}")
61        {
62            let value =
63                resolve_option_placeholder(&placeholder[..close]).unwrap_or(&placeholder[..close]);
64            push_part(&mut parts, PartKind::Placeholder, value.to_owned());
65            cursor += 2 + close + 2;
66            continue;
67        }
68
69        let Some(character) = remainder.chars().next() else {
70            break;
71        };
72        push_part(&mut parts, PartKind::Text, character.to_string());
73        cursor += character.len_utf8();
74    }
75
76    parts
77}
78
79/// Parse one tldr Markdown page without performing any I/O.
80///
81/// # Errors
82///
83/// Returns [`TldrParseError::MissingCommandHeading`] when no `# command`
84/// heading is present.
85pub fn parse_tldr_page(
86    markdown: &str,
87    location: TldrPageLocation,
88) -> Result<TldrDocument, TldrParseError> {
89    let sanitized = mask_terminal_controls(markdown).0;
90    let markdown = sanitized.as_deref().unwrap_or(markdown);
91    let normalized = markdown.replace("\r\n", "\n").replace('\r', "\n");
92    let mut title = String::new();
93    let mut description = Vec::new();
94    let mut more_information = None;
95    let mut examples = Vec::new();
96    let mut pending_page_description = None;
97    let mut pending_example_description = None;
98
99    for line in normalized.lines() {
100        let trimmed = line.trim();
101        if trimmed.is_empty() {
102            flush_description_paragraph(&mut pending_page_description, &mut description);
103            continue;
104        }
105
106        if title.is_empty()
107            && let Some(heading) = trimmed.strip_prefix("# ")
108        {
109            flush_description_paragraph(&mut pending_page_description, &mut description);
110            title = flatten_markdown(heading);
111            continue;
112        }
113
114        if let Some(quote) = trimmed.strip_prefix('>') {
115            let quote = flatten_markdown(quote);
116            if let Some(value) = strip_prefix_ascii_case(&quote, "More information:") {
117                flush_description_paragraph(&mut pending_page_description, &mut description);
118                let value = value.trim();
119                if !value.is_empty() {
120                    more_information = Some(value.to_owned());
121                }
122            } else if quote.is_empty() {
123                flush_description_paragraph(&mut pending_page_description, &mut description);
124            } else {
125                append_soft_line(&mut pending_page_description, quote);
126            }
127            continue;
128        }
129
130        flush_description_paragraph(&mut pending_page_description, &mut description);
131
132        if let Some(item) = trimmed.strip_prefix("- ") {
133            flush_pending(&mut pending_example_description, &mut examples);
134            if let Some((example_description, command)) = extract_trailing_code(item) {
135                examples.push(make_example(example_description, command));
136            } else {
137                let value = flatten_markdown(item);
138                if !value.is_empty() {
139                    pending_example_description = Some(value);
140                }
141            }
142            continue;
143        }
144
145        if let Some(command) = standalone_code(trimmed)
146            && let Some(example_description) = pending_example_description.take()
147        {
148            examples.push(make_example(example_description, command.to_owned()));
149            continue;
150        }
151
152        if pending_example_description.is_some()
153            && line.chars().next().is_some_and(char::is_whitespace)
154        {
155            append_soft_line(&mut pending_example_description, flatten_markdown(trimmed));
156        }
157    }
158
159    flush_description_paragraph(&mut pending_page_description, &mut description);
160    flush_pending(&mut pending_example_description, &mut examples);
161    if title.is_empty() {
162        return Err(TldrParseError::MissingCommandHeading);
163    }
164
165    Ok(TldrDocument {
166        title,
167        description,
168        more_information,
169        examples,
170        platform: location.platform,
171        language: location.language,
172        source_path: location.source_path,
173        origin: TldrOrigin::TldrPages,
174    })
175}
176
177#[derive(Debug, Clone, Copy, PartialEq, Eq)]
178enum PartKind {
179    Text,
180    Placeholder,
181}
182
183fn push_part(parts: &mut Vec<TldrCommandPart>, kind: PartKind, value: String) {
184    if value.is_empty() {
185        return;
186    }
187    match (parts.last_mut(), kind) {
188        (Some(TldrCommandPart::Text { value: previous }), PartKind::Text)
189        | (Some(TldrCommandPart::Placeholder { value: previous }), PartKind::Placeholder) => {
190            previous.push_str(&value);
191        }
192        (_, PartKind::Text) => parts.push(TldrCommandPart::Text { value }),
193        (_, PartKind::Placeholder) => parts.push(TldrCommandPart::Placeholder { value }),
194    }
195}
196
197fn resolve_option_placeholder(value: &str) -> Option<&str> {
198    let choices = value.strip_prefix('[')?.strip_suffix(']')?;
199    let (_, long) = choices.split_once('|')?;
200    (!long.is_empty()).then_some(long)
201}
202
203fn flush_pending(pending: &mut Option<String>, examples: &mut Vec<TldrExample>) {
204    if let Some(description) = pending.take() {
205        examples.push(make_example(description, String::new()));
206    }
207}
208
209fn append_soft_line(paragraph: &mut Option<String>, line: String) {
210    if line.is_empty() {
211        return;
212    }
213    if let Some(paragraph) = paragraph {
214        paragraph.push(' ');
215        paragraph.push_str(&line);
216    } else {
217        *paragraph = Some(line);
218    }
219}
220
221fn flush_description_paragraph(pending: &mut Option<String>, paragraphs: &mut Vec<String>) {
222    if let Some(paragraph) = pending.take() {
223        paragraphs.push(paragraph);
224    }
225}
226
227fn make_example(mut description: String, command: String) -> TldrExample {
228    let description_len = description
229        .trim_end()
230        .trim_end_matches(':')
231        .trim_end()
232        .len();
233    description.truncate(description_len);
234    TldrExample {
235        description,
236        command_parts: parse_tldr_command(&command),
237        command,
238    }
239}
240
241fn extract_trailing_code(value: &str) -> Option<(String, String)> {
242    let trimmed = value.trim_end();
243    let close = trimmed.strip_suffix('`')?;
244    let open = close.rfind('`')?;
245    let command = &close[open + 1..];
246    if command.is_empty() || command.contains('`') {
247        return None;
248    }
249    let description = flatten_markdown(close[..open].trim_end().trim_end_matches(':'));
250    Some((description, command.to_owned()))
251}
252
253fn standalone_code(value: &str) -> Option<&str> {
254    value.strip_prefix('`')?.strip_suffix('`')
255}
256
257fn strip_prefix_ascii_case<'a>(value: &'a str, prefix: &str) -> Option<&'a str> {
258    let candidate = value.get(..prefix.len())?;
259    candidate
260        .eq_ignore_ascii_case(prefix)
261        .then(|| &value[prefix.len()..])
262}
263
264fn flatten_markdown(value: &str) -> String {
265    let mut flattened = flatten_links(value);
266    for marker in ["**", "__", "*", "_"] {
267        flattened = strip_paired_marker(&flattened, marker);
268    }
269    flattened = flattened.replace(['`', '<', '>'], "");
270    flattened.split_whitespace().collect::<Vec<_>>().join(" ")
271}
272
273fn flatten_links(value: &str) -> String {
274    let mut flattened = String::new();
275    let mut remainder = value;
276    while let Some(open) = remainder.find('[') {
277        flattened.push_str(&remainder[..open]);
278        let after_open = &remainder[open + 1..];
279        let Some(label_end) = after_open.find("](") else {
280            flattened.push_str(&remainder[open..]);
281            return flattened;
282        };
283        let after_target_open = &after_open[label_end + 2..];
284        let Some(target_end) = after_target_open.find(')') else {
285            flattened.push_str(&remainder[open..]);
286            return flattened;
287        };
288        flattened.push_str(&after_open[..label_end]);
289        remainder = &after_target_open[target_end + 1..];
290    }
291    flattened.push_str(remainder);
292    flattened
293}
294
295fn strip_paired_marker(value: &str, marker: &str) -> String {
296    let mut stripped = String::new();
297    let mut remainder = value;
298    while let Some(open) = remainder.find(marker) {
299        let after_open = &remainder[open + marker.len()..];
300        let Some(close) = after_open.find(marker) else {
301            break;
302        };
303        stripped.push_str(&remainder[..open]);
304        stripped.push_str(&after_open[..close]);
305        remainder = &after_open[close + marker.len()..];
306    }
307    stripped.push_str(remainder);
308    stripped
309}
310
311#[cfg(test)]
312mod tests {
313    use mant_ir::TldrCommandPart;
314
315    use super::{TldrPageLocation, TldrParseError, parse_tldr_command, parse_tldr_page};
316
317    const PAGE: &str = r"# tar
318
319> Archiving utility.
320> More information: <https://www.gnu.org/software/tar>.
321
322- Create an archive:
323  `tar {{[-c|--create]}} {{path/to/archive.tar}} {{path/to/file}}`
324
325- Extract an archive: `tar --extract --file {{path/to/archive.tar}}`
326";
327
328    fn location() -> TldrPageLocation {
329        TldrPageLocation {
330            platform: "linux".to_owned(),
331            language: "en".to_owned(),
332            source_path: "/cache/pages/linux/tar.md".to_owned(),
333        }
334    }
335
336    #[test]
337    fn parses_examples_markup_and_long_option_placeholders() {
338        let page = parse_tldr_page(PAGE, location()).expect("valid tldr page");
339
340        assert_eq!(page.title, "tar");
341        assert_eq!(page.description, ["Archiving utility."]);
342        assert_eq!(
343            page.more_information.as_deref(),
344            Some("https://www.gnu.org/software/tar.")
345        );
346        assert_eq!(page.examples.len(), 2);
347        assert_eq!(
348            page.examples[0].command,
349            "tar {{[-c|--create]}} {{path/to/archive.tar}} {{path/to/file}}"
350        );
351        assert_eq!(
352            page.examples[0].command_parts,
353            [
354                TldrCommandPart::Text {
355                    value: "tar ".to_owned()
356                },
357                TldrCommandPart::Placeholder {
358                    value: "--create".to_owned()
359                },
360                TldrCommandPart::Text {
361                    value: " ".to_owned()
362                },
363                TldrCommandPart::Placeholder {
364                    value: "path/to/archive.tar".to_owned()
365                },
366                TldrCommandPart::Text {
367                    value: " ".to_owned()
368                },
369                TldrCommandPart::Placeholder {
370                    value: "path/to/file".to_owned()
371                },
372            ]
373        );
374    }
375
376    #[test]
377    fn preserves_escaped_braces_and_unicode_text() {
378        assert_eq!(
379            parse_tldr_command(r"echo \{\{不是占位符\}\} {{值}}"),
380            [
381                TldrCommandPart::Text {
382                    value: "echo {{不是占位符}} ".to_owned()
383                },
384                TldrCommandPart::Placeholder {
385                    value: "值".to_owned()
386                },
387            ]
388        );
389    }
390
391    #[test]
392    fn accepts_inline_examples_and_flattens_description_markup() {
393        let page = parse_tldr_page(
394            "# demo\n> Use **demo** with [docs](https://example.test).\n- Run it: `demo _x_`\n",
395            location(),
396        )
397        .expect("valid tldr page");
398
399        assert_eq!(page.description, ["Use demo with docs."]);
400        assert_eq!(page.examples[0].description, "Run it");
401        assert_eq!(page.examples[0].command, "demo _x_");
402    }
403
404    #[test]
405    fn commonmark_soft_breaks_do_not_become_rendered_line_breaks() {
406        let page = parse_tldr_page(
407            "# demo\n\n> A description wrapped in the source\n> remains one rendered paragraph.\n>\n> A distinct paragraph remains distinct.\n> More information: <https://example.test/demo>.\n\n- Run a command whose explanation is\n  wrapped only for source readability:\n\n  `demo --long-option value`\n",
408            location(),
409        )
410        .expect("valid source-wrapped tldr page");
411
412        assert_eq!(
413            page.description,
414            [
415                "A description wrapped in the source remains one rendered paragraph.",
416                "A distinct paragraph remains distinct.",
417            ]
418        );
419        assert_eq!(
420            page.more_information.as_deref(),
421            Some("https://example.test/demo.")
422        );
423        assert_eq!(
424            page.examples[0].description,
425            "Run a command whose explanation is wrapped only for source readability"
426        );
427        assert_eq!(page.examples[0].command, "demo --long-option value");
428    }
429
430    #[test]
431    fn masks_terminal_control_characters_before_parsing() {
432        let page = parse_tldr_page(
433            "# de\u{1b}[2Jmo\n> safe\u{85} description\n- Run: `demo\u{7}`\n",
434            location(),
435        )
436        .expect("valid tldr page");
437
438        assert_eq!(page.title, "de [2Jmo");
439        assert_eq!(page.description, ["safe description"]);
440        assert_eq!(page.examples[0].command, "demo ");
441    }
442
443    #[test]
444    fn retains_an_example_description_when_its_command_is_missing() {
445        let page =
446            parse_tldr_page("# demo\n- Explain only:\n", location()).expect("valid tldr page");
447        assert_eq!(page.examples[0].description, "Explain only");
448        assert!(page.examples[0].command.is_empty());
449    }
450
451    #[test]
452    fn rejects_a_page_without_a_command_heading() {
453        assert_eq!(
454            parse_tldr_page("> description only", location()),
455            Err(TldrParseError::MissingCommandHeading)
456        );
457    }
458}
mant_engine/tldr/parser.rs

mant_engine/tldr/
parser.rs
