mant_core/
groff_html.rs

1//! Parses man-db/groff HTML when source-level libmandoc reports unsupported roff.
2
3use mant_ast::{
4    Block, DefinitionItem, DocumentMeta, DocumentSchema, DocumentSource, Inline, LayoutHint,
5    ListItem, ListKind, MantDocument, Producer, Section, SourceFormat,
6};
7use scraper::{ElementRef, Html, Selector};
8
9/// Normalize one complete `man -Thtml` document into `ManT`'s stable AST.
10#[must_use]
11pub fn parse_groff_html(html: &str, source_path: Option<String>) -> MantDocument {
12    let document = Html::parse_document(html);
13    let mut sections = Vec::new();
14    let mut next_id = 1;
15    if let Ok(body_selector) = Selector::parse("body")
16        && let Some(body) = document.select(&body_selector).next()
17    {
18        parse_body(body, &mut sections, &mut next_id);
19    }
20
21    let mut sections = nest_sections(sections);
22    crate::definitions::identify_definitions(&mut sections, &std::collections::HashSet::new());
23    MantDocument {
24        schema: DocumentSchema::V3,
25        producer: Producer {
26            name: "mant".to_owned(),
27            version: env!("CARGO_PKG_VERSION").to_owned(),
28            engine: None,
29        },
30        source: DocumentSource {
31            format: SourceFormat::GroffHtml,
32            path: source_path,
33            renderer: Some("man -Thtml".to_owned()),
34        },
35        meta: DocumentMeta::default(),
36        blocks: Vec::new(),
37        diagnostics: Vec::new(),
38        sections,
39    }
40}
41
42struct FlatSection {
43    level: u8,
44    section: Section,
45}
46
47fn parse_body(body: ElementRef<'_>, sections: &mut Vec<FlatSection>, next_id: &mut usize) {
48    for child in body.children() {
49        if let Some(text) = child.value().as_text() {
50            let text = normalize_text(text.text.as_ref());
51            if !text.is_empty()
52                && let Some(current) = sections.last_mut()
53            {
54                current.section.blocks.push(paragraph(text, 0));
55            }
56            continue;
57        }
58        let Some(element) = ElementRef::wrap(child) else {
59            continue;
60        };
61        let tag = element.value().name();
62        if matches!(tag, "h1" | "hr" | "br") || is_toc_link(element) {
63            continue;
64        }
65        if let Some(level) = heading_level(tag) {
66            let title = normalize_text(&element.text().collect::<String>());
67            if !title.is_empty() {
68                sections.push(FlatSection {
69                    level,
70                    section: Section {
71                        id: format!("groff-section-{}", *next_id),
72                        title,
73                        spacing_before_lines: u16::from(!sections.is_empty()),
74                        blocks: Vec::new(),
75                        children: Vec::new(),
76                        source: None,
77                    },
78                });
79                *next_id += 1;
80            }
81            continue;
82        }
83        let Some(current) = sections.last_mut() else {
84            continue;
85        };
86        if tag == "table" {
87            current.section.blocks.extend(parse_layout_table(element));
88        } else if let Some(block) = parse_block(element, parse_indent(element)) {
89            current.section.blocks.push(block);
90        }
91    }
92}
93
94fn is_toc_link(element: ElementRef<'_>) -> bool {
95    element.value().name() == "a"
96        && element
97            .value()
98            .attr("href")
99            .is_some_and(|href| href.starts_with('#'))
100}
101
102fn heading_level(tag: &str) -> Option<u8> {
103    match tag {
104        "h2" => Some(2),
105        "h3" => Some(3),
106        "h4" => Some(4),
107        "h5" => Some(5),
108        "h6" => Some(6),
109        _ => None,
110    }
111}
112
113fn nest_sections(mut flat: Vec<FlatSection>) -> Vec<Section> {
114    fn take_one(flat: &mut [FlatSection], index: &mut usize) -> Section {
115        let level = flat[*index].level;
116        let mut section = std::mem::replace(
117            &mut flat[*index].section,
118            Section {
119                id: String::new(),
120                title: String::new(),
121                spacing_before_lines: 0,
122                blocks: Vec::new(),
123                children: Vec::new(),
124                source: None,
125            },
126        );
127        *index += 1;
128        while *index < flat.len() && flat[*index].level > level {
129            section.children.push(take_one(flat, index));
130        }
131        section
132    }
133
134    let mut roots = Vec::new();
135    let mut index = 0;
136    while index < flat.len() {
137        roots.push(take_one(&mut flat, &mut index));
138    }
139    roots
140}
141
142fn parse_layout_table(table: ElementRef<'_>) -> Vec<Block> {
143    let (Ok(row_selector), Ok(cell_selector)) = (Selector::parse("tr"), Selector::parse("td"))
144    else {
145        return Vec::new();
146    };
147    let mut blocks = Vec::new();
148    for row in table.select(&row_selector) {
149        let mut cumulative_width = 0;
150        for cell in row.select(&cell_selector) {
151            let indent = percent_to_columns(cumulative_width);
152            for child in cell.children().filter_map(ElementRef::wrap) {
153                if matches!(child.value().name(), "p" | "pre" | "ul" | "ol" | "dl")
154                    && let Some(block) = parse_block(child, indent)
155                {
156                    blocks.push(block);
157                }
158            }
159            cumulative_width += cell
160                .value()
161                .attr("width")
162                .and_then(parse_percentage)
163                .unwrap_or(0);
164        }
165    }
166    blocks
167}
168
169fn parse_block(element: ElementRef<'_>, indent_columns: u16) -> Option<Block> {
170    let layout = LayoutHint {
171        indent_columns,
172        ..LayoutHint::default()
173    };
174    match element.value().name() {
175        "p" => {
176            let children = parse_inline_children(element, false);
177            if children.is_empty() {
178                Some(Block::VerticalSpace {
179                    lines: 1,
180                    source: None,
181                })
182            } else {
183                Some(Block::Paragraph {
184                    children,
185                    layout,
186                    source: None,
187                })
188            }
189        }
190        "pre" => {
191            let mut children = parse_inline_children(element, true);
192            trim_pre_boundaries(&mut children);
193            (!children.is_empty()).then_some(Block::Preformatted {
194                children,
195                language: None,
196                layout,
197                source: None,
198            })
199        }
200        "ul" | "ol" => parse_list(element, layout),
201        "dl" => parse_definition_list(element, layout),
202        _ => {
203            let children = parse_inline_children(element, false);
204            (!children.is_empty()).then_some(Block::Paragraph {
205                children,
206                layout,
207                source: None,
208            })
209        }
210    }
211}
212
213fn parse_list(element: ElementRef<'_>, layout: LayoutHint) -> Option<Block> {
214    let kind = if element.value().name() == "ol" {
215        ListKind::Ordered
216    } else {
217        ListKind::Bullet
218    };
219    let start = (kind == ListKind::Ordered)
220        .then(|| element.value().attr("start")?.parse().ok())
221        .flatten();
222    let items = element
223        .children()
224        .filter_map(ElementRef::wrap)
225        .filter(|child| child.value().name() == "li")
226        .filter_map(|item| {
227            let mut blocks = item
228                .children()
229                .filter_map(ElementRef::wrap)
230                .filter_map(|child| {
231                    matches!(child.value().name(), "p" | "pre" | "ul" | "ol" | "dl")
232                        .then(|| parse_block(child, 0))
233                        .flatten()
234                })
235                .collect::<Vec<_>>();
236            if blocks.is_empty() {
237                let children = parse_inline_children(item, false);
238                if !children.is_empty() {
239                    blocks.push(Block::Paragraph {
240                        children,
241                        layout: LayoutHint::default(),
242                        source: None,
243                    });
244                }
245            }
246            (!blocks.is_empty()).then_some(ListItem { blocks })
247        })
248        .collect::<Vec<_>>();
249
250    (!items.is_empty()).then_some(Block::List {
251        kind,
252        start,
253        compact: false,
254        items,
255        layout,
256        source: None,
257    })
258}
259
260fn parse_definition_list(element: ElementRef<'_>, layout: LayoutHint) -> Option<Block> {
261    let mut items = Vec::new();
262    let mut terms = Vec::new();
263
264    for child in element.children().filter_map(ElementRef::wrap) {
265        match child.value().name() {
266            "dt" => {
267                let term = parse_inline_children(child, false);
268                if !term.is_empty() {
269                    terms.push(term);
270                }
271            }
272            "dd" => {
273                let mut description = child
274                    .children()
275                    .filter_map(ElementRef::wrap)
276                    .filter_map(|nested| {
277                        matches!(nested.value().name(), "p" | "pre" | "ul" | "ol" | "dl")
278                            .then(|| parse_block(nested, 0))
279                            .flatten()
280                    })
281                    .collect::<Vec<_>>();
282                if description.is_empty() {
283                    let children = parse_inline_children(child, false);
284                    if !children.is_empty() {
285                        description.push(Block::Paragraph {
286                            children,
287                            layout: LayoutHint::default(),
288                            source: None,
289                        });
290                    }
291                }
292                if !terms.is_empty() || !description.is_empty() {
293                    items.push(DefinitionItem {
294                        identity: None,
295                        inline_term: crate::mandoc::inline::terms_fit_inline(
296                            &terms,
297                            crate::mandoc::inline::DEFAULT_INLINE_TERM_MAX_WIDTH,
298                        ),
299                        terms: std::mem::take(&mut terms),
300                        description,
301                        spacing_before_lines: None,
302                    });
303                }
304            }
305            _ => {}
306        }
307    }
308    if !terms.is_empty() {
309        items.push(DefinitionItem {
310            identity: None,
311            inline_term: crate::mandoc::inline::terms_fit_inline(
312                &terms,
313                crate::mandoc::inline::DEFAULT_INLINE_TERM_MAX_WIDTH,
314            ),
315            terms,
316            description: Vec::new(),
317            spacing_before_lines: None,
318        });
319    }
320
321    (!items.is_empty()).then_some(Block::DefinitionList {
322        items,
323        compact: false,
324        layout,
325        source: None,
326    })
327}
328
329fn parse_inline_children(element: ElementRef<'_>, preserve_newlines: bool) -> Vec<Inline> {
330    let mut children = Vec::new();
331    for child in element.children() {
332        if let Some(text) = child.value().as_text() {
333            let value = if preserve_newlines {
334                normalize_pre_text(text.text.as_ref())
335            } else {
336                normalize_inline_text(text.text.as_ref())
337            };
338            if !value.is_empty() {
339                children.push(Inline::Text { value });
340            }
341        } else if let Some(element) = ElementRef::wrap(child) {
342            children.extend(parse_inline_element(element, preserve_newlines));
343        }
344    }
345    children
346}
347
348fn parse_inline_element(element: ElementRef<'_>, preserve_newlines: bool) -> Vec<Inline> {
349    let children = parse_inline_children(element, preserve_newlines);
350    match element.value().name() {
351        "br" => vec![Inline::LineBreak],
352        "b" | "strong" => (!children.is_empty())
353            .then_some(Inline::Strong { children })
354            .into_iter()
355            .collect(),
356        "i" | "em" => (!children.is_empty())
357            .then_some(Inline::Emphasis { children })
358            .into_iter()
359            .collect(),
360        "code" | "tt" => {
361            let value = inline_text(&children);
362            (!value.is_empty())
363                .then_some(Inline::Code { value })
364                .into_iter()
365                .collect()
366        }
367        "a" => element
368            .value()
369            .attr("href")
370            .map_or(children.clone(), |target| {
371                if children.is_empty() {
372                    Vec::new()
373                } else {
374                    vec![Inline::ExternalLink {
375                        uri: target.to_owned(),
376                        title: element.value().attr("title").map(str::to_owned),
377                        children,
378                    }]
379                }
380            }),
381        _ => children,
382    }
383}
384
385fn parse_indent(element: ElementRef<'_>) -> u16 {
386    element
387        .value()
388        .attr("style")
389        .and_then(|style| {
390            style.split(';').find_map(|declaration| {
391                let (name, value) = declaration.split_once(':')?;
392                (name.trim().eq_ignore_ascii_case("margin-left"))
393                    .then(|| parse_percentage(value.trim()))
394                    .flatten()
395            })
396        })
397        .map_or(0, percent_to_columns)
398}
399
400fn parse_percentage(value: &str) -> Option<u16> {
401    value.trim().strip_suffix('%')?.trim().parse().ok()
402}
403
404fn percent_to_columns(percent: u16) -> u16 {
405    ((u32::from(percent) * 80 + 50) / 100)
406        .try_into()
407        .unwrap_or(u16::MAX)
408}
409
410fn paragraph(value: String, indent_columns: u16) -> Block {
411    Block::Paragraph {
412        children: vec![Inline::Text { value }],
413        layout: LayoutHint {
414            indent_columns,
415            ..LayoutHint::default()
416        },
417        source: None,
418    }
419}
420
421fn normalize_text(value: &str) -> String {
422    value.split_whitespace().collect::<Vec<_>>().join(" ")
423}
424
425/// Collapse HTML formatting whitespace while keeping one boundary space.
426///
427/// Inline nodes are parsed independently, so dropping the whitespace around a
428/// `<b>` or `<i>` element would accidentally concatenate adjacent words.
429fn normalize_inline_text(value: &str) -> String {
430    let mut normalized = String::new();
431    let mut pending_space = false;
432    for character in value.chars() {
433        if character.is_whitespace() {
434            pending_space = true;
435        } else {
436            if pending_space {
437                normalized.push(' ');
438                pending_space = false;
439            }
440            normalized.push(character);
441        }
442    }
443    if pending_space {
444        normalized.push(' ');
445    }
446    normalized
447}
448
449fn normalize_pre_text(value: &str) -> String {
450    value.replace("\r\n", "\n").replace('\r', "\n")
451}
452
453/// Groff normally formats `<pre>` on separate HTML source lines. Those source
454/// delimiters are not part of the rendered manual and must not create blank
455/// terminal rows. Interior newlines remain untouched.
456fn trim_pre_boundaries(children: &mut Vec<Inline>) {
457    if let Some(value) = first_text_mut(children) {
458        *value = value.strip_prefix('\n').unwrap_or(value).to_owned();
459    }
460    if let Some(value) = last_text_mut(children) {
461        *value = value.strip_suffix('\n').unwrap_or(value).to_owned();
462    }
463    prune_empty_inline(children);
464}
465
466fn first_text_mut(children: &mut [Inline]) -> Option<&mut String> {
467    for child in children {
468        match child {
469            Inline::Text { value } => return Some(value),
470            Inline::Strong { children }
471            | Inline::Emphasis { children }
472            | Inline::ExternalLink { children, .. }
473            | Inline::EmailLink { children, .. }
474            | Inline::ManualReference { children, .. }
475            | Inline::SectionReference { children, .. } => {
476                if let Some(value) = first_text_mut(children) {
477                    return Some(value);
478                }
479            }
480            Inline::Code { .. } | Inline::Anchor { .. } | Inline::LineBreak => {}
481        }
482    }
483    None
484}
485
486fn last_text_mut(children: &mut [Inline]) -> Option<&mut String> {
487    for child in children.iter_mut().rev() {
488        match child {
489            Inline::Text { value } => return Some(value),
490            Inline::Strong { children }
491            | Inline::Emphasis { children }
492            | Inline::ExternalLink { children, .. }
493            | Inline::EmailLink { children, .. }
494            | Inline::ManualReference { children, .. }
495            | Inline::SectionReference { children, .. } => {
496                if let Some(value) = last_text_mut(children) {
497                    return Some(value);
498                }
499            }
500            Inline::Code { .. } | Inline::Anchor { .. } | Inline::LineBreak => {}
501        }
502    }
503    None
504}
505
506fn prune_empty_inline(children: &mut Vec<Inline>) {
507    for child in children.iter_mut() {
508        match child {
509            Inline::Strong { children }
510            | Inline::Emphasis { children }
511            | Inline::ExternalLink { children, .. }
512            | Inline::EmailLink { children, .. }
513            | Inline::ManualReference { children, .. }
514            | Inline::SectionReference { children, .. } => prune_empty_inline(children),
515            Inline::Text { .. }
516            | Inline::Code { .. }
517            | Inline::Anchor { .. }
518            | Inline::LineBreak => {}
519        }
520    }
521    children.retain(|child| match child {
522        Inline::Text { value } | Inline::Code { value } => !value.is_empty(),
523        Inline::Strong { children }
524        | Inline::Emphasis { children }
525        | Inline::ExternalLink { children, .. }
526        | Inline::EmailLink { children, .. }
527        | Inline::ManualReference { children, .. }
528        | Inline::SectionReference { children, .. } => !children.is_empty(),
529        Inline::Anchor { .. } | Inline::LineBreak => true,
530    });
531}
532
533fn inline_text(children: &[Inline]) -> String {
534    let mut value = String::new();
535    for child in children {
536        match child {
537            Inline::Text { value: text } | Inline::Code { value: text } => value.push_str(text),
538            Inline::Strong { children }
539            | Inline::Emphasis { children }
540            | Inline::ExternalLink { children, .. }
541            | Inline::EmailLink { children, .. }
542            | Inline::ManualReference { children, .. }
543            | Inline::SectionReference { children, .. } => value.push_str(&inline_text(children)),
544            Inline::Anchor { .. } => {}
545            Inline::LineBreak => value.push('\n'),
546        }
547    }
548    value
549}
550
551#[cfg(test)]
552mod tests {
553    use mant_ast::{Block, Inline, LayoutHint, SourceFormat};
554
555    use super::{inline_text, parse_groff_html};
556
557    #[test]
558    fn parses_sections_indentation_and_inline_formatting() {
559        let document = parse_groff_html(
560            r##"<body>
561              <h1>TEST(1)</h1><a href="#NAME">NAME</a><br><hr>
562              <h2>NAME<a name="NAME"></a></h2>
563              <p style="margin-left:9%">See <b>mant</b> and <i>friends</i>.</p>
564              <h2>OPTIONS</h2><h3>Output</h3><p>details</p>
565            </body>"##,
566            None,
567        );
568
569        assert_eq!(document.source.format, SourceFormat::GroffHtml);
570        assert_eq!(document.sections.len(), 2);
571        assert_eq!(document.sections[0].title, "NAME");
572        assert_eq!(document.sections[1].children[0].title, "Output");
573        let Block::Paragraph {
574            children, layout, ..
575        } = &document.sections[0].blocks[0]
576        else {
577            panic!("expected paragraph");
578        };
579        assert_eq!(layout.indent_columns, 7);
580        assert!(matches!(children[1], Inline::Strong { .. }));
581        assert!(matches!(children[3], Inline::Emphasis { .. }));
582        assert_eq!(inline_text(children), "See mant and friends.");
583    }
584
585    #[test]
586    fn flattens_groff_layout_tables_with_cumulative_indentation() {
587        let document = parse_groff_html(
588            r#"<body><h2>OPTIONS</h2><table><tr>
589              <td width="9%"></td><td width="3%"><p><b>-c</b></p></td>
590              <td width="6%"></td><td width="82%"><p>sort by ctime</p></td>
591            </tr></table></body>"#,
592            None,
593        );
594        let blocks = &document.sections[0].blocks;
595        let [Block::DefinitionList { items, layout, .. }] = blocks.as_slice() else {
596            panic!("hanging groff table should become a semantic definition");
597        };
598        assert_eq!(layout.indent_columns, 7);
599        assert!(
600            items[0]
601                .identity
602                .as_ref()
603                .is_some_and(|identity| { identity.names == ["-c"] })
604        );
605        assert!(matches!(
606            items[0].description.as_slice(),
607            [Block::Paragraph {
608                layout: mant_ast::LayoutHint {
609                    indent_columns: 3,
610                    ..
611                },
612                ..
613            }]
614        ));
615    }
616
617    #[test]
618    fn removes_only_html_source_boundaries_from_preformatted_content() {
619        let document = parse_groff_html(
620            "<body><h2>EXAMPLES</h2><pre>\n<b>git</b> one\ngit two\n</pre></body>",
621            None,
622        );
623        let Block::Preformatted { children, .. } = &document.sections[0].blocks[0] else {
624            panic!("expected preformatted block");
625        };
626        assert_eq!(inline_text(children), "git one\ngit two");
627        assert!(matches!(children[0], Inline::Strong { .. }));
628    }
629
630    #[test]
631    fn normalizes_inline_whitespace_and_preserves_breaks_through_transparent_tags() {
632        let document = parse_groff_html(
633            "<body><h2>TEXT</h2><p>alpha\n<font color=red><b>beta</b></font>\t<i>gamma</i><br>delta</p></body>",
634            None,
635        );
636        let Block::Paragraph { children, .. } = &document.sections[0].blocks[0] else {
637            panic!("expected paragraph");
638        };
639
640        assert_eq!(inline_text(children), "alpha beta gamma\ndelta");
641        assert!(matches!(children[1], Inline::Strong { .. }));
642        assert!(matches!(children[3], Inline::Emphasis { .. }));
643        assert!(matches!(children[4], Inline::LineBreak));
644    }
645
646    #[test]
647    fn parses_native_html_lists_and_definition_lists() {
648        let document = parse_groff_html(
649            r"<body><h2>OPTIONS</h2>
650              <ul><li>first</li><li><p>second</p></li></ul>
651              <dl><dt><b>-a</b></dt><dt><b>--all</b></dt>
652                  <dd><p>Show all entries.</p><pre>ls -a</pre></dd></dl>
653            </body>",
654            None,
655        );
656
657        assert!(matches!(
658            document.sections[0].blocks[0],
659            Block::List { ref items, .. } if items.len() == 2
660        ));
661        let Block::DefinitionList { items, .. } = &document.sections[0].blocks[1] else {
662            panic!("expected definition list");
663        };
664        assert_eq!(items[0].terms.len(), 2);
665        assert_eq!(items[0].description.len(), 2);
666        assert!(matches!(
667            items[0].description[1],
668            Block::Preformatted { .. }
669        ));
670    }
671
672    #[test]
673    fn ignores_empty_table_rows_and_restarts_widths_for_each_row() {
674        let document = parse_groff_html(
675            r#"<body><h2>TABLE</h2><table>
676              <tr><td width="100%"></td></tr>
677              <tr><td width="15%"></td><td width="85%"><p>twelve</p></td></tr>
678              <tr><td width="25%"></td><td width="75%"><p>twenty</p></td></tr>
679            </table></body>"#,
680            None,
681        );
682
683        assert!(matches!(
684            document.sections[0].blocks.as_slice(),
685            [
686                Block::Paragraph {
687                    layout: LayoutHint {
688                        indent_columns: 12,
689                        ..
690                    },
691                    ..
692                },
693                Block::Paragraph {
694                    layout: LayoutHint {
695                        indent_columns: 20,
696                        ..
697                    },
698                    ..
699                }
700            ]
701        ));
702    }
703
704    #[test]
705    fn excludes_groff_document_chrome_from_sections() {
706        let document = parse_groff_html(
707            r##"<body>
708              <h1>LS(1)</h1><a href="#NAME">NAME</a><br><hr>
709              generated renderer text
710              <h2>NAME<a name="NAME"></a></h2><p>ls - list files</p>
711              <h2>DESCRIPTION</h2><p>List directory contents.</p>
712            </body>"##,
713            Some("ls.html".to_owned()),
714        );
715
716        assert_eq!(
717            document
718                .sections
719                .iter()
720                .map(|section| section.title.as_str())
721                .collect::<Vec<_>>(),
722            ["NAME", "DESCRIPTION"]
723        );
724        assert_eq!(document.source.path.as_deref(), Some("ls.html"));
725        assert_eq!(document.sections[0].blocks.len(), 1);
726    }
727
728    #[test]
729    fn preserves_indentation_across_repeated_layout_table_rows() {
730        let document = parse_groff_html(
731            r#"<body><h2>DESCRIPTION</h2><table>
732              <tr><td width="9%"></td><td width="91%"><p>first</p></td></tr>
733              <tr><td width="9%"></td><td width="91%"><p>second</p></td></tr>
734              <tr><td width="9%"></td><td width="91%"><p>third</p></td></tr>
735            </table></body>"#,
736            None,
737        );
738        let indented = document.sections[0]
739            .blocks
740            .iter()
741            .filter(|block| {
742                matches!(
743                    block,
744                    Block::Paragraph {
745                        layout: mant_ast::LayoutHint {
746                            indent_columns: 7,
747                            ..
748                        },
749                        ..
750                    }
751                )
752            })
753            .count();
754        assert_eq!(indented, 3);
755    }
756}
mant_core/groff_html.rs

mant_core/
groff_html.rs
