webfetch/convert/
markdown.rs

1//! Markdown conversion. Unlike the text path, markdown keeps links inline as
2//! `[text](url)` for maximum fidelity — the right trade-off when the consumer
3//! wants a faithful, re-renderable document rather than minimal tokens.
4//!
5//! Links are still collected into a reference list, so a `--json` caller gets
6//! the same recoverable URLs the text path gives it; and the same schemes the
7//! text path refuses (`javascript:`, `mailto:`, in-page anchors) are left as
8//! plain text rather than emitted as live markdown links.
9
10use ego_tree::NodeRef;
11use scraper::node::Node;
12use scraper::Html;
13
14use super::text::RefCollector;
15use crate::extract;
16use crate::types::UrlReference;
17
18fn walk(node: NodeRef<Node>, out: &mut String, refs: &mut RefCollector) {
19    match node.value() {
20        Node::Text(t) => out.push_str(&t[..]),
21        Node::Element(el) => {
22            let name = el.name();
23            if super::is_skippable(name) {
24                return;
25            }
26
27            let prefix = match name {
28                "h1" => Some("\n# "),
29                "h2" => Some("\n## "),
30                "h3" => Some("\n### "),
31                "h4" => Some("\n#### "),
32                "h5" => Some("\n##### "),
33                "h6" => Some("\n###### "),
34                "li" => Some("\n- "),
35                "blockquote" => Some("\n> "),
36                _ => None,
37            };
38
39            if name == "br" {
40                out.push('\n');
41                return;
42            }
43
44            if name == "a" {
45                let mut inner = String::new();
46                for child in node.children() {
47                    walk(child, &mut inner, refs);
48                }
49                let inner = inner.trim().to_string();
50                match el.attr("href").and_then(|href| refs.resolve(href)) {
51                    Some(url) => {
52                        refs.index_for(url.clone(), &inner);
53                        out.push_str(&format!("[{inner}]({url})"));
54                    }
55                    None => out.push_str(&inner),
56                }
57                return;
58            }
59
60            if name == "code" {
61                let mut inner = String::new();
62                for child in node.children() {
63                    walk(child, &mut inner, refs);
64                }
65                out.push_str(&format!("`{}`", inner.trim()));
66                return;
67            }
68
69            if let Some(p) = prefix {
70                out.push_str(p);
71            }
72            for child in node.children() {
73                walk(child, out, refs);
74            }
75            if matches!(
76                name,
77                "p" | "div" | "section" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6"
78            ) {
79                out.push('\n');
80            }
81        }
82        _ => {}
83    }
84}
85
86/// Convert a parsed document to markdown, also returning the links it contains.
87pub fn markdown_with_refs(doc: &Html, base_url: &str) -> (String, Vec<UrlReference>) {
88    let root = match extract::content_root(doc) {
89        Some(el) => el,
90        None => return (String::new(), Vec::new()),
91    };
92    let mut refs = RefCollector::new(base_url);
93    let mut out = String::new();
94    for child in root.children() {
95        walk(child, &mut out, &mut refs);
96    }
97    (out, refs.references)
98}
99
100/// [`markdown_with_refs`] for callers holding raw HTML.
101pub fn html_to_markdown(html: &str, base_url: &str) -> String {
102    markdown_with_refs(&Html::parse_document(html), base_url).0
103}
104
105#[cfg(test)]
106mod tests {
107    use super::*;
108
109    #[test]
110    fn unfetchable_schemes_stay_plain_text() {
111        let html = r#"<article><p><a href="javascript:alert(1)">click</a>
112            and <a href="/ok">ok</a></p></article>"#;
113        let (md, refs) = markdown_with_refs(&Html::parse_document(html), "https://x.test/");
114        assert!(!md.contains("javascript:"), "md: {md}");
115        assert!(!md.contains("[click]("), "js link must not stay live: {md}");
116        assert!(md.contains("click"), "anchor text is still kept: {md}");
117        assert!(md.contains("[ok](https://x.test/ok)"), "md: {md}");
118        assert_eq!(refs.len(), 1);
119    }
120}
webfetch/convert/markdown.rs

webfetch/convert/
markdown.rs
