html2md/
lib.rs

1use std::boxed::Box;
2use std::collections::HashMap;
3use std::ffi::CStr;
4use std::ffi::CString;
5use std::os::raw::c_char;
6use std::sync::LazyLock;
7
8use regex::Regex;
9
10use html5ever::driver::ParseOpts;
11use html5ever::parse_document;
12use html5ever::tendril::TendrilSink;
13
14pub use markup5ever_rcdom::{Handle, NodeData, RcDom};
15
16pub mod anchors;
17pub mod codes;
18pub mod common;
19pub mod containers;
20pub mod dummy;
21pub mod headers;
22pub mod iframes;
23pub mod images;
24pub mod lists;
25pub mod markup5ever_rcdom;
26pub mod paragraphs;
27pub mod quotes;
28pub mod styles;
29pub mod tables;
30
31use crate::anchors::AnchorHandler;
32use crate::codes::CodeHandler;
33use crate::containers::ContainerHandler;
34use crate::dummy::DummyHandler;
35use crate::dummy::HtmlCherryPickHandler;
36use crate::dummy::IdentityHandler;
37use crate::headers::HeaderHandler;
38use crate::iframes::IframeHandler;
39use crate::images::ImgHandler;
40use crate::lists::ListHandler;
41use crate::lists::ListItemHandler;
42use crate::paragraphs::ParagraphHandler;
43use crate::quotes::QuoteHandler;
44use crate::styles::StyleHandler;
45use crate::tables::TableHandler;
46
47static EXCESSIVE_WHITESPACE_PATTERN: LazyLock<Regex> =
48    LazyLock::new(|| Regex::new("\\s{2,}").unwrap()); // for HTML on-the-fly cleanup
49static EMPTY_LINE_PATTERN: LazyLock<Regex> = LazyLock::new(|| Regex::new("(?m)^ +$").unwrap()); // for Markdown post-processing
50static EXCESSIVE_NEWLINE_PATTERN: LazyLock<Regex> =
51    LazyLock::new(|| Regex::new("\\n{3,}").unwrap()); // for Markdown post-processing
52static TRAILING_SPACE_PATTERN: LazyLock<Regex> =
53    LazyLock::new(|| Regex::new("(?m)(\\S) $").unwrap()); // for Markdown post-processing
54static LEADING_NEWLINES_PATTERN: LazyLock<Regex> = LazyLock::new(|| Regex::new("^\\n+").unwrap()); // for Markdown post-processing
55static LAST_WHITESPACE_PATTERN: LazyLock<Regex> = LazyLock::new(|| Regex::new("\\s+$").unwrap()); // for Markdown post-processing
56static START_OF_LINE_PATTERN: LazyLock<Regex> = LazyLock::new(|| Regex::new("(^|\\n) *$").unwrap()); // for Markdown escaping
57static MARKDOWN_STARTONLY_KEYCHARS: LazyLock<Regex> =
58    LazyLock::new(|| Regex::new(r"^(\s*)([=>+\-#])").unwrap()); // for Markdown escaping
59static MARKDOWN_MIDDLE_KEYCHARS: LazyLock<Regex> =
60    LazyLock::new(|| Regex::new(r"[<>*\\_~]").unwrap()); // for Markdown escaping
61
62/// Custom variant of main function. Allows to pass custom tag<->tag factory pairs
63/// in order to register custom tag handler for tags you want.
64///
65/// You can also override standard tag handlers this way
66/// # Arguments
67/// `html` is source HTML as `String`
68/// `custom` is custom tag handler producers for tags you want, can be empty
69pub fn parse_html_custom(
70    html: &str,
71    custom: &HashMap<String, Box<dyn TagHandlerFactory>>,
72) -> String {
73    let dom = parse_document(RcDom::default(), ParseOpts::default())
74        .from_utf8()
75        .read_from(&mut html.as_bytes())
76        .unwrap();
77    let mut result = StructuredPrinter::default();
78    walk(&dom.document, &mut result, custom);
79
80    clean_markdown(&result.data)
81}
82
83/// Main function of this library. Parses incoming HTML, converts it into Markdown
84/// and returns converted string.
85/// # Arguments
86/// `html` is source HTML as `String`
87pub fn parse_html(html: &str) -> String {
88    parse_html_custom(html, &HashMap::default())
89}
90
91/// Same as `parse_html` but retains all "span" html elements intact
92/// Markdown parsers usually strip them down when rendering but they
93/// may be useful for later processing
94pub fn parse_html_extended(html: &str) -> String {
95    let mut tag_factory: HashMap<String, Box<dyn TagHandlerFactory>> = HashMap::new();
96
97    tag_factory.insert(
98        String::from("span"),
99        Box::new(HtmlCherryPickHandler::default),
100    );
101    parse_html_custom(html, &tag_factory)
102}
103
104/// Recursively walk through all DOM tree and handle all elements according to
105/// HTML tag -> Markdown syntax mapping. Text content is trimmed to one whitespace according to HTML5 rules.
106///
107/// # Arguments
108/// `input` is DOM tree or its subtree
109/// `result` is output holder with position and context tracking
110/// `custom` is custom tag hadler producers for tags you want, can be empty
111pub fn walk(
112    input: &Handle,
113    result: &mut StructuredPrinter,
114    custom: &HashMap<String, Box<dyn TagHandlerFactory>>,
115) {
116    let mut handler: Box<dyn TagHandler> = Box::new(DummyHandler);
117    let mut tag_name = String::default();
118    match input.data {
119        NodeData::Document | NodeData::Doctype { .. } | NodeData::ProcessingInstruction { .. } => {}
120        NodeData::Text { ref contents } => {
121            let mut text = contents.borrow().to_string();
122            let inside_pre = result.parent_chain.iter().any(|tag| tag == "pre");
123            if inside_pre {
124                // this is preformatted text, insert as it is
125                result.append_str(&text);
126            } else if !(text.trim().is_empty()
127                && (result.data.ends_with('\n') || result.data.ends_with(' ')))
128            {
129                // in case it's not just a whitespace after the newline or another whitespace
130
131                // regular text, collapse whitespace and newlines in text
132                let inside_code = result.parent_chain.iter().any(|tag| tag == "code");
133                if !inside_code {
134                    text = escape_markdown(result, &text);
135                }
136                let minified_text = EXCESSIVE_WHITESPACE_PATTERN.replace_all(&text, " ");
137                let minified_text = minified_text.trim_matches(|ch: char| ch == '\n' || ch == '\r');
138                result.append_str(minified_text);
139            }
140        }
141        NodeData::Comment { .. } => {} // ignore comments
142        NodeData::Element { ref name, .. } => {
143            tag_name = name.local.to_string();
144            let inside_pre = result.parent_chain.iter().any(|tag| tag == "pre");
145            if inside_pre {
146                // don't add any html tags inside the pre section
147                handler = Box::new(DummyHandler);
148            } else if custom.contains_key(&tag_name) {
149                // have user-supplied factory, instantiate a handler for this tag
150                let factory = custom.get(&tag_name).unwrap();
151                handler = factory.instantiate();
152            } else {
153                // no user-supplied factory, take one of built-in ones
154                handler = match tag_name.as_ref() {
155                    // containers
156                    "div" | "section" | "header" | "footer" => Box::new(ContainerHandler),
157                    // pagination, breaks
158                    "p" | "br" | "hr" => Box::new(ParagraphHandler::default()),
159                    "q" | "cite" | "blockquote" => Box::new(QuoteHandler::default()),
160                    // spoiler tag
161                    "details" | "summary" => Box::new(HtmlCherryPickHandler::default()),
162                    // formatting
163                    "b" | "i" | "s" | "strong" | "em" | "del" => Box::new(StyleHandler::default()),
164                    "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => Box::new(HeaderHandler::default()),
165                    "pre" | "code" => Box::new(CodeHandler::default()),
166                    // images, links
167                    "img" => Box::new(ImgHandler::default()),
168                    "a" => Box::new(AnchorHandler::default()),
169                    // lists
170                    "ol" | "ul" | "menu" => Box::new(ListHandler),
171                    "li" => Box::new(ListItemHandler::default()),
172                    // as-is
173                    "sub" | "sup" => Box::new(IdentityHandler),
174                    // tables, handled fully internally as markdown can't have nested content in tables
175                    // supports only single tables as of now
176                    "table" => Box::new(TableHandler),
177                    "iframe" => Box::new(IframeHandler),
178                    // other
179                    "html" | "head" | "body" => Box::new(DummyHandler),
180                    _ => Box::new(DummyHandler),
181                };
182            }
183        }
184    }
185
186    // handle this tag, while it's not in parent chain
187    // and doesn't have child siblings
188    handler.handle(input, result);
189
190    // save this tag name as parent for child nodes
191    result.parent_chain.push(tag_name.to_string()); // e.g. it was ["body"] and now it's ["body", "p"]
192    let current_depth = result.parent_chain.len(); // e.g. it was 1 and now it's 2
193
194    // create space for siblings of next level
195    result.siblings.insert(current_depth, vec![]);
196
197    for child in input.children.borrow().iter() {
198        if handler.skip_descendants() {
199            continue;
200        }
201
202        walk(child, result, custom);
203
204        if let NodeData::Element { ref name, .. } = child.data {
205            result
206                .siblings
207                .get_mut(&current_depth)
208                .unwrap()
209                .push(name.local.to_string())
210        };
211    }
212
213    // clear siblings of next level
214    result.siblings.remove(&current_depth);
215
216    // release parent tag
217    result.parent_chain.pop();
218
219    // finish handling of tag - parent chain now doesn't contain this tag itself again
220    handler.after_handle(result);
221}
222
223/// This conversion should only be applied to text tags
224///
225/// Escapes text inside HTML tags so it won't be recognized as Markdown control sequence
226/// like list start or bold text style
227fn escape_markdown(result: &StructuredPrinter, text: &str) -> String {
228    // always escape bold/italic/strikethrough
229    let mut data = MARKDOWN_MIDDLE_KEYCHARS
230        .replace_all(text, "\\$0")
231        .to_string();
232
233    // if we're at the start of the line we need to escape list- and quote-starting sequences
234    if START_OF_LINE_PATTERN.is_match(&result.data) {
235        data = MARKDOWN_STARTONLY_KEYCHARS
236            .replace(&data, "$1\\$2")
237            .to_string();
238    }
239
240    // no handling of more complicated cases such as
241    // ![] or []() ones, for now this will suffice
242    data
243}
244
245/// Called after all processing has been finished
246///
247/// Clears excessive punctuation that would be trimmed by renderer anyway
248fn clean_markdown(text: &str) -> String {
249    // remove redundant newlines
250    let intermediate = EMPTY_LINE_PATTERN.replace_all(text, ""); // empty line with trailing spaces, replace with just newline
251    let intermediate = EXCESSIVE_NEWLINE_PATTERN.replace_all(&intermediate, "\n\n"); // > 3 newlines - not handled by markdown anyway
252    let intermediate = TRAILING_SPACE_PATTERN.replace_all(&intermediate, "$1"); // trim space if it's just one
253    let intermediate = LEADING_NEWLINES_PATTERN.replace_all(&intermediate, ""); // trim leading newlines
254    let intermediate = LAST_WHITESPACE_PATTERN.replace_all(&intermediate, ""); // trim last newlines
255
256    intermediate.into_owned()
257}
258
259/// Intermediate result of HTML -> Markdown conversion.
260///
261/// Holds context in the form of parent tags and siblings chain
262/// and resulting string of markup content with the current position.
263#[derive(Debug, Default)]
264pub struct StructuredPrinter {
265    /// Chain of parents leading to upmost <html> tag
266    pub parent_chain: Vec<String>,
267
268    /// Siblings of currently processed tag in order where they're appearing in html
269    pub siblings: HashMap<usize, Vec<String>>,
270
271    /// resulting markdown document
272    pub data: String,
273}
274
275impl StructuredPrinter {
276    /// Inserts newline
277    pub fn insert_newline(&mut self) {
278        self.append_str("\n");
279    }
280
281    /// Append string to the end of the printer
282    pub fn append_str(&mut self, it: &str) {
283        self.data.push_str(it);
284    }
285
286    /// Insert string at specified position of printer, adjust position to the end of inserted string
287    pub fn insert_str(&mut self, pos: usize, it: &str) {
288        self.data.insert_str(pos, it);
289    }
290}
291
292/// [`TagHandler`] factory. This trait is required in providing proper
293/// custom tag parsing capabilities to users of this library.
294///
295/// The problem with directly providing tag handlers is that they're not stateless.
296/// Once tag handler is parsing some tag, it holds data, such as start position, indent etc.
297///
298/// Since there is a blanket impl on `Fn() -> TagHandler`, any function or closure returning `TagHandler` can
299/// be used in place of a dedicated `TagHandlerFactory` instance, including trait functions like
300/// [`Default::default`].
301///
302/// # Examples
303///
304/// ```
305/// use std::collections::HashMap;
306/// use html2md::{parse_html_custom, TagHandlerFactory, dummy::DummyHandler};
307///
308/// # let html = "";
309/// let mut tag_factory: HashMap<String, Box<dyn TagHandlerFactory>> = HashMap::new();
310/// tag_factory.insert(
311///     String::from("span"),
312///     Box::new(DummyHandler::default),
313/// );
314/// let markdown = parse_html_custom(html, &tag_factory);
315/// ```
316///
317pub trait TagHandlerFactory {
318    fn instantiate(&self) -> Box<dyn TagHandler>;
319}
320
321impl<F: Fn() -> T, T: TagHandler + 'static> TagHandlerFactory for F {
322    fn instantiate(&self) -> Box<dyn TagHandler> {
323        Box::new(self())
324    }
325}
326
327/// Trait interface describing abstract handler of arbitrary HTML tag.
328pub trait TagHandler {
329    /// Handle tag encountered when walking HTML tree.
330    /// This is executed before the children processing
331    fn handle(&mut self, tag: &Handle, printer: &mut StructuredPrinter);
332
333    /// Executed after all children of this tag have been processed
334    fn after_handle(&mut self, printer: &mut StructuredPrinter);
335
336    fn skip_descendants(&self) -> bool {
337        false
338    }
339}
340
341/// FFI variant for HTML -> Markdown conversion for calling from other languages
342#[unsafe(no_mangle)]
343#[allow(clippy::not_unsafe_ptr_arg_deref)]
344pub extern "C" fn parse(html: *const c_char) -> *const c_char {
345    let in_html = unsafe { CStr::from_ptr(html) };
346    let out_md = parse_html(&in_html.to_string_lossy());
347
348    CString::new(out_md).unwrap().into_raw()
349}
350
351/// Expose the JNI interface for android below
352#[cfg(target_os = "android")]
353#[allow(non_snake_case)]
354pub mod android {
355    extern crate jni;
356
357    use super::parse_html;
358    use super::parse_html_extended;
359
360    use self::jni::JNIEnv;
361    use self::jni::objects::{JClass, JString};
362    use self::jni::sys::jstring;
363
364    #[no_mangle]
365    pub unsafe extern "C" fn Java_com_kanedias_html2md_Html2Markdown_parse(
366        mut env: JNIEnv,
367        _clazz: JClass,
368        html: JString,
369    ) -> jstring {
370        let html_java: String = env
371            .get_string(html)
372            .expect("Couldn't get java string!")
373            .into();
374        let markdown = parse_html(&html_java);
375        let output = env
376            .new_string(markdown)
377            .expect("Couldn't create java string!");
378        output.into_raw()
379    }
380
381    #[no_mangle]
382    pub unsafe extern "C" fn Java_com_kanedias_html2md_Html2Markdown_parseExtended(
383        mut env: JNIEnv,
384        _clazz: JClass,
385        html: JString,
386    ) -> jstring {
387        let html_java: String = env
388            .get_string(html)
389            .expect("Couldn't get java string!")
390            .into();
391        let markdown = parse_html_extended(&html_java);
392        let output = env
393            .new_string(markdown)
394            .expect("Couldn't create java string!");
395        output.into_raw()
396    }
397}
html2md/lib.rs

html2md/
lib.rs
