use lazy_static::lazy_static;
use std::boxed::Box;
use std::borrow::Borrow;
use std::collections::HashMap;
use std::os::raw::{c_char};
use std::ffi::{CString, CStr};
use regex::Regex;
use html5ever::parse_document;
use html5ever::driver::ParseOpts;
use html5ever::tendril::TendrilSink;
pub use markup5ever_rcdom::{RcDom, Handle, NodeData};
pub mod common;
pub mod dummy;
pub mod anchors;
pub mod paragraphs;
pub mod images;
pub mod headers;
pub mod lists;
pub mod styles;
pub mod codes;
pub mod quotes;
pub mod tables;
pub mod containers;
pub mod iframes;
use crate::dummy::DummyHandler;
use crate::dummy::IdentityHandler;
use crate::dummy::HtmlCherryPickHandler;
use crate::paragraphs::ParagraphHandler;
use crate::anchors::AnchorHandler;
use crate::images::ImgHandler;
use crate::headers::HeaderHandler;
use crate::lists::ListItemHandler;
use crate::lists::ListHandler;
use crate::styles::StyleHandler;
use crate::codes::CodeHandler;
use crate::quotes::QuoteHandler;
use crate::tables::TableHandler;
use crate::containers::ContainerHandler;
use crate::iframes::IframeHandler;
lazy_static! {
static ref EXCESSIVE_WHITESPACE_PATTERN: Regex = Regex::new("\\s{2,}").unwrap();
static ref EMPTY_LINE_PATTERN: Regex = Regex::new("(?m)^ +$").unwrap(); static ref EXCESSIVE_NEWLINE_PATTERN: Regex = Regex::new("\\n{3,}").unwrap(); static ref TRAILING_SPACE_PATTERN: Regex = Regex::new("(?m)(\\S) $").unwrap(); static ref LEADING_NEWLINES_PATTERN: Regex = Regex::new("^\\n+").unwrap(); static ref LAST_WHITESPACE_PATTERN: Regex = Regex::new("\\s+$").unwrap();
static ref START_OF_LINE_PATTERN: Regex = Regex::new("(^|\\n) *$").unwrap(); static ref MARKDOWN_STARTONLY_KEYCHARS: Regex = Regex::new(r"^(\s*)([=>+\-#])").unwrap(); static ref MARKDOWN_MIDDLE_KEYCHARS: Regex = Regex::new(r"[<>*\\_~]").unwrap(); }
pub fn parse_html_custom(html: &str, custom: &HashMap<String, Box<dyn TagHandlerFactory>>) -> String {
let dom = parse_document(RcDom::default(), ParseOpts::default()).from_utf8().read_from(&mut html.as_bytes()).unwrap();
let mut result = StructuredPrinter::default();
walk(&dom.document, &mut result, custom);
return clean_markdown(&result.data);
}
pub fn parse_html(html: &str) -> String {
parse_html_custom(html, &HashMap::default())
}
pub fn parse_html_extended(html: &str) -> String {
struct SpanAsIsTagFactory;
impl TagHandlerFactory for SpanAsIsTagFactory {
fn instantiate(&self) -> Box<dyn TagHandler> {
return Box::new(HtmlCherryPickHandler::default());
}
}
let mut tag_factory: HashMap<String, Box<dyn TagHandlerFactory>> = HashMap::new();
tag_factory.insert(String::from("span"), Box::new(SpanAsIsTagFactory{}));
return parse_html_custom(html, &tag_factory);
}
fn walk(input: &Handle, result: &mut StructuredPrinter, custom: &HashMap<String, Box<dyn TagHandlerFactory>>) {
let mut handler : Box<dyn TagHandler> = Box::new(DummyHandler::default());
let mut tag_name = String::default();
match input.data {
NodeData::Document | NodeData::Doctype {..} | NodeData::ProcessingInstruction {..} => {},
NodeData::Text { ref contents } => {
let mut text = contents.borrow().to_string();
let inside_pre = result.parent_chain.iter().any(|tag| tag == "pre");
if inside_pre {
result.append_str(&text);
} else if !(text.trim().len() == 0 && (result.data.chars().last() == Some('\n') || result.data.chars().last() == Some(' '))) {
let inside_code = result.parent_chain.iter().any(|tag| tag == "code");
if !inside_code {
text = escape_markdown(result, &text);
}
let minified_text = EXCESSIVE_WHITESPACE_PATTERN.replace_all(&text, " ");
let minified_text = minified_text.trim_matches(|ch: char| ch == '\n' || ch == '\r');
result.append_str(&minified_text);
}
}
NodeData::Comment { .. } => {}, NodeData::Element { ref name, .. } => {
tag_name = name.local.to_string();
let inside_pre = result.parent_chain.iter().any(|tag| tag == "pre");
if inside_pre {
handler = Box::new(DummyHandler::default());
}else if custom.contains_key(&tag_name) {
let factory = custom.get(&tag_name).unwrap();
handler = factory.instantiate();
} else {
handler = match tag_name.as_ref() {
"div" | "section" | "header" | "footer" => Box::new(ContainerHandler::default()),
"p" | "br" | "hr" => Box::new(ParagraphHandler::default()),
"q" | "cite" | "blockquote" => Box::new(QuoteHandler::default()),
"details" | "summary" => Box::new(HtmlCherryPickHandler::default()),
"b" | "i" | "s" | "strong" | "em" | "del" => Box::new(StyleHandler::default()),
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => Box::new(HeaderHandler::default()),
"pre" | "code" => Box::new(CodeHandler::default()),
"img" => Box::new(ImgHandler::default()),
"a" => Box::new(AnchorHandler::default()),
"ol" | "ul" | "menu" => Box::new(ListHandler::default()),
"li" => Box::new(ListItemHandler::default()),
"sub" | "sup" => Box::new(IdentityHandler::default()),
"table" => Box::new(TableHandler::default()),
"iframe" => Box::new(IframeHandler::default()),
"html" | "head" | "body" => Box::new(DummyHandler::default()),
_ => Box::new(DummyHandler::default())
};
}
}
}
handler.handle(&input, result);
result.parent_chain.push(tag_name.to_string()); let current_depth = result.parent_chain.len();
result.siblings.insert(current_depth, vec![]);
for child in input.children.borrow().iter() {
if handler.skip_descendants() {
continue;
}
walk(child.borrow(), result, custom);
match child.data {
NodeData::Element { ref name, .. } => result.siblings.get_mut(¤t_depth).unwrap().push(name.local.to_string()),
_ => {}
};
}
result.siblings.remove(¤t_depth);
result.parent_chain.pop();
handler.after_handle(result);
}
fn escape_markdown(result: &StructuredPrinter, text: &str) -> String {
let mut data = MARKDOWN_MIDDLE_KEYCHARS.replace_all(&text, "\\$0").to_string();
if START_OF_LINE_PATTERN.is_match(&result.data) {
data = MARKDOWN_STARTONLY_KEYCHARS.replace(&data, "$1\\$2").to_string();
}
return data;
}
fn clean_markdown(text: &str) -> String {
let intermediate = EMPTY_LINE_PATTERN.replace_all(&text, ""); let intermediate = EXCESSIVE_NEWLINE_PATTERN.replace_all(&intermediate, "\n\n"); let intermediate = TRAILING_SPACE_PATTERN.replace_all(&intermediate, "$1"); let intermediate = LEADING_NEWLINES_PATTERN.replace_all(&intermediate, ""); let intermediate = LAST_WHITESPACE_PATTERN.replace_all(&intermediate, "");
return intermediate.into_owned();
}
#[derive(Debug, Default)]
pub struct StructuredPrinter {
pub parent_chain: Vec<String>,
pub siblings: HashMap<usize, Vec<String>>,
pub data: String,
}
impl StructuredPrinter {
pub fn insert_newline(&mut self) {
self.append_str("\n");
}
pub fn append_str(&mut self, it: &str) {
self.data.push_str(it);
}
pub fn insert_str(&mut self, pos: usize, it: &str) {
self.data.insert_str(pos, it);
}
}
pub trait TagHandlerFactory {
fn instantiate(&self) -> Box<dyn TagHandler>;
}
pub trait TagHandler {
fn handle(&mut self, tag: &Handle, printer: &mut StructuredPrinter);
fn after_handle(&mut self, printer: &mut StructuredPrinter);
fn skip_descendants(&self) -> bool {
return false;
}
}
#[no_mangle]
pub extern fn parse(html: *const c_char) -> *const c_char {
let in_html = unsafe { CStr::from_ptr(html) };
let out_md = parse_html(&in_html.to_string_lossy());
CString::new(out_md).unwrap().into_raw()
}
#[cfg(target_os="android")]
#[allow(non_snake_case)]
pub mod android {
extern crate jni;
use super::parse_html;
use super::parse_html_extended;
use self::jni::JNIEnv;
use self::jni::objects::{JClass, JString};
use self::jni::sys::jstring;
#[no_mangle]
pub unsafe extern fn Java_com_kanedias_html2md_Html2Markdown_parse(env: JNIEnv, _clazz: JClass, html: JString) -> jstring {
let html_java : String = env.get_string(html).expect("Couldn't get java string!").into();
let markdown = parse_html(&html_java);
let output = env.new_string(markdown).expect("Couldn't create java string!");
output.into_inner()
}
#[no_mangle]
pub unsafe extern fn Java_com_kanedias_html2md_Html2Markdown_parseExtended(env: JNIEnv, _clazz: JClass, html: JString) -> jstring {
let html_java : String = env.get_string(html).expect("Couldn't get java string!").into();
let markdown = parse_html_extended(&html_java);
let output = env.new_string(markdown).expect("Couldn't create java string!");
output.into_inner()
}
}