mumu-html 0.1.1

HTML manipulation and tools plugin for the Lava language
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

use mumu::parser::types::Value;
use regex::Regex;

pub fn extract_text_bridge(_interp: &mut mumu::parser::interpreter::Interpreter, mut args: Vec<Value>) -> Result<Value, String> {
    if args.len() != 1 {
        return Err("html:extract_text(html) => expected 1 argument".to_string());
    }
    let html_val = args.remove(0);
    let html = match html_val {
        Value::SingleString(s) => s,
        Value::StrArray(ss) if ss.len() == 1 => ss[0].clone(),
        _ => return Err("html:extract_text => argument must be a string".to_string()),
    };
    Ok(Value::SingleString(extract_visible_text(&html)))
}

/// Extracts visible text from HTML by removing <script>, <style>, <noscript> and all tags.
pub fn extract_visible_text(html: &str) -> String {
    // Remove <script>, <style>, <noscript> blocks and their content (Rust regex doesn't support backreferences)
    let script_re = Regex::new(r"(?is)<script[^>]*>.*?</script>").unwrap();
    let style_re = Regex::new(r"(?is)<style[^>]*>.*?</style>").unwrap();
    let noscript_re = Regex::new(r"(?is)<noscript[^>]*>.*?</noscript>").unwrap();
    let html = script_re.replace_all(html, "");
    let html = style_re.replace_all(&html, "");
    let html = noscript_re.replace_all(&html, "");
    // Remove all tags
    let tag_re = Regex::new(r"(?is)<[^>]+>").unwrap();
    let text = tag_re.replace_all(&html, " ");
    // Collapse whitespace
    let ws_re = Regex::new(r"[ \t\r\n]+").unwrap();
    ws_re.replace_all(&text, " ").trim().to_string()
}