dom-content-extraction 0.4.2

Rust implementation of Content extraction via text density paper
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

#[cfg(feature = "markdown")]
use dom_content_extraction::extract_content_as_markdown;
use dom_content_extraction::{
    DensityTree, get_content, get_node_text, scraper::Html,
};
use std::fs;

fn process_lorem_ipsum() {
    println!("Processing Lorem Ipsum example...");
    let html_content =
        fs::read_to_string("html/lorem_ipsum.html").expect("Unable to read file");
    let document = Html::parse_document(&html_content);
    let mut dtree = DensityTree::from_document(&document).unwrap();
    let _ = dtree.calculate_density_sum();
    let extracted_content = dtree.extract_content(&document).unwrap();
    println!("Extracted content:\n{}", extracted_content);
}

fn process_test_4_html() {
    println!("Processing test_4 example...");
    let html_content =
        fs::read_to_string("html/test_4.html").expect("Unable to read file");
    let document = Html::parse_document(&html_content);
    let dtree = DensityTree::from_document(&document).unwrap();

    let sorted_nodes = dtree.sorted_nodes();
    let densest_node = sorted_nodes.last().unwrap();

    println!(
        "Highest density node:\n{}",
        get_node_text(densest_node.node_id, &document).unwrap()
    );
}

fn process_toy() {
    let html = r#"
        <!DOCTYPE html><html><body>
            <nav>Navigation</nav>
            <article>
                <h1>Main Article</h1>
                <p>This is the primary content that should be extracted.</p>
            </article>
            <footer>Footer</footer>
        </body></html>
    "#;

    println!("Extracting content from toy html: \n{}\n ", html);
    let document = Html::parse_document(html);
    let content = get_content(&document).unwrap();
    println!("{}", content);
}

#[cfg(feature = "markdown")]
fn process_lorem_ipsum_markdown() {
    println!("Processing Lorem Ipsum example as Markdown...");
    let html_content =
        fs::read_to_string("html/lorem_ipsum.html").expect("Unable to read file");
    let document = Html::parse_document(&html_content);
    let mut dtree = DensityTree::from_document(&document).unwrap();
    dtree.calculate_density_sum().unwrap();

    let markdown_content = extract_content_as_markdown(&dtree, &document).unwrap();
    println!("Extracted markdown content:\n{}", markdown_content);
}

fn main() {
    let arg = std::env::args().nth(1).unwrap_or_default();

    match arg.as_str() {
        "lorem-ipsum" => process_lorem_ipsum(),
        "test4" => process_test_4_html(),
        "toy" => process_toy(),
        #[cfg(feature = "markdown")]
        "lorem-ipsum-markdown" => process_lorem_ipsum_markdown(),
        _ => {
            eprintln!("Usage: check <lorem-ipsum|test4|toy|lorem-ipsum-markdown>");
            std::process::exit(1);
        }
    }
}