dom-content-extraction 0.4.4

Rust implementation of Content extraction via text density paper
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

use dom_content_extraction::{DensityTree, get_node_text};
use scraper::Html;
use std::fs;

fn main() {
    let html_content =
        fs::read_to_string("html/test_1.html").expect("Unable to read file");
    let document = Html::parse_document(&html_content);
    let mut dtree = DensityTree::from_document(&document).unwrap();
    dtree.calculate_density_sum().unwrap();

    println!("Density analysis for test_1.html:");
    println!("================================");

    // Get nodes sorted by density sum
    let sorted_nodes = dtree.sorted_nodes();

    for (i, node) in sorted_nodes.iter().enumerate() {
        if let Ok(text) = get_node_text(node.node_id, &document)
            && !text.trim().is_empty()
        {
            println!(
                "\nNode {} (density_sum: {:.2}):",
                i,
                node.density_sum.unwrap_or(0.0)
            );
            println!("Text: '{}'", text.trim());
        }
    }

    // Show the max density node
    if let Some(max_node) = dtree.get_max_density_sum_node() {
        println!("\n=== MAX DENSITY NODE ===");
        println!(
            "Density sum: {:.2}",
            max_node.value().density_sum.unwrap_or(0.0)
        );
        if let Ok(text) = get_node_text(max_node.value().node_id, &document) {
            println!("Content: '{}'", text.trim());
        }
    }
}