dom-content-extraction 0.3.3

Rust implementation of Content extraction via text density paper
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

use clap::{Parser, Subcommand};
use dom_content_extraction::{scraper::Html, DensityTree};
use std::fs;

#[derive(Parser)]
#[command(author, version, about, long_about = None)]
struct Cli {
    #[command(subcommand)]
    command: Commands,
}

#[derive(Subcommand)]
enum Commands {
    LoremIpsum,
}

fn main() {
    let cli = Cli::parse();

    match &cli.command {
        Commands::LoremIpsum => {
            println!("Processing Lorem Ipsum example...");
            process_lorem_ipsum();
        }
    }
}

fn process_lorem_ipsum() {
    let html_content =
        fs::read_to_string("html/lorem_ipsum.html").expect("Unable to read file");
    let document = Html::parse_document(&html_content);
    let mut dtree = DensityTree::from_document(&document);
    dtree.calculate_density_sum();
    let extracted_content = dtree.extract_content(&document);
    println!("Extracted content:\n{}", extracted_content);
}