readah 0.1.3

Key component of Readability module extracted from https://gitlab.com/news-flash/article_scraper.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

use headless_chrome::{types::PrintToPdfOptions, Browser, LaunchOptions};
use html2text;
use http_req::request;
use pdfium_render::prelude::*;
use crate::readability::Readability;
use url::Url;

pub async fn get_html_headless(url: &str) -> anyhow::Result<String> {
    let options = LaunchOptions {
        headless: true,
        window_size: Some((820, 1180)),
        ..Default::default()
    };

    let browser = Browser::new(options)?;

    let tab = browser.new_tab()?;

    tab.navigate_to(url)?;
    tab.wait_until_navigated();
    let text = tab.get_content()?;

    Ok(text)
}
pub async fn get_webpage_text_headless(url: &str) -> anyhow::Result<String> {
    // set the headless Chrome to open a webpage in portrait mode of certain width and height
    // here in an iPad resolution, is a way to pursuade webserver to send less non-essential
    // data, and make the virtual browser to show the central content, for websites
    // with responsive design, with less clutter
    let options = LaunchOptions {
        headless: true,
        window_size: Some((820, 1180)),
        ..Default::default()
    };

    let browser = Browser::new(options)?;

    let tab = browser.new_tab()?;

    tab.navigate_to(url)?;
    tab.wait_until_navigated();

    let pdf_options: Option<PrintToPdfOptions> = Some(PrintToPdfOptions {
        landscape: Some(false),
        display_header_footer: Some(false),
        print_background: Some(false),
        scale: Some(0.5),
        paper_width: Some(11.0),
        paper_height: Some(17.0),
        margin_top: Some(0.1),
        margin_bottom: Some(0.1),
        margin_left: Some(0.1),
        margin_right: Some(0.1),
        // page_ranges: Some("1-2".to_string()),
        ignore_invalid_page_ranges: Some(true),
        prefer_css_page_size: Some(false),
        transfer_mode: None,
        ..Default::default()
    });

    let pdf_data = tab.print_to_pdf(pdf_options)?;

    let pdf_as_vec = pdf_data.to_vec();
    //code below uses dynamically linked libpdfium.dylib on a M1 Mac
    //it takes some efforts to bind libpdfium on different platforms
    //please visit https://github.com/ajrcarey/pdfium-render/tree/master
    //for more details
    let text = Pdfium::new(
        Pdfium::bind_to_library(Pdfium::pdfium_platform_library_name_at_path(
            // "/home/ubuntu/pdfium/lib/",
            "/Users/jaykchen/Downloads/pdfium-mac-arm64/lib/libpdfium.dylib",
        ))
        .or_else(|_| Pdfium::bind_to_system_library())?,
    )
    .load_pdf_from_byte_vec(pdf_as_vec, Some(""))?
    .pages()
    .iter()
    .map(|page| page.text().unwrap().all())
    .collect::<Vec<String>>()
    .join(" ");

    Ok(text)
}

pub async fn extract_article_text_http(url: &str) -> anyhow::Result<String> {
    let parsed_url = Url::parse(url)?;
    let scheme = parsed_url.scheme();
    let host = parsed_url.host_str().unwrap_or("");
    let base_url = Url::parse(&format!("{}://{}", scheme, host)).unwrap();

    let mut writer = Vec::new(); //container for body of a response
    let _ = request::get(url, &mut writer)?;

    let res = Readability::extract(&String::from_utf8(writer).unwrap(), Some(base_url)).await?;

    let output = html2text::from_read(res.to_string().as_bytes(), 80);

    Ok(output.to_string())
}

pub async fn extract_article_text_from_html(url: &str, html_str: String) -> anyhow::Result<String> {
    let parsed_url = Url::parse(url)?;
    let scheme = parsed_url.scheme();
    let host = parsed_url.host_str().unwrap_or("");
    let base_url = Url::parse(&format!("{}://{}", scheme, host))?;

    let res = Readability::extract(&html_str, Some(base_url)).await?;
    let output = html2text::from_read(res.to_string().as_bytes(), 80);

    Ok(output)
}

pub async fn text_to_use(url: &str) -> anyhow::Result<String> {
    let pdf_text = get_webpage_text_headless(url).await?;
    let html_str = get_html_headless(url).await?;
    let readah_text = extract_article_text_from_html(url, html_str).await?;

    let readah_text_len = readah_text.split_whitespace().count();
    let pdf_text_len = pdf_text.split_whitespace().count();

    let lots_of_text_on_page = pdf_text_len > 999;
    let readah_sees_lots_of_texts = readah_text_len > 500;

    if lots_of_text_on_page && readah_sees_lots_of_texts {
        return Ok(readah_text.to_string());
    }

    Ok(pdf_text.to_string())
}