lectito 0.3.0

Article extraction and readability parsing for Lectito.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

use super::regexes::RegexPattern;

pub const TAGS_TO_SCORE: &[&str] = &["section", "h2", "h3", "h4", "h5", "h6", "p", "td", "pre"];

pub const DEPRECATED_SIZE_ATTRIBUTE_ELEMS: &[&str] = &["table", "th", "td", "hr", "pre"];

pub const DEFAULT_CLASSES_TO_PRESERVE: &[&str] = &["page"];

pub const TITLE_SEPARATORS: &[&str] = &[" | ", " - ", " – ", " — ", " \\ ", " / ", " > ", " » "];

pub const PRESENTATIONAL_ATTRIBUTES: &[&str] = &[
    "align",
    "background",
    "bgcolor",
    "border",
    "cellpadding",
    "cellspacing",
    "frame",
    "hspace",
    "rules",
    "style",
    "valign",
    "vspace",
];

pub fn normalize_spaces(text: &str) -> String {
    RegexPattern::NormalizeWhitespace
        .to_regex()
        .replace_all(text, " ")
        .into_owned()
}

pub fn has_display_none(style: Option<&str>) -> bool {
    style
        .unwrap_or_default()
        .split(';')
        .filter_map(|declaration| declaration.split_once(':'))
        .any(|(property, value)| {
            property.trim().eq_ignore_ascii_case("display") && value.trim().eq_ignore_ascii_case("none")
        })
}

pub fn selector(pattern: &str) -> scraper::Selector {
    scraper::Selector::parse(pattern).expect("internal selector should parse")
}