article_scraper 3.0.0-alpha1

Scrap article contents from the web. Powered by fivefilters full text feed configurations & mozilla readability.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

use crate::{dom, error::FullTextParserError, util::Util};
use dom_query::NodeRef;
use url::Url;

#[derive(Debug, Clone)]
pub struct ImageObject {
    width: Option<u32>,
    height: Option<u32>,
    url: Option<Url>,
    description: Option<String>,
    name: Option<String>,
    parent_url: Option<Url>,
}

impl ImageObject {
    pub fn parse_node(node: &NodeRef) -> Option<Self> {
        if !dom::tag_name_is(node, "div") {
            return None;
        }

        let item_prop_image = dom::attr(node, "itemprop")
            .map(|prop| prop == "image")
            .unwrap_or(false);
        let item_type_image = Util::has_schema_org_type(node, "ImageObject");

        if !item_prop_image && !item_type_image {
            return None;
        }

        let meta_nodes = dom::elements_by_tag_name(node, "meta");

        let mut width = None;
        let mut height = None;
        let mut url = None;
        let mut description = None;
        let mut name = None;
        let mut parent_url = None;

        if let Some(parent) = node.parent()
            && dom::tag_name_is(&parent, "a")
        {
            let href = dom::attr(&parent, "href");
            parent_url = href.and_then(|href| Url::parse(&href).ok());
        }

        for meta_node in meta_nodes {
            let item_prop = dom::attr(&meta_node, "itemprop");
            let content_prop = dom::attr(&meta_node, "content");

            if let (Some(item_prop), Some(content_prop)) = (item_prop, content_prop) {
                if item_prop == "width" {
                    width = content_prop.parse::<u32>().ok();
                } else if item_prop == "height" {
                    height = content_prop.parse::<u32>().ok();
                } else if item_prop == "url" {
                    url = Url::parse(&content_prop).ok();
                } else if item_prop == "description" {
                    description = Some(content_prop);
                } else if item_prop == "name" {
                    name = Some(content_prop);
                }
            }
        }

        url.as_ref()?;

        Some(Self {
            width,
            height,
            url,
            description,
            name,
            parent_url,
        })
    }

    /// Replaces `node` (or its `<a>` parent) with an `<imageobject>` in the same place.
    pub fn replace(&self, node: &NodeRef) -> Result<(), FullTextParserError> {
        let parent = node.parent().ok_or(FullTextParserError::Dom)?;

        if dom::tag_name_is(&parent, "a") {
            return self.replace(&parent);
        }

        let tree = node.tree;
        let root = tree.new_element("imageobject");
        dom::replace_with(node, &root);

        let a = tree.new_element("a");
        dom::append_child(&root, &a);

        let img = tree.new_element("img");
        dom::append_child(&a, &img);

        if let Some(width) = self.width {
            img.set_attr("width", &width.to_string());
        }

        if let Some(height) = self.height {
            img.set_attr("height", &height.to_string());
        }

        if let Some(description) = self.description.as_deref() {
            img.set_attr("alt", description);
        }

        if let Some(name) = self.name.as_deref() {
            img.set_attr("title", name);
        }

        // The image links to the page its `<a>` parent pointed to, otherwise to itself.
        if let Some(href) = self.parent_url.as_ref().or(self.url.as_ref()) {
            a.set_attr("href", href.as_str());
        }

        if let Some(url) = self.url.as_ref() {
            img.set_attr("src", url.as_str());
        }

        Ok(())
    }
}