article_scraper 3.0.0-alpha1

Scrap article contents from the web. Powered by fivefilters full text feed configurations & mozilla readability.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

use crate::{dom, error::FullTextParserError, util::Util};
use dom_query::NodeRef;
use url::Url;

pub struct VideoObject {
    thumbnail_url: Option<String>,
    content_url: Option<Url>,
    embed_url: Option<Url>,
    description: Option<String>,
    name: Option<String>,
}

impl VideoObject {
    pub fn parse_node(node: &NodeRef) -> Option<Self> {
        if !dom::tag_name_is(node, "div") {
            return None;
        }

        let item_prop_video = dom::attr(node, "itemprop")
            .map(|prop| prop == "video")
            .unwrap_or(false);
        let item_type_video = Util::has_schema_org_type(node, "VideoObject");

        if !item_prop_video && !item_type_video {
            return None;
        }

        let meta_nodes = dom::elements_by_tag_name(node, "meta");

        let mut thumbnail_url = None;
        let mut content_url = None;
        let mut embed_url = None;
        let mut description = None;
        let mut name = None;

        for meta_node in meta_nodes {
            let item_prop = dom::attr(&meta_node, "itemprop");
            let content_prop = dom::attr(&meta_node, "content");

            if let (Some(item_prop), Some(content_prop)) = (item_prop, content_prop) {
                if item_prop == "thumbnailUrl" {
                    thumbnail_url = Some(content_prop);
                } else if item_prop == "contentURL" {
                    content_url = Self::parse_url(&content_prop);
                } else if item_prop == "embedURL" {
                    embed_url = Self::parse_url(&content_prop);
                } else if item_prop == "description" {
                    description = Some(content_prop);
                } else if item_prop == "name" {
                    name = Some(content_prop);
                }
            }
        }

        if thumbnail_url.is_none()
            && content_url.is_none()
            && embed_url.is_none()
            && description.is_none()
            && name.is_none()
        {
            return None;
        }

        Some(Self {
            thumbnail_url,
            content_url,
            embed_url,
            description,
            name,
        })
    }

    fn parse_url(url: &str) -> Option<Url> {
        let url = if url.starts_with("//") {
            format!("https:{url}")
        } else {
            url.into()
        };

        Url::parse(&url)
            .inspect_err(|err| tracing::error!(%err, "parse video object url"))
            .ok()
    }

    /// Replaces `node` with a `<videoobject>` in the same place.
    pub fn replace(&self, node: &NodeRef) -> Result<(), FullTextParserError> {
        node.parent().ok_or(FullTextParserError::Dom)?;

        let tree = node.tree;
        let root = tree.new_element("videoobject");
        dom::replace_with(node, &root);

        if let Some(name) = self.name.as_deref() {
            let title = tree.new_element("h3");
            dom::append_child(&root, &title);
            dom::append_child(&title, &tree.new_text(name));
        }

        if self.name != self.description
            && let Some(description) = self.description.as_deref()
        {
            let desc = tree.new_element("p");
            dom::append_child(&root, &desc);
            dom::append_child(&desc, &tree.new_text(description));
        }

        let a = tree.new_element("a");
        dom::append_child(&root, &a);
        if let Some(embed_url) = self.embed_url.as_ref() {
            a.set_attr("href", embed_url.as_str());
        } else if let Some(content_url) = self.content_url.as_ref() {
            a.set_attr("href", content_url.as_str());
        }

        let img = tree.new_element("img");
        dom::append_child(&a, &img);
        if let Some(thumbnail_url) = self.thumbnail_url.as_deref() {
            img.set_attr("src", thumbnail_url);
        }

        Ok(())
    }
}