web-parser 0.1.3

This website parser library allows asynchronous search, fetching and extracting data from web-pages in multiple formats.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

use crate::prelude::*;
use super::Nodes;

/// The HTML node
#[derive(Debug, Clone)]
pub struct Node<'a> {
    element: scraper::ElementRef<'a>,
    selector: scraper::Selector
}

impl<'a> Node<'a> {
    /// Creates a new Node from scraper::ElementRef
    pub(crate) fn new(element: scraper::ElementRef<'a>) -> Self {
        Self {
            element,
            selector: scraper::Selector::parse("*").unwrap()
        }
    }

    /// Select HTML node by CSS selector
    pub fn select(&self, selector: &'static str) -> Result<Option<Node<'a>>> {
        let sel = scraper::Selector::parse(selector).map_err(Error::from)?;
        
        let node = self.element
            .select(&sel)
            .next()
            .map(Node::new);
        
        Ok(node)
    }

    /// Select HTML nodes by CSS selector
    pub fn select_all(&mut self, selector: &'static str) -> Result<Option<Nodes>> {
        self.selector = scraper::Selector::parse(selector).map_err(Error::from)?;
        let mut nodes = self.element.select(&self.selector).peekable();

        if nodes.peek().is_some() {
            Ok(Some(Nodes::new(None, Some(nodes))))
        } else {
            Ok(None)
        }
    }

    /// Returns a node parent
    pub fn parent(&self) -> Option<Node<'a>> {
        self.element.parent()
            .and_then(|node| scraper::ElementRef::wrap(node))
            .map(Node::new)
    }

    /// Returns a node attribute
    pub fn attr(&self, name: &str) -> Option<&str> {
        self.element.value().attr(name)
    }

    /// Returns a node text contents
    pub fn text(&self) -> String {
        self.element.text().collect()
    }

    /// Returns a node inner HTML
    pub fn html(&self) -> String {
        self.element.html()
    }

    /// Returns node content from node excluding tags from black list
    pub fn filter_text(&self, black_list: &[&str]) -> String {
        Self::filter_elem_text(self.element, black_list)
            .split_whitespace()
            .collect::<Vec<_>>()
            .join(" ")
    }

    /// Returns node content from element excluding tags from black list
    fn filter_elem_text(node: scraper::element_ref::ElementRef, black_list: &[&str]) -> String {
        let tag_name = node.value().name();

        // filtering by black list:
        if black_list.contains(&tag_name) {
            return String::new();
        }

        // collecting text:
        let mut result = String::new();

        for child in node.children() {
            match child.value() {
                scraper::node::Node::Text(text) => {
                    result.push(' ');
                    result.push_str(text);
                }
                scraper::node::Node::Element(_) => {
                    if let Some(child_element) = scraper::ElementRef::wrap(child) {
                        result.push(' ');
                        result.push_str(&Self::filter_elem_text(child_element, black_list));
                    }
                }
                _ => {}
            }
        }
        result
    }
}