rust_scrapling 0.2.1

A Rust port of Scrapling - modern web scraping framework with CSS selectors, async HTTP, and spider-based crawling
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

use crate::core::TextHandler;
use crate::parser::selector::Selector;

/// A collection of Selector items with batch operations.
#[derive(Debug, Clone)]
pub struct Selectors {
    items: Vec<Selector>,
}

impl Selectors {
    /// Create a new Selectors from a Vec of Selector.
    pub fn new(items: Vec<Selector>) -> Self {
        Self { items }
    }

    /// Number of selectors in the collection.
    pub fn len(&self) -> usize {
        self.items.len()
    }

    /// Whether the collection is empty.
    pub fn is_empty(&self) -> bool {
        self.items.is_empty()
    }

    /// Return the first selector, if any.
    pub fn first(&self) -> Option<&Selector> {
        self.items.first()
    }

    /// Return the last selector, if any.
    pub fn last(&self) -> Option<&Selector> {
        self.items.last()
    }

    /// Run a CSS selector across all items and flatten results.
    pub fn css(&self, selector: &str) -> Selectors {
        let items: Vec<Selector> = self
            .items
            .iter()
            .flat_map(|s| s.css(selector).into_iter())
            .collect();
        Selectors::new(items)
    }

    /// Apply regex across all items' text content.
    pub fn re(
        &self,
        pattern: &str,
        replace_entities: bool,
        clean_match: bool,
        case_sensitive: bool,
    ) -> Vec<TextHandler> {
        self.items
            .iter()
            .flat_map(|s| s.re(pattern, replace_entities, clean_match, case_sensitive))
            .collect()
    }

    /// Apply regex across all items and return the first match.
    pub fn re_first(
        &self,
        pattern: &str,
        replace_entities: bool,
        clean_match: bool,
        case_sensitive: bool,
    ) -> Option<TextHandler> {
        for s in &self.items {
            if let Some(m) = s.re_first(pattern, replace_entities, clean_match, case_sensitive) {
                return Some(m);
            }
        }
        None
    }

    /// Get text of the first item, or a default.
    pub fn get_first(&self, default: Option<TextHandler>) -> Option<TextHandler> {
        self.items
            .first()
            .map(|s| s.get_all_text("", true, &[], None))
            .or(default)
    }

    /// Get text of all items.
    pub fn getall(&self) -> Vec<TextHandler> {
        self.items
            .iter()
            .map(|s| s.get_all_text("", true, &[], None))
            .collect()
    }

    /// Search for the first item matching a predicate.
    pub fn search<F>(&self, func: F) -> Option<&Selector>
    where
        F: Fn(&Selector) -> bool,
    {
        self.items.iter().find(|s| func(s))
    }

    /// Filter items by a predicate, returning a new Selectors.
    pub fn filter<F>(&self, func: F) -> Selectors
    where
        F: Fn(&Selector) -> bool,
    {
        let items: Vec<Selector> = self.items.iter().filter(|s| func(s)).cloned().collect();
        Selectors::new(items)
    }
}

impl std::ops::Index<usize> for Selectors {
    type Output = Selector;
    fn index(&self, index: usize) -> &Self::Output {
        &self.items[index]
    }
}

impl IntoIterator for Selectors {
    type Item = Selector;
    type IntoIter = std::vec::IntoIter<Selector>;
    fn into_iter(self) -> Self::IntoIter {
        self.items.into_iter()
    }
}

impl<'a> IntoIterator for &'a Selectors {
    type Item = &'a Selector;
    type IntoIter = std::slice::Iter<'a, Selector>;
    fn into_iter(self) -> Self::IntoIter {
        self.items.iter()
    }
}