spider-lib 3.0.4

A Rust-based web scraping framework inspired by Scrapy (Python).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

use spider_lib::prelude::*;
use spider_lib::route_by_rule;

#[scraped_item]
struct RuleRoutedItem {
    page_kind: String,
    title: String,
    url: String,
}

struct RuleSpider;

impl RuleSpider {
    async fn parse_listing(
        &self,
        response: Response,
        _state: &(),
    ) -> Result<ParseOutput<RuleRoutedItem>, SpiderError> {
        let mut output = ParseOutput::new();

        let title = response
            .css("title::text")?
            .get()
            .unwrap_or_default()
            .trim()
            .to_string();

        output.add_item(RuleRoutedItem {
            page_kind: "listing".to_string(),
            title,
            url: response.url.to_string(),
        });

        Ok(output)
    }

    async fn parse_book(
        &self,
        response: Response,
        _state: &(),
    ) -> Result<ParseOutput<RuleRoutedItem>, SpiderError> {
        let mut output = ParseOutput::new();

        let title = response
            .css(".product_main h1::text")?
            .get()
            .unwrap_or_default()
            .trim()
            .to_string();

        output.add_item(RuleRoutedItem {
            page_kind: "book".to_string(),
            title,
            url: response.url.to_string(),
        });

        Ok(output)
    }

    async fn parse_default(
        &self,
        response: Response,
        _state: &(),
    ) -> Result<ParseOutput<RuleRoutedItem>, SpiderError> {
        let mut output = ParseOutput::new();
        output.add_item(RuleRoutedItem {
            page_kind: "default".to_string(),
            title: response
                .discovery_rule_name()
                .unwrap_or_else(|| "unmatched".to_string()),
            url: response.url.to_string(),
        });
        Ok(output)
    }
}

#[async_trait]
impl Spider for RuleSpider {
    type Item = RuleRoutedItem;
    type State = ();

    fn start_requests(&self) -> Result<StartRequests<'_>, SpiderError> {
        Ok(StartRequests::Urls(vec!["https://books.toscrape.com"]))
    }

    async fn parse(
        &self,
        response: Response,
        state: &Self::State,
    ) -> Result<ParseOutput<Self::Item>, SpiderError> {
        route_by_rule!(
            response,
            "listing" => self.parse_listing(response, state).await,
            "book" => self.parse_book(response, state).await,
            _ => self.parse_default(response, state).await,
        )
    }
}

#[tokio::main]
async fn main() -> Result<(), SpiderError> {
    let listing_rule = DiscoveryRule::new("listing")
        .with_allow_patterns(["https://books.toscrape.com/"])
        .with_allowed_tags(["a"])
        .with_allowed_attributes(["href"])
        .with_follow_allow_patterns(["*catalogue/*.html"])
        .with_follow_deny_patterns(["*/page-*.html"])
        .with_denied_link_types([LinkType::Image, LinkType::Script, LinkType::Stylesheet]);

    let book_rule = DiscoveryRule::new("book").with_allow_patterns(["*catalogue/*.html"]);

    let crawler = CrawlerBuilder::new(RuleSpider)
        .discovery_mode(DiscoveryMode::HtmlLinks)
        .discover_same_site_only(true)
        .add_discovery_rule(book_rule)
        .add_discovery_rule(listing_rule)
        .limit(6)
        .log_level(log::LevelFilter::Info)
        .build()
        .await?;

    crawler.start_crawl().await
}