kumo 0.3.2

An async web crawling framework for Rust - Scrapy for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

//! Scrapes all books from https://books.toscrape.com, following pagination.
//! Demonstrates: multi-page crawl, allowed_domains, max_depth, RateLimiter, retry, JsonStore.
//!
//! Run with: cargo run --example books

use kumo::prelude::*;
use serde::Serialize;

#[derive(Debug, Serialize)]
struct Book {
    title: String,
    price: String,
    rating: String,
    url: String,
}

struct BooksSpider;

#[async_trait::async_trait]
impl Spider for BooksSpider {
    type Item = Book;

    fn name(&self) -> &str {
        "books"
    }

    fn start_urls(&self) -> Vec<String> {
        vec!["https://books.toscrape.com/catalogue/page-1.html".into()]
    }

    fn allowed_domains(&self) -> Vec<&str> {
        vec!["books.toscrape.com"]
    }

    fn max_depth(&self) -> Option<usize> {
        Some(60) // site has 50 pages
    }

    async fn parse(&self, res: &Response) -> Result<Output<Self::Item>, KumoError> {
        // Extract books on this page.
        let books: Vec<Book> = res
            .css("article.product_pod")
            .iter()
            .map(|el| {
                let title = el
                    .css("h3 a")
                    .first()
                    .and_then(|a| a.attr("title"))
                    .unwrap_or_default();
                let price = el
                    .css(".price_color")
                    .first()
                    .map(|e| e.text())
                    .unwrap_or_default();
                let rating = el
                    .css(".star-rating")
                    .first()
                    .and_then(|e| e.attr("class"))
                    .and_then(|cls| cls.split_whitespace().nth(1).map(String::from))
                    .unwrap_or_default();
                let href = el
                    .css("h3 a")
                    .first()
                    .and_then(|a| a.attr("href"))
                    .unwrap_or_default();
                Book {
                    title,
                    price,
                    rating,
                    url: res.urljoin(&href),
                }
            })
            .collect();

        // Follow the "next" pagination link if present.
        let next_url = res
            .css("li.next a")
            .first()
            .and_then(|el| el.attr("href"))
            .map(|href| res.urljoin(&href));

        let mut output = Output::new().items(books);
        if let Some(url) = next_url {
            output = output.follow(url);
        }
        Ok(output)
    }
}

#[tokio::main]
async fn main() -> Result<(), KumoError> {
    tracing_subscriber::fmt()
        .with_env_filter("kumo=info")
        .init();

    let stats = CrawlEngine::builder()
        .concurrency(3)
        .middleware(RateLimiter::per_second(2.0))
        .middleware(DefaultHeaders::new().user_agent("kumo/0.2"))
        .store(JsonStore::new("books.json")?)
        .crawl_delay(std::time::Duration::from_millis(300))
        .retry(2, std::time::Duration::from_millis(500))
        .respect_robots_txt(true)
        .run(BooksSpider)
        .await?;

    println!(
        "Done — scraped {} books from {} pages ({} errors)",
        stats.items_scraped, stats.pages_crawled, stats.errors
    );
    Ok(())
}