kumo 0.5.0

An async web crawling framework for Rust - Scrapy for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

use std::time::Duration;

use kumo::prelude::*;
use serde::Serialize;

#[derive(Debug, Serialize)]
struct Page {
    url: String,
    title: String,
}

struct PoliteSpider;

#[async_trait::async_trait]
impl Spider for PoliteSpider {
    type Item = Page;

    fn name(&self) -> &str {
        "polite"
    }

    fn start_urls(&self) -> Vec<String> {
        vec!["https://quotes.toscrape.com".to_string()]
    }

    fn allowed_domains(&self) -> Vec<&str> {
        vec!["quotes.toscrape.com"]
    }

    async fn parse(&self, response: &Response) -> Result<Output<Self::Item>, KumoError> {
        let title = response
            .css("title")
            .first()
            .map(|title| title.text())
            .unwrap_or_default();

        let mut output = Output::new().item(Page {
            url: response.url().to_string(),
            title,
        });

        if let Some(next) = response
            .css("li.next a")
            .first()
            .and_then(|link| link.attr("href"))
            .map(|href| response.urljoin(&href))
        {
            output = output.request(
                CrawlRequest::get(next)
                    .priority(10)
                    .meta("source", "pagination"),
            );
        }

        Ok(output)
    }
}

#[tokio::main]
async fn main() -> Result<(), KumoError> {
    let stats = CrawlEngine::builder()
        .concurrency(16)
        .politeness(
            PolitenessPolicy::new()
                .per_domain_concurrency(2)
                .per_domain_delay(Duration::from_millis(500)),
        )
        .fingerprint_policy(FingerprintPolicy::default().strip_tracking_params(true))
        .store(JsonlStore::new("polite-pages.jsonl")?)
        .run(PoliteSpider)
        .await?;

    println!(
        "crawled {} pages, scheduled {}, deduped {}",
        stats.pages_crawled, stats.scheduled, stats.deduped
    );
    Ok(())
}