kumo 0.2.3

An async web crawling framework for Rust — Scrapy for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

#![cfg(feature = "persistence")]

use kumo::{
    CrawlRequest,
    frontier::{FileFrontier, Frontier},
    request::FrontierRequest,
};
use reqwest::header::{HeaderName, HeaderValue};
use std::time::Duration;
use tempfile::tempdir;

#[tokio::test]
async fn new_frontier_is_empty() {
    let dir = tempdir().unwrap();
    let f = FileFrontier::open(dir.path()).unwrap();
    assert!(f.is_empty().await);
}

#[tokio::test]
async fn push_and_pop() {
    let dir = tempdir().unwrap();
    let f = FileFrontier::open(dir.path()).unwrap();
    assert!(f.push("https://example.com".into(), 0).await);
    let item = f.pop().await.unwrap();
    assert_eq!(item.0, "https://example.com");
    assert_eq!(item.1, 0);
    assert_eq!(item.2, 0);
}

#[tokio::test]
async fn deduplication_works() {
    let dir = tempdir().unwrap();
    let f = FileFrontier::open(dir.path()).unwrap();
    assert!(f.push("https://example.com".into(), 0).await);
    assert!(!f.push("https://example.com".into(), 0).await);
    assert_eq!(f.len().await, 1);
}

#[tokio::test]
async fn resumes_queue_from_disk() {
    let dir = tempdir().unwrap();
    {
        let f = FileFrontier::open(dir.path()).unwrap();
        f.push("https://a.com".into(), 0).await;
        f.push("https://b.com".into(), 1).await;
        f.flush().await.unwrap();
    }
    let f2 = FileFrontier::open(dir.path()).unwrap();
    assert_eq!(f2.len().await, 2);
    let first = f2.pop().await.unwrap();
    assert_eq!(first.0, "https://a.com");
}

#[tokio::test]
async fn resumes_dedup_from_disk() {
    let dir = tempdir().unwrap();
    {
        let f = FileFrontier::open(dir.path()).unwrap();
        f.push("https://a.com".into(), 0).await;
        f.flush().await.unwrap();
    }
    let f2 = FileFrontier::open(dir.path()).unwrap();
    f2.pop().await;
    assert!(!f2.push("https://a.com".into(), 0).await);
}

#[tokio::test]
async fn flush_replaces_state_files_atomically_without_temp_leftovers() {
    let dir = tempdir().unwrap();
    let f = FileFrontier::open(dir.path()).unwrap();

    f.push_request(CrawlRequest::get("https://example.com/a"), 0)
        .await;
    f.flush().await.unwrap();

    assert!(dir.path().join("queue.json").exists());
    assert!(dir.path().join("seen.json").exists());
    assert!(!dir.path().join("queue.json.tmp").exists());
    assert!(!dir.path().join("seen.json.tmp").exists());

    let f = FileFrontier::open(dir.path()).unwrap();
    assert_eq!(f.len().await, 1);
}

#[tokio::test]
async fn request_metadata_survives_flush_and_resume() {
    let dir = tempdir().unwrap();
    {
        let f = FileFrontier::open(dir.path()).unwrap();
        f.push_request(
            CrawlRequest::post("https://example.com/api", br#"{"page":2}"#.to_vec())
                .header(
                    HeaderName::from_static("x-api-key"),
                    HeaderValue::from_static("secret"),
                )
                .priority(10)
                .meta("kind", "listing"),
            2,
        )
        .await;
        f.flush().await.unwrap();
    }

    let f = FileFrontier::open(dir.path()).unwrap();
    let queued = f.pop_request().await.unwrap();
    assert_eq!(queued.depth(), 2);
    assert_eq!(queued.request().url(), "https://example.com/api");
    assert_eq!(queued.request().method_ref(), reqwest::Method::POST);
    assert_eq!(queued.request().body_bytes(), Some(&br#"{"page":2}"#[..]));
    assert_eq!(
        queued.request().headers()["x-api-key"],
        HeaderValue::from_static("secret")
    );
    assert_eq!(queued.request().priority_value(), 10);
    assert_eq!(
        queued.request().meta_value("kind"),
        Some(&serde_json::json!("listing"))
    );
}

#[tokio::test]
async fn scheduled_retry_survives_flush_and_resume() {
    let dir = tempdir().unwrap();
    {
        let f = FileFrontier::open(dir.path()).unwrap();
        f.push_request_force(
            FrontierRequest::new(CrawlRequest::get("https://example.com/retry"), 3, 2)
                .scheduled_after(Duration::from_secs(30)),
        )
        .await;
        f.flush().await.unwrap();
    }

    let f = FileFrontier::open(dir.path()).unwrap();
    let queued = f.pop_request().await.unwrap();
    assert_eq!(queued.depth(), 3);
    assert_eq!(queued.retry_count(), 2);
    assert_eq!(queued.request().url(), "https://example.com/retry");
    assert!(queued.scheduled_at().is_some());
}

#[tokio::test]
async fn dont_filter_allows_duplicate_url() {
    let dir = tempdir().unwrap();
    let f = FileFrontier::open(dir.path()).unwrap();
    assert!(
        f.push_request(CrawlRequest::get("https://example.com"), 0)
            .await
    );
    assert!(
        f.push_request(
            CrawlRequest::get("https://example.com").dont_filter(true),
            0,
        )
        .await
    );
    assert_eq!(f.len().await, 2);
}