kumo 0.3.0

An async web crawling framework for Rust - Scrapy for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

use kumo::middleware::{FetchRequest, Middleware, ProxyRotator};

fn make_request() -> FetchRequest {
    FetchRequest::new("https://example.com", 0)
}

#[tokio::test]
async fn round_robin_assigns_proxies_in_order() {
    let rotator = ProxyRotator::new(vec!["http://p1:8080", "http://p2:8080"]);
    let mut req = make_request();
    rotator.before_request(&mut req).await.unwrap();
    assert_eq!(req.proxy.as_deref(), Some("http://p1:8080"));
    rotator.before_request(&mut req).await.unwrap();
    assert_eq!(req.proxy.as_deref(), Some("http://p2:8080"));
    rotator.before_request(&mut req).await.unwrap();
    assert_eq!(req.proxy.as_deref(), Some("http://p1:8080"));
}

#[tokio::test]
async fn random_picks_from_pool() {
    let proxies = vec!["http://p1:8080", "http://p2:8080", "http://p3:8080"];
    let rotator = ProxyRotator::random(proxies.clone());
    for _ in 0..30 {
        let mut req = make_request();
        rotator.before_request(&mut req).await.unwrap();
        let picked = req.proxy.unwrap();
        assert!(
            proxies.contains(&picked.as_str()),
            "unexpected proxy: {picked}"
        );
    }
}

#[tokio::test]
async fn empty_pool_leaves_proxy_none() {
    let rotator = ProxyRotator::new(Vec::<String>::new());
    let mut req = make_request();
    rotator.before_request(&mut req).await.unwrap();
    assert!(req.proxy.is_none());
}