kumo 0.5.0

An async web crawling framework for Rust - Scrapy for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

use std::time::Duration;

use kumo::robots::parse_crawl_delay_for_agent;

#[test]
fn parses_matching_agent_crawl_delay() {
    let txt = "\
User-agent: *
Crawl-delay: 2

User-agent: other
Crawl-delay: 10
";

    assert_eq!(
        parse_crawl_delay_for_agent(txt, "kumo"),
        Some(Duration::from_secs(2))
    );
}

#[test]
fn matching_named_agent_overrides_wildcard() {
    let txt = "\
User-agent: *
Crawl-delay: 5

User-agent: kumo
Crawl-delay: 1.5
";

    assert_eq!(
        parse_crawl_delay_for_agent(txt, "kumo"),
        Some(Duration::from_millis(1500))
    );
}

#[test]
fn ignores_invalid_crawl_delay_values() {
    let txt = "\
User-agent: *
Crawl-delay: -1

User-agent: kumo
Crawl-delay: nope
";

    assert_eq!(parse_crawl_delay_for_agent(txt, "kumo"), None);
}