rust_scrapling 0.4.0

A Rust port of Scrapling - modern web scraping framework with CSS selectors, async HTTP, and spider-based crawling
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

use rust_scrapling::spiders::request::SpiderRequest;

#[test]
fn test_basic_creation() {
    let req = SpiderRequest::new("https://example.com/page");
    assert_eq!(req.url(), "https://example.com/page");
    assert_eq!(req.method(), "GET");
    assert_eq!(req.priority(), 0);
    assert!(!req.dont_filter());
    assert_eq!(req.retry_count(), 0);
    assert!(req.callback_name().is_none());
    assert!(req.body().is_none());
    assert!(!req.fingerprint().is_empty());
}

#[test]
fn test_builder() {
    let req = SpiderRequest::builder("https://example.com")
        .method("POST")
        .priority(5)
        .session_id("sess1")
        .callback("parse_item")
        .dont_filter(true)
        .header("Accept", "text/html")
        .body("{\"key\":\"value\"}")
        .meta("depth", serde_json::json!(2))
        .build();

    assert_eq!(req.method(), "POST");
    assert_eq!(req.priority(), 5);
    assert_eq!(req.session_id(), "sess1");
    assert_eq!(req.callback_name(), Some("parse_item"));
    assert!(req.dont_filter());
    assert_eq!(req.headers().get("Accept").unwrap(), "text/html");
    assert_eq!(req.body(), Some("{\"key\":\"value\"}"));
    assert_eq!(req.meta("depth"), Some(&serde_json::json!(2)));
}

#[test]
fn test_fingerprint_equality() {
    let r1 = SpiderRequest::new("https://example.com/page");
    let r2 = SpiderRequest::new("https://example.com/page");
    assert_eq!(r1, r2);
    assert_eq!(r1.fingerprint(), r2.fingerprint());
}

#[test]
fn test_fingerprint_inequality() {
    let r1 = SpiderRequest::new("https://example.com/page1");
    let r2 = SpiderRequest::new("https://example.com/page2");
    assert_ne!(r1, r2);
    assert_ne!(r1.fingerprint(), r2.fingerprint());
}

#[test]
fn test_copy() {
    let req = SpiderRequest::new("https://example.com");
    let copy = req.copy();
    assert_eq!(req, copy);
    assert_eq!(req.url(), copy.url());
}

#[test]
fn test_domain() {
    let req = SpiderRequest::new("https://www.example.com/path?q=1");
    assert_eq!(req.domain(), Some("www.example.com".to_string()));
}

#[test]
fn test_ordering() {
    let low = SpiderRequest::builder("https://example.com/low")
        .priority(1)
        .build();
    let high = SpiderRequest::builder("https://example.com/high")
        .priority(10)
        .build();
    assert!(high > low);
}

#[test]
fn test_ord_consistent_with_partial_eq_for_same_priority_different_fingerprint() {
    // Ord's contract requires a == b iff a.cmp(b) == Equal. Two requests with
    // the same priority but different URLs (hence different fingerprints)
    // are not PartialEq-equal, so cmp must not report them Equal either —
    // it previously compared only priority, breaking the contract.
    use std::cmp::Ordering;
    let a = SpiderRequest::builder("https://example.com/a")
        .priority(1)
        .build();
    let b = SpiderRequest::builder("https://example.com/b")
        .priority(1)
        .build();
    assert_ne!(a, b);
    assert_ne!(a.cmp(&b), Ordering::Equal);
    // Priority still dominates the comparison.
    assert_eq!(a.cmp(&b), a.fingerprint().cmp(b.fingerprint()));
}

#[test]
fn test_ord_equal_only_for_equal_requests() {
    use std::cmp::Ordering;
    let a = SpiderRequest::builder("https://example.com/same")
        .priority(3)
        .build();
    let b = SpiderRequest::builder("https://example.com/same")
        .priority(3)
        .build();
    assert_eq!(a, b);
    assert_eq!(a.cmp(&b), Ordering::Equal);
}

#[test]
fn test_meta() {
    let mut req = SpiderRequest::new("https://example.com");
    assert!(req.meta("key").is_none());
    req.set_meta("key", serde_json::json!("value"));
    assert_eq!(req.meta("key"), Some(&serde_json::json!("value")));
}

#[test]
fn test_setters() {
    let mut req = SpiderRequest::new("https://example.com");
    req.set_dont_filter(true);
    assert!(req.dont_filter());
    req.set_session_id("abc");
    assert_eq!(req.session_id(), "abc");
    req.increment_retry();
    assert_eq!(req.retry_count(), 1);
    req.increment_retry();
    assert_eq!(req.retry_count(), 2);
}

#[test]
fn test_priority_values() {
    let r1 = SpiderRequest::builder("https://a.com").priority(-5).build();
    let r2 = SpiderRequest::builder("https://b.com").priority(0).build();
    let r3 = SpiderRequest::builder("https://c.com").priority(5).build();
    assert!(r3 > r2);
    assert!(r2 > r1);
}