turboscraper 0.1.1

A high-performance, concurrent web scraping framework for Rust with built-in support for retries, storage backends, and concurrent request handling
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

use reqwest::Method;
use serde::Serialize;
use serde_json::Value;
use std::collections::HashMap;
use url::Url;

use crate::core::SpiderCallback;

#[derive(Debug, Clone, Serialize)]
pub struct HttpRequest {
    pub url: Url,
    pub callback: SpiderCallback,
    pub meta: Option<Value>,
    pub depth: usize, // Tracks the actual depth of the request
    #[serde(with = "http_serde::method")]
    pub method: Method,
    pub headers: HashMap<String, String>,
    pub body: Option<String>,
}

impl HttpRequest {
    pub fn new(url: Url, callback: SpiderCallback, depth: usize) -> Self {
        Self {
            url,
            callback,
            meta: None,
            depth,
            method: Method::GET,
            headers: HashMap::new(),
            body: None,
        }
    }

    pub fn with_method(mut self, method: Method) -> Self {
        self.method = method;
        self
    }

    pub fn with_header<K, V>(mut self, key: K, value: V) -> Self
    where
        K: Into<String>,
        V: Into<String>,
    {
        self.headers.insert(key.into(), value.into());
        self
    }

    pub fn with_headers(mut self, headers: HashMap<String, String>) -> Self {
        self.headers.extend(headers);
        self
    }

    pub fn with_body<T: Into<String>>(mut self, body: T) -> Self {
        self.body = Some(body.into());
        self
    }

    pub fn with_meta<T: serde::Serialize>(mut self, meta: T) -> crate::ScraperResult<Self> {
        self.meta = Some(serde_json::to_value(meta).unwrap());
        Ok(self)
    }
}