scraper-trail 0.2.0

Scraping framework and tools
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

use crate::multi_value::MultiValue;
use crate::{
    exchange::{Exchange, Response},
    request::Request,
};
use http::{StatusCode, header::HeaderMap};
use std::borrow::Cow;
use std::collections::HashMap;

#[derive(Debug, thiserror::Error)]
pub enum Error {
    #[error("HTTP client error")]
    Http(#[from] reqwest::Error),
    #[error("Invalid header")]
    Header(#[from] crate::request::HeaderError),
    #[error("Header value serialization error")]
    HeaderValueToStr(#[from] http::header::ToStrError),
    #[error("Unexpected status")]
    UnexpectedStatus {
        status_code: http::StatusCode,
        body: Option<String>,
    },
}

pub async fn json_send<'a>(
    client: &reqwest::Client,
    request: Request<'a>,
) -> Result<crate::exchange::Exchange<'a, serde_json::Value>, Error> {
    let builder = build_request(client, &request)?;
    let response = builder.send().await?;
    let status_code = response.status();
    let headers = response.headers();
    let headers = response_headers_to_index_map(headers)?;

    if status_code == StatusCode::OK {
        let json = response.json().await?;

        Ok(Exchange {
            request,
            response: Response {
                headers,
                data: json,
            },
        })
    } else {
        // We attempt to retrieve the body for better error messages, but ignore any failure here.
        let body = response.text().await.ok();

        Err(Error::UnexpectedStatus { status_code, body })
    }
}

pub async fn text_send<'a>(
    client: &reqwest::Client,
    request: Request<'a>,
) -> Result<crate::exchange::Exchange<'a, String>, Error> {
    let builder = build_request(client, &request)?;
    let response = builder.send().await?;
    let status_code = response.status();
    let headers = response.headers();
    let headers = response_headers_to_index_map(headers)?;

    if status_code == StatusCode::OK {
        let text = response.text().await?;

        Ok(Exchange {
            request,
            response: Response {
                headers,
                data: text,
            },
        })
    } else {
        // We attempt to retrieve the body for better error messages, but ignore any failure here.
        let body = response.text().await.ok();

        Err(Error::UnexpectedStatus { status_code, body })
    }
}

fn build_request<'a>(
    client: &reqwest::Client,
    request: &'a Request<'a>,
) -> Result<reqwest::RequestBuilder, crate::request::HeaderError> {
    let builder = client
        .request(request.method.clone(), request.url.clone())
        .headers(request.header_map()?);

    Ok(if let Some(body) = request.body.as_ref() {
        builder.body(body.to_string())
    } else {
        builder
    })
}

fn response_headers_to_index_map(
    response_headers: &HeaderMap,
) -> Result<HashMap<Cow<'static, str>, MultiValue<'static>>, http::header::ToStrError> {
    let mut result: HashMap<Cow<'static, str>, MultiValue<'static>> = HashMap::new();

    for (name, value) in response_headers {
        let value = value.to_str()?;

        match result.entry(name.as_str().to_string().into()) {
            std::collections::hash_map::Entry::Occupied(mut entry) => {
                let multi_value = entry.get_mut();
                multi_value.push(value.to_string());
            }
            std::collections::hash_map::Entry::Vacant(entry) => {
                entry.insert(MultiValue::new(value.to_string()));
            }
        }
    }

    Ok(result)
}