ingester 0.0.1

Turn paginated APIs into rate-limited async streams of parsed items
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

use bytes::Bytes;
use http::{HeaderMap, Method, header::InvalidHeaderValue};
use url::Url;

/// One HTTP request that an [`Endpoint`](crate::Endpoint) makes.
#[derive(Debug, Clone)]
pub struct Request {
    pub method: Method,
    pub url: Url,
    pub headers: HeaderMap,
    pub body: Option<Bytes>,
}

#[cfg(feature = "reqwest")]
impl From<reqwest::Request> for Request {
    fn from(r: reqwest::Request) -> Self {
        Self {
            method: r.method().clone(),
            url: r.url().clone(),
            headers: r.headers().clone(),
            body: r
                .body()
                .and_then(|b| b.as_bytes().map(Bytes::copy_from_slice)),
        }
    }
}

impl Request {
    /// Make a `GET` request to `url`.
    pub fn get(url: Url) -> Self {
        Self::new(Method::GET, url)
    }

    /// Make a `POST` request to `url`.
    pub fn post(url: Url) -> Self {
        Self::new(Method::POST, url)
    }

    pub fn new(method: Method, url: Url) -> Self {
        Self {
            method,
            url,
            headers: HeaderMap::new(),
            body: None,
        }
    }

    /// Set a header. This function panics if `value` is not a valid header
    /// value.
    pub fn header(mut self, name: impl http::header::IntoHeaderName, value: &str) -> Self {
        let value = value
            .parse()
            .expect("header value contains invalid characters");
        self.headers.insert(name, value);
        self
    }

    /// Set a header. If `value` is not valid, this function returns an
    /// error and does not panic.
    pub fn try_header(
        mut self,
        name: impl http::header::IntoHeaderName,
        value: &str,
    ) -> Result<Self, InvalidHeaderValue> {
        let value = value.parse()?;
        self.headers.insert(name, value);
        Ok(self)
    }

    pub fn body(mut self, body: impl Into<Bytes>) -> Self {
        self.body = Some(body.into());
        self
    }
}