ds-http-client 0.1.0

HTTP client to download files or query API with User-Agent set
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

//! # HTTP client
//!
//! High-level interface to Hyper/reqwest HTTP client.
//!
//! This library is optimized to work with Nasqueron Datasources components.

use std::collections::HashMap;
use std::io::Error as IOError;
use std::path::Path;

use lazy_static::lazy_static;
use reqwest::{Client as ReqwestClient, RequestBuilder};
use reqwest::ClientBuilder;
use reqwest::Error as ReqwestError;
use reqwest::IntoUrl;
use reqwest::Response;
use reqwest::header::{HeaderMap, HeaderValue};
use tokio::fs::File;
use tokio::io::AsyncWriteExt;

/*   -------------------------------------------------------------
     User agent

     The USER_AGENT variable is computed at build time.
     - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

lazy_static! {
    pub static ref USER_AGENT: String = format!(
        "{}/{}",
        env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")
    );
}

/// Gets the default user agent
pub fn get_user_agent () -> &'static str {
    &USER_AGENT
}

/*   -------------------------------------------------------------
     HTTP client
     - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

/// HTTP client
pub struct Client {
    client: ReqwestClient,
}

impl Client {
    pub fn new(headers: Option<HeaderMap>) -> Self {
        let client = ClientBuilder::new()
            .default_headers(build_default_headers(headers))
            .gzip(true)
            .deflate(true)
            .build()
            .expect("Can't build HTTP client");

        Self {
            client,
        }
    }

    pub async fn get<T>(&self, url: T) -> Result<Response, Error>
    where T: IntoUrl {
        let request = self.client.get(url);
        self.run(request).await
    }

    pub async fn get_with_headers<T>(&self, url: T, headers: HashMap<String, String>) -> Result<Response, Error>
    where T: IntoUrl {
        let headers = parse_headers(headers);

        let request = self.client
            .get(url)
            .headers(headers);

        self.run(request).await
    }

    pub async fn run(&self, request: RequestBuilder) -> Result<Response, Error> {
        request
            .send()
            .await
            .map_err(|error| Error::Reqwest(error))
    }

    pub async fn download<P, T>(&self, url: T, target_path: P) -> Result<usize, Error>
    where T: IntoUrl, P: AsRef<Path> {
        let mut file =  File::create(target_path)
            .await
            .map_err(|error| Error::IO(error))?;

        let mut target_content = self.get(url).await?;
        let mut bytes_read = 0;
        while let Some(chunk) = target_content
            .chunk()
            .await
            .map_err(|error| Error::Reqwest(error))?
        {
            bytes_read += file.write(chunk.as_ref())
                .await
                .map_err(|error| Error::IO(error))?;
        }

        Ok(bytes_read)
    }
}

/*   -------------------------------------------------------------
     HTTP client utilities
     - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

pub fn parse_headers(headers: HashMap<String, String>) -> HeaderMap {
    headers
        .iter()
        .map(|(name, value)| (
            name.parse().expect("Can't parse header name"),
            value.parse().expect("Can't parse header value")
        ))
        .collect()
}

fn build_default_headers(headers: Option<HeaderMap>) -> HeaderMap {
    let mut headers = headers
        .unwrap_or(HeaderMap::new());

    // RFC 7231 states User-Agent header SHOULD be sent.
    if !headers.contains_key("User-Agent") {
        headers.append("User-Agent", HeaderValue::from_static(get_user_agent()));
    }

    headers
}

/*   -------------------------------------------------------------
     HTTP client error
     - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

/// HTTP client error
#[derive(Debug)]
pub enum Error {
    /// Represents an underlying error from Reqwest HTTP client when processing a request.
    Reqwest(ReqwestError),

    /// Represents an IO error when doing file operations.
    IO(IOError),
}