minreq/
request.rs

1use crate::connection::Connection;
2use crate::http_url::{HttpUrl, Port};
3#[cfg(feature = "proxy")]
4use crate::proxy::Proxy;
5use crate::{Error, Response, ResponseLazy};
6use std::collections::HashMap;
7use std::fmt;
8use std::fmt::Write;
9use tokio::io::{AsyncReadExt, AsyncWriteExt};
10/// A URL type for requests.
11pub type URL = String;
12
13/// An HTTP request method.
14#[derive(Clone, PartialEq, Eq, Debug)]
15pub enum Method {
16    /// The GET method
17    Get,
18    /// The HEAD method
19    Head,
20    /// The POST method
21    Post,
22    /// The PUT method
23    Put,
24    /// The DELETE method
25    Delete,
26    /// The CONNECT method
27    Connect,
28    /// The OPTIONS method
29    Options,
30    /// The TRACE method
31    Trace,
32    /// The PATCH method
33    Patch,
34    /// A custom method, use with care: the string will be embedded in
35    /// your request as-is.
36    Custom(String),
37}
38
39impl fmt::Display for Method {
40    /// Formats the Method to the form in the HTTP request,
41    /// ie. Method::Get -> "GET", Method::Post -> "POST", etc.
42    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
43        match *self {
44            Method::Get => write!(f, "GET"),
45            Method::Head => write!(f, "HEAD"),
46            Method::Post => write!(f, "POST"),
47            Method::Put => write!(f, "PUT"),
48            Method::Delete => write!(f, "DELETE"),
49            Method::Connect => write!(f, "CONNECT"),
50            Method::Options => write!(f, "OPTIONS"),
51            Method::Trace => write!(f, "TRACE"),
52            Method::Patch => write!(f, "PATCH"),
53            Method::Custom(ref s) => write!(f, "{}", s),
54        }
55    }
56}
57
58/// An HTTP request.
59///
60/// Generally created by the [`minreq::get`](fn.get.html)-style
61/// functions, corresponding to the HTTP method we want to use.
62///
63/// # Example
64///
65/// ```
66/// let request = minreq::post("http://example.com");
67/// ```
68///
69/// After creating the request, you would generally call
70/// [`send`](struct.Request.html#method.send) or
71/// [`send_lazy`](struct.Request.html#method.send_lazy) on it, as it
72/// doesn't do much on its own.
73#[derive(Clone, PartialEq, Eq, Debug)]
74pub struct Request {
75    pub(crate) method: Method,
76    url: URL,
77    params: String,
78    pub headers: HashMap<String, String>,
79    body: Option<Vec<u8>>,
80    pub(crate) max_headers_size: Option<usize>,
81    pub(crate) max_status_line_len: Option<usize>,
82    max_redirects: usize,
83    pub redirect: bool,
84    #[cfg(feature = "proxy")]
85    pub(crate) proxy: Option<Proxy>,
86}
87
88impl Request {
89    /// Creates a new HTTP `Request`.
90    ///
91    /// This is only the request's data, it is not sent yet. For
92    /// sending the request, see [`send`](struct.Request.html#method.send).
93    ///
94    /// If `urlencoding` is not enabled, it is the responsibility of the
95    /// user to ensure there are no illegal characters in the URL.
96    ///
97    /// If `urlencoding` is enabled, the resource part of the URL will be
98    /// encoded. Any URL special characters (e.g. &, #, =) are not encoded
99    /// as they are assumed to be meaningful parameters etc.
100    pub fn new<T: Into<URL>>(method: Method, url: T) -> Request {
101        let mut headers = HashMap::new();
102        headers.insert("user-agent".to_owned(), "minreq/1.0".to_owned());
103        headers.insert("accept".to_owned(), "*/*".to_owned());
104        Request {
105            method,
106            url: url.into(),
107            params: String::new(),
108            headers,
109            body: None,
110            max_headers_size: None,
111            max_status_line_len: None,
112            max_redirects: 100,
113            redirect: false,
114            #[cfg(feature = "proxy")]
115            proxy: None,
116        }
117    }
118
119    /// Add headers to the request this is called on. Use this
120    /// function to add headers to your requests.
121    pub fn with_headers<T, K, V>(mut self, headers: T) -> Request
122    where
123        T: IntoIterator<Item = (K, V)>,
124        K: Into<String>,
125        V: Into<String>,
126    {
127        let headers = headers.into_iter().map(|(k, v)| (k.into(), v.into()));
128        self.headers.extend(headers);
129        self
130    }
131
132    /// Adds a header to the request this is called on. Use this
133    /// function to add headers to your requests.
134    pub fn with_header<T: Into<String>, U: Into<String>>(mut self, key: T, value: U) -> Request {
135        self.headers.insert(key.into(), value.into());
136        self
137    }
138    pub fn with_redirect(mut self, follow: bool) -> Request {
139        self.redirect = follow;
140        self
141    }
142    /// Sets the request body.
143    pub fn with_body<T: Into<Vec<u8>>>(mut self, body: T) -> Request {
144        let body = body.into();
145        let body_length = body.len();
146        self.body = Some(body);
147        self.with_header("Content-Length", format!("{}", body_length))
148    }
149
150    /// Adds given key and value as query parameter to request url
151    /// (resource).
152    ///
153    /// If `urlencoding` is not enabled, it is the responsibility
154    /// of the user to ensure there are no illegal characters in the
155    /// key or value.
156    ///
157    /// If `urlencoding` is enabled, the key and value are both encoded.
158    pub fn with_param<T: Into<String>, U: Into<String>>(mut self, key: T, value: U) -> Request {
159        let key = key.into();
160        let key = urlencoding::encode(&key);
161        let value = value.into();
162        let value = urlencoding::encode(&value);
163
164        if !self.params.is_empty() {
165            self.params.push('&');
166        }
167        self.params.push_str(&key);
168        self.params.push('=');
169        self.params.push_str(&value);
170        self
171    }
172
173    /// Sets the max redirects we follow until giving up. 100 by
174    /// default.
175    ///
176    /// Warning: setting this to a very high number, such as 1000, may
177    /// cause a stack overflow if that many redirects are followed. If
178    /// you have a use for so many redirects that the stack overflow
179    /// becomes a problem, please open an issue.
180    pub fn with_max_redirects(mut self, max_redirects: usize) -> Request {
181        self.max_redirects = max_redirects;
182        self
183    }
184
185    /// Sets the maximum size of all the headers this request will
186    /// accept.
187    ///
188    /// If this limit is passed, the request will close the connection
189    /// and return an [Error::HeadersOverflow] error.
190    ///
191    /// The maximum length is counted in bytes, including line-endings
192    /// and other whitespace. Both normal and trailing headers count
193    /// towards this cap.
194    ///
195    /// `None` disables the cap, and may cause the program to use any
196    /// amount of memory if the server responds with a lot of headers
197    /// (or an infinite amount). In minreq versions 2.x.x, the default
198    /// is None, so setting this manually is recommended when talking
199    /// to untrusted servers.
200    pub fn with_max_headers_size<S: Into<Option<usize>>>(mut self, max_headers_size: S) -> Request {
201        self.max_headers_size = max_headers_size.into();
202        self
203    }
204
205    /// Sets the maximum length of the status line this request will
206    /// accept.
207    ///
208    /// If this limit is passed, the request will close the connection
209    /// and return an [Error::StatusLineOverflow] error.
210    ///
211    /// The maximum length is counted in bytes, including the
212    /// line-ending `\r\n`.
213    ///
214    /// `None` disables the cap, and may cause the program to use any
215    /// amount of memory if the server responds with a long (or
216    /// infinite) status line. In minreq versions 2.x.x, the default
217    /// is None, so setting this manually is recommended when talking
218    /// to untrusted servers.
219    pub fn with_max_status_line_length<S: Into<Option<usize>>>(
220        mut self,
221        max_status_line_len: S,
222    ) -> Request {
223        self.max_status_line_len = max_status_line_len.into();
224        self
225    }
226
227    /// Sets the proxy to use.
228    #[cfg(feature = "proxy")]
229    pub fn with_proxy(mut self, proxy: Proxy) -> Request {
230        self.proxy = Some(proxy);
231        self
232    }
233
234    /// Sends this request to the host.
235    ///
236    /// # Errors
237    ///
238    /// Returns `Err` if we run into an error while sending the
239    /// request, or receiving/parsing the response. The specific error
240    /// is described in the `Err`, and it can be any
241    /// [`minreq::Error`](enum.Error.html) except
242    /// [`SerdeJsonError`](enum.Error.html#variant.SerdeJsonError) and
243    /// [`InvalidUtf8InBody`](enum.Error.html#variant.InvalidUtf8InBody).
244    pub async fn send(self) -> Result<Response, Error> {
245        let parsed_request = ParsedRequest::new(self)?;
246        let is_head = parsed_request.config.method == Method::Head;
247        if parsed_request.url.https {
248            #[cfg(feature = "https")]
249            let response = Connection::new(parsed_request).send_https().await?;
250            #[cfg(feature = "https")]
251            return Response::create(response, is_head).await;
252            #[cfg(not(feature = "https"))]
253            return Err(Error::HttpsFeatureNotEnabled);
254        } else {
255            let response = Connection::new(parsed_request).send().await?;
256            Response::create(response, is_head).await
257        }
258    }
259
260    /// Sends this request to the host, loaded lazily.
261    ///
262    /// # Errors
263    ///
264    /// See [`send`](struct.Request.html#method.send).
265    pub async fn send_lazy(self) -> Result<ResponseLazy, Error> {
266        let parsed_request = ParsedRequest::new(self)?;
267        if parsed_request.url.https {
268            #[cfg(feature = "https")]
269            return Connection::new(parsed_request).send_https().await;
270            #[cfg(not(feature = "https"))]
271            return Err(Error::HttpsFeatureNotEnabled);
272        } else {
273            Connection::new(parsed_request).send().await
274        }
275    }
276    pub async fn send_with_stream<W, F>(
277        self,
278        stream: &mut W,
279        progress_fn: F,
280    ) -> Result<Response, Error>
281    where
282        F: Fn(u64, u64),
283        W: tokio::io::AsyncWrite + std::marker::Unpin,
284    {
285        let mut response = self.send_lazy().await?;
286        let default_content_lenth = String::from("0");
287        let content_lenth = response
288            .headers
289            .get("content-length")
290            .unwrap_or(&default_content_lenth);
291        let content_lenth = content_lenth.trim();
292        let content_lenth =
293            u64::from_str_radix(content_lenth, 10).map_err(|_| Error::MalformedContentLength)?;
294        let total_download =
295            copy_with_progress(&mut response.stream, stream, content_lenth, progress_fn).await?;
296        let mut response = Response::create(response, true).await?;
297        response.download_size = total_download;
298        Ok(response)
299    }
300}
301
302#[derive(Debug)]
303pub(crate) struct ParsedRequest {
304    pub(crate) url: HttpUrl,
305    pub(crate) redirects: Vec<HttpUrl>,
306    pub(crate) config: Request,
307}
308
309impl ParsedRequest {
310    #[allow(unused_mut)]
311    fn new(mut config: Request) -> Result<ParsedRequest, Error> {
312        let mut url = HttpUrl::parse(&config.url, None)?;
313
314        if !config.params.is_empty() {
315            if url.path_and_query.contains('?') {
316                url.path_and_query.push('&');
317            } else {
318                url.path_and_query.push('?');
319            }
320            url.path_and_query.push_str(&config.params);
321        }
322
323        #[cfg(feature = "proxy")]
324        // Set default proxy from environment variables
325        //
326        // Curl documentation: https://everything.curl.dev/usingcurl/proxies/env
327        //
328        // Accepted variables are `http_proxy`, `https_proxy`, `HTTPS_PROXY`, `ALL_PROXY`
329        //
330        // Note: https://everything.curl.dev/usingcurl/proxies/env#http_proxy-in-lower-case-only
331        if config.proxy.is_none() {
332            // Set HTTP proxies if request's protocol is HTTPS and they're given
333            if url.https {
334                if let Ok(proxy) =
335                    std::env::var("https_proxy").map_err(|_| std::env::var("HTTPS_PROXY"))
336                {
337                    if let Ok(proxy) = Proxy::new(proxy) {
338                        config.proxy = Some(proxy);
339                    }
340                }
341            }
342            // Set HTTP proxies if request's protocol is HTTP and they're given
343            else if let Ok(proxy) = std::env::var("http_proxy") {
344                if let Ok(proxy) = Proxy::new(proxy) {
345                    config.proxy = Some(proxy);
346                }
347            }
348            // Set any given proxies if neither of HTTP/HTTPS were given
349            else if let Ok(proxy) =
350                std::env::var("all_proxy").map_err(|_| std::env::var("ALL_PROXY"))
351            {
352                if let Ok(proxy) = Proxy::new(proxy) {
353                    config.proxy = Some(proxy);
354                }
355            }
356        }
357
358        Ok(ParsedRequest {
359            url,
360            redirects: Vec::new(),
361            config,
362        })
363    }
364
365    fn get_http_head(&self) -> String {
366        let mut http = String::with_capacity(32);
367
368        // NOTE: As of 2.10.0, the fragment is intentionally left out of the request, based on:
369        // - [RFC 3986 section 3.5](https://datatracker.ietf.org/doc/html/rfc3986#section-3.5):
370        //   "...the fragment identifier is not used in the scheme-specific
371        //   processing of a URI; instead, the fragment identifier is separated
372        //   from the rest of the URI prior to a dereference..."
373        // - [RFC 7231 section 9.5](https://datatracker.ietf.org/doc/html/rfc7231#section-9.5):
374        //   "Although fragment identifiers used within URI references are not
375        //   sent in requests..."
376
377        // Add the request line and the "Host" header
378        write!(
379            http,
380            "{} {} HTTP/1.1\r\nHost: {}",
381            self.config.method, self.url.path_and_query, self.url.host
382        )
383        .unwrap();
384        if let Port::Explicit(port) = self.url.port {
385            write!(http, ":{}", port).unwrap();
386        }
387        http += "\r\n";
388
389        // Add other headers
390        for (k, v) in &self.config.headers {
391            write!(http, "{}: {}\r\n", k, v).unwrap();
392        }
393        if self.config.method == Method::Post
394            || self.config.method == Method::Put
395            || self.config.method == Method::Patch
396        {
397            let not_length = |key: &String| {
398                let key = key.to_lowercase();
399                key != "content-length" && key != "transfer-encoding"
400            };
401            if self.config.headers.keys().all(not_length) {
402                // A user agent SHOULD send a Content-Length in a request message when no Transfer-Encoding
403                // is sent and the request method defines a meaning for an enclosed payload body.
404                // refer: https://tools.ietf.org/html/rfc7230#section-3.3.2
405
406                // A client MUST NOT send a message body in a TRACE request.
407                // refer: https://tools.ietf.org/html/rfc7231#section-4.3.8
408                // similar line found for GET, HEAD, CONNECT and DELETE.
409
410                http += "Content-Length: 0\r\n";
411            }
412        }
413        http += "\r\n";
414        http
415    }
416
417    /// Returns the HTTP request as bytes, ready to be sent to
418    /// the server.
419    pub(crate) fn as_bytes(&self) -> Vec<u8> {
420        let mut head = self.get_http_head().into_bytes();
421        if let Some(body) = &self.config.body {
422            head.extend(body);
423        }
424        head
425    }
426
427    /// Returns the redirected version of this Request, unless an
428    /// infinite redirection loop was detected, or the redirection
429    /// limit was reached.
430    pub(crate) fn redirect_to(&mut self, url: &str) -> Result<(), Error> {
431        if url.contains("://") {
432            let mut url = HttpUrl::parse(url, Some(&self.url)).map_err(|_| {
433                // TODO: Uncomment this for 3.0
434                // Error::InvalidProtocolInRedirect
435                Error::IoError(std::io::Error::new(
436                    std::io::ErrorKind::Other,
437                    "was redirected to an absolute url with an invalid protocol",
438                ))
439            })?;
440            log::trace!("Redirect to absolute url: {:?}", url);
441            std::mem::swap(&mut url, &mut self.url);
442            self.redirects.push(url);
443        } else {
444            // The url does not have the protocol part, assuming it's
445            // a relative resource.
446            log::trace!("Redirect to relatively url: {:?}", url);
447            let mut absolute_url = String::new();
448            self.url.write_base_url_to(&mut absolute_url).unwrap();
449            absolute_url.push_str(url);
450            let mut url = HttpUrl::parse(&absolute_url, Some(&self.url))?;
451            std::mem::swap(&mut url, &mut self.url);
452            self.redirects.push(url);
453        }
454
455        if self.redirects.len() > self.config.max_redirects {
456            Err(Error::TooManyRedirections)
457        } else if self
458            .redirects
459            .iter()
460            .any(|redirect_url| redirect_url == &self.url)
461        {
462            Err(Error::InfiniteRedirectionLoop)
463        } else {
464            Ok(())
465        }
466    }
467}
468
469/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to
470/// [Method::Get](enum.Method.html).
471pub fn get<T: Into<URL>>(url: T) -> Request {
472    Request::new(Method::Get, url)
473}
474
475/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to
476/// [Method::Head](enum.Method.html).
477pub fn head<T: Into<URL>>(url: T) -> Request {
478    Request::new(Method::Head, url)
479}
480
481/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to
482/// [Method::Post](enum.Method.html).
483pub fn post<T: Into<URL>>(url: T) -> Request {
484    Request::new(Method::Post, url)
485}
486
487/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to
488/// [Method::Put](enum.Method.html).
489pub fn put<T: Into<URL>>(url: T) -> Request {
490    Request::new(Method::Put, url)
491}
492
493/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to
494/// [Method::Delete](enum.Method.html).
495pub fn delete<T: Into<URL>>(url: T) -> Request {
496    Request::new(Method::Delete, url)
497}
498
499/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to
500/// [Method::Connect](enum.Method.html).
501pub fn connect<T: Into<URL>>(url: T) -> Request {
502    Request::new(Method::Connect, url)
503}
504
505/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to
506/// [Method::Options](enum.Method.html).
507pub fn options<T: Into<URL>>(url: T) -> Request {
508    Request::new(Method::Options, url)
509}
510
511/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to
512/// [Method::Trace](enum.Method.html).
513pub fn trace<T: Into<URL>>(url: T) -> Request {
514    Request::new(Method::Trace, url)
515}
516
517/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to
518/// [Method::Patch](enum.Method.html).
519pub fn patch<T: Into<URL>>(url: T) -> Request {
520    Request::new(Method::Patch, url)
521}
522
523const DOWNLOAD_BUFFER_SIZEE: usize = 4096;
524async fn copy_with_progress<R: ?Sized, W: ?Sized>(
525    reader: &mut R,
526    writer: &mut W,
527    total: u64,
528    f: impl Fn(u64, u64),
529) -> Result<u64, std::io::Error>
530where
531    R: tokio::io::AsyncRead + std::marker::Unpin,
532    W: tokio::io::AsyncWrite + std::marker::Unpin,
533{
534    let mut buffer = vec![0u8; DOWNLOAD_BUFFER_SIZEE];
535    let mut total_len = 0u64;
536    while let Ok(len) = reader.read_exact(&mut buffer).await {
537        writer.write_all(&buffer[0..len]).await?;
538        total_len += len as u64;
539        f(total, total_len);
540        writer.write_all(&buffer[0..len]).await?;
541        if len == 0 || total_len == total || len < DOWNLOAD_BUFFER_SIZEE {
542            break;
543        }
544    }
545    Ok(total_len)
546}
547
548
549#[cfg(test)]
550mod parsing_tests {
551
552    use std::collections::HashMap;
553
554    use super::{ParsedRequest, get};
555
556    #[test]
557    fn test_headers() {
558        let mut headers = HashMap::new();
559        headers.insert("foo".to_string(), "bar".to_string());
560        headers.insert("foo".to_string(), "baz".to_string());
561
562        let req = get("http://www.example.org/test/res").with_headers(headers.clone());
563
564        assert_eq!(req.headers, headers);
565    }
566
567    #[test]
568    fn test_multiple_params() {
569        let req = get("http://www.example.org/test/res")
570            .with_param("foo", "bar")
571            .with_param("asd", "qwe");
572        let req = ParsedRequest::new(req).unwrap();
573        assert_eq!(&req.url.path_and_query, "/test/res?foo=bar&asd=qwe");
574    }
575
576    #[test]
577    fn test_domain() {
578        let req = get("http://www.example.org/test/res").with_param("foo", "bar");
579        let req = ParsedRequest::new(req).unwrap();
580        assert_eq!(&req.url.host, "www.example.org");
581    }
582
583    #[test]
584    fn test_protocol() {
585        let req =
586            ParsedRequest::new(get("http://www.example.org/").with_param("foo", "bar")).unwrap();
587        assert!(!req.url.https);
588        let req =
589            ParsedRequest::new(get("https://www.example.org/").with_param("foo", "bar")).unwrap();
590        assert!(req.url.https);
591    }
592}
593
594#[cfg(all(test))]
595mod encoding_tests {
596    use super::{ParsedRequest, get};
597
598    #[test]
599    fn test_with_param() {
600        let req = get("http://www.example.org").with_param("foo", "bar");
601        let req = ParsedRequest::new(req).unwrap();
602        assert_eq!(&req.url.path_and_query, "/?foo=bar");
603
604        let req = get("http://www.example.org").with_param("ówò", "what's this? 👀");
605        let req = ParsedRequest::new(req).unwrap();
606        assert_eq!(
607            &req.url.path_and_query,
608            "/?%C3%B3w%C3%B2=what%27s%20this%3F%20%F0%9F%91%80"
609        );
610    }
611
612    #[test]
613    fn test_on_creation() {
614        let req = ParsedRequest::new(get("http://www.example.org/?foo=bar#baz")).unwrap();
615        assert_eq!(&req.url.path_and_query, "/?foo=bar");
616
617        let req = ParsedRequest::new(get("http://www.example.org/?ówò=what's this? 👀")).unwrap();
618        assert_eq!(
619            &req.url.path_and_query,
620            "/?%C3%B3w%C3%B2=what%27s%20this?%20%F0%9F%91%80"
621        );
622    }
623}
minreq/request.rs

minreq/
request.rs
