url_cleaner_engine/glue/
http_client_config.rs

1//! Rules for how to make a [`reqwest::blocking::Client`].
2
3use std::collections::HashSet;
4
5use serde::{Serialize, Deserialize};
6#[cfg(feature = "http")]
7use reqwest::header::HeaderMap;
8
9#[expect(unused_imports, reason = "Used in docs.")]
10use crate::types::*;
11use crate::glue::*;
12use crate::util::*;
13
14/// Rules for how to make a [`reqwest::blocking::Client`].
15#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Suitability)]
16#[serde(deny_unknown_fields)]
17pub struct HttpClientConfig {
18    /// The headers to send by default.
19    #[serde(default, skip_serializing_if = "is_default", with = "serde_headermap")]
20    pub default_headers: HeaderMap,
21    /// The redirect policy.
22    ///
23    /// Somewhat nuanced so check [`RedirectPolicy`]'s docs.
24    #[serde(default, skip_serializing_if = "is_default")]
25    pub redirect_policy: RedirectPolicy,
26    /// The value passed to [`reqwest::blocking::ClientBuilder::https_only`].
27    ///
28    /// Defaults to [`false`].
29    #[serde(default, skip_serializing_if = "is_default")]
30    pub https_only: bool,
31    /// The value passed to [`reqwest::blocking::ClientBuilder::referer`].
32    ///
33    /// Defaults to [`false`] and frankly there's no legitimate reason for the header to exist or for you to turn it on.
34    #[serde(default, skip_serializing_if = "is_default")]
35    pub referer: bool,
36    /// Proxies to use.
37    ///
38    /// All proxies supported by [`reqwest`] should always be supported, but if I missed anything let me know.
39    #[serde(default, skip_serializing_if = "is_default")]
40    pub proxies: Vec<ProxyConfig>,
41    /// The value passed to [`reqwest::blocking::ClientBuilder::no_proxy`].
42    ///
43    /// Defaults to [`false`].
44    #[serde(default, skip_serializing_if = "is_default")]
45    pub no_proxy: bool,
46    /// Extra PEM encoded TLS certificates to trust.
47    ///
48    /// See [`reqwest::blocking::ClientBuilder::add_root_certificate`] and [`reqwest::tls::Certificate::from_pem`] for details.
49    ///
50    /// Defaults to an empty list.
51    #[serde(default, skip_serializing_if = "is_default")]
52    pub extra_root_certificates: HashSet<String>
53}
54
55/// The policy on how to handle [HTTP redirects](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Redirections).
56///
57/// Defaults to [`Self::Limited`] with a value of `10`, as that's what reqwest does.
58///
59/// For the default config (and all real use) it's recommended to use [`Self::None`] in a [`Action::Repeat`].
60///
61/// That has the added benefit of not sending a request to the final URL.
62#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Suitability)]
63#[serde(deny_unknown_fields)]
64pub enum RedirectPolicy {
65    /// If a request encounters [`Self::Limited::0`] redirects, the entire request fails.
66    ///
67    /// So if your policy is `RedirectPolicy::Limited(0)`, any redirects at all will return an error.
68    Limited(usize),
69    /// Don't follow redirects and instead return the page doing the redirecting.
70    None
71}
72
73impl Default for RedirectPolicy {
74    /// [`Self::Limited`] with a value of `10`, as that's what reqwest does.
75    fn default() -> Self {
76        Self::Limited(10)
77    }
78}
79
80impl From<RedirectPolicy> for reqwest::redirect::Policy {
81    fn from(value: RedirectPolicy) -> Self {
82        match value {
83            RedirectPolicy::Limited(x) => Self::limited(x),
84            RedirectPolicy::None => Self::none()
85        }
86    }
87}
88
89impl HttpClientConfig {
90    /// Makes a [`reqwest::blocking::Client`].
91    /// # Errors
92    /// If a call to [`ProxyConfig::make`] returns an error, that error is returned.
93    ///
94    /// If the call to [`reqwest::blocking::ClientBuilder::build`] returns an error, that error is returned.
95    pub fn make(&self) -> reqwest::Result<reqwest::blocking::Client> {
96        let mut temp = reqwest::blocking::Client::builder().default_headers(self.default_headers.clone())
97            .redirect(self.redirect_policy.clone().into())
98            .https_only(self.https_only)
99            .referer(self.referer);
100        for proxy in &self.proxies {
101            temp = temp.proxy(proxy.clone().make()?);
102        }
103        if self.no_proxy {temp = temp.no_proxy();}
104        for cert in &self.extra_root_certificates {
105            temp = temp.add_root_certificate(reqwest::tls::Certificate::from_pem(cert.as_bytes())?);
106        }
107        temp.build()
108    }
109}
110
111/// Rules for updating a [`HttpClientConfig`].
112#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize, Suitability)]
113#[serde(deny_unknown_fields)]
114pub struct HttpClientConfigDiff {
115    /// If [`Some`], overwrites [`HttpClientConfig::redirect_policy`].
116    #[serde(default, skip_serializing_if = "is_default")]
117    pub redirect_policy: Option<RedirectPolicy>,
118    /// Appends each header into [`HttpClientConfig::default_headers`].
119    #[serde(default, skip_serializing_if = "is_default", with = "serde_headermap")]
120    pub add_default_headers: HeaderMap,
121    /// If [`Some`], overwrites [`HttpClientConfig::https_only`].
122    #[serde(default, skip_serializing_if = "is_default")]
123    pub https_only: Option<bool>,
124    /// If [`Some`], overwrites [`HttpClientConfig::proxies`].
125    #[serde(default, skip_serializing_if = "is_default")]
126    pub set_proxies: Option<Vec<ProxyConfig>>,
127    /// Appends each [`ProxyConfig`] to [`HttpClientConfig::proxies`].
128    #[serde(default, skip_serializing_if = "is_default")]
129    pub add_proxies: Vec<ProxyConfig>,
130    /// If [`Some`], overwrites [`HttpClientConfig::no_proxy`].
131    #[serde(default, skip_serializing_if = "is_default")]
132    pub no_proxy: Option<bool>,
133    /// If [`Some`], overwrites [`HttpClientConfig::referer`].
134    #[serde(default, skip_serializing_if = "is_default")]
135    pub referer: Option<bool>,
136    /// Adds to [`HttpClientConfig::extra_root_certificates`].
137    #[serde(default, skip_serializing_if = "is_default")]
138    pub add_extra_root_certificates: HashSet<String>,
139    /// Removes from [`HttpClientConfig::extra_root_certificates`].
140    #[serde(default, skip_serializing_if = "is_default")]
141    pub remove_extra_root_certificates: HashSet<String>
142}
143
144impl HttpClientConfigDiff {
145    /// Applies the diff.
146    ///
147    /// If you want to apply `self` multiple times, use [`Self::apply_multiple`] as it's slightly faster than [`Clone::clone`]ing this then using [`Self::apply_once`] on each clone.
148    pub fn apply_once(self, to: &mut HttpClientConfig) {
149        debug!(HttpClientConfigDiff::apply_once, &self, to);
150        if let Some(new_redirect_policy) = self.redirect_policy {to.redirect_policy = new_redirect_policy;}
151        to.default_headers.extend(self.add_default_headers);
152        if let Some(https_only) = self.https_only {to.https_only = https_only;}
153        if let Some(set_proxies) = self.set_proxies {to.proxies = set_proxies;}
154        to.proxies.extend(self.add_proxies);
155        if let Some(no_proxy) = self.no_proxy {to.no_proxy = no_proxy;}
156        if let Some(referer) = self.referer {to.no_proxy = referer;}
157        to.extra_root_certificates.extend(self.add_extra_root_certificates);
158        to.extra_root_certificates.retain(|extra_root_certificate| !self.remove_extra_root_certificates.contains(extra_root_certificate));
159    }
160
161    /// Applies the diff.
162    ///
163    /// If you only want to apply `self` once, use [`Self::apply_once`].
164    pub fn apply_multiple(&self, to: &mut HttpClientConfig) {
165        debug!(HttpClientConfigDiff::apply_multiple, self, to);
166        if let Some(new_redirect_policy) = &self.redirect_policy {to.redirect_policy = new_redirect_policy.clone();}
167        to.default_headers.extend(self.add_default_headers.clone());
168        if let Some(https_only) = self.https_only {to.https_only = https_only;}
169        if let Some(set_proxies) = &self.set_proxies {to.proxies.clone_from(set_proxies);}
170        to.proxies.extend(self.add_proxies.clone());
171        if let Some(no_proxy) = self.no_proxy {to.no_proxy = no_proxy;}
172        if let Some(referer) = self.referer {to.no_proxy = referer;}
173        to.extra_root_certificates.extend(self.add_extra_root_certificates.clone());
174        to.extra_root_certificates.retain(|extra_root_certificate| !self.remove_extra_root_certificates.contains(extra_root_certificate));
175    }
176}
url_cleaner_engine/glue/http_client_config.rs

url_cleaner_engine/glue/
http_client_config.rs
