kodik_utils/
client_ext.rs

1use reqwest::{
2    Client, RequestBuilder, Response, StatusCode,
3    header::{ACCEPT, ACCEPT_LANGUAGE, HeaderMap, HeaderName, HeaderValue, USER_AGENT},
4};
5use serde::{Serialize, de::DeserializeOwned};
6use std::{fmt::Debug, future::Future, time::Duration};
7use tokio::time;
8use ua_generator::{
9    fastrand::{self, Rng},
10    ua,
11};
12
13pub trait ClientExt {
14    /// Posts data to the given URL and deserializes the response as JSON.
15    ///
16    /// # Errors
17    ///
18    /// Returns an [`Error`] if:
19    /// - A network request fails.
20    /// - The response cannot be deserialized into the target type `T`.
21    /// - An invalid URL is provided (though `reqwest` usually handles this during `post`).
22    fn post_form_as_json<T, F>(&self, url: &str, form: &F) -> impl Future<Output = crate::Result<T>> + Send
23    where
24        T: DeserializeOwned + Debug,
25        F: Serialize + Sync + ?Sized;
26
27    /// Posts JSON data to the given URL and deserializes the response as JSON.
28    ///
29    /// # Errors
30    ///
31    /// Returns an [`Error`] if:
32    /// - A network request fails.
33    /// - The response cannot be deserialized into the target type `T`.
34    /// - An invalid URL is provided (though `reqwest` usually handles this during `post`).
35    fn post_json_as_json<T, J>(&self, url: &str, json: &J) -> impl Future<Output = crate::Result<T>> + Send
36    where
37        T: DeserializeOwned + Debug,
38        J: Serialize + Sync + ?Sized;
39
40    /// Posts JSON data to the given URL returns the response body as a string.
41    ///
42    /// # Errors
43    ///
44    /// Returns an [`Error`] if:
45    /// - A network request fails.
46    /// - An invalid URL is provided (though `reqwest` usually handles this during `post`).
47    fn post_json_as_text<J>(&self, url: &str, json: &J) -> impl Future<Output = crate::Result<String>> + Send
48    where
49        J: Serialize + Sync + ?Sized;
50
51    /// Fetches data from the given URL and returns the response body as a string.
52    ///
53    /// # Errors
54    ///
55    /// Returns an [`Error`] if:
56    /// - A network request fails.
57    /// - The response body cannot be read as a string.
58    /// - An invalid URL is provided (though `reqwest` usually handles this during `get`).
59    fn fetch_as_text(&self, url: &str) -> impl Future<Output = crate::Result<String>> + Send;
60
61    /// Fetches data from the given URL and deserializes it as JSON.
62    ///
63    /// # Errors
64    ///
65    /// Returns an [`Error`] if:
66    /// - A network request fails.
67    /// - The response cannot be deserialized into the target type `T`.
68    /// - An invalid URL is provided (though `reqwest` usually handles this during `get`).
69    fn fetch_as_json<T: DeserializeOwned + Debug>(&self, url: &str) -> impl Future<Output = crate::Result<T>> + Send;
70
71    fn patch_json_as_json<T, J>(&self, url: &str, json: &J) -> impl Future<Output = crate::Result<T>> + Send
72    where
73        T: DeserializeOwned + Debug,
74        J: Serialize + Sync + ?Sized;
75
76    fn patch_json_as_text<J>(&self, url: &str, json: &J) -> impl Future<Output = crate::Result<String>> + Send
77    where
78        J: Serialize + Sync + ?Sized;
79}
80
81impl ClientExt for Client {
82    async fn post_form_as_json<T, F>(&self, url: &str, form: &F) -> crate::Result<T>
83    where
84        T: DeserializeOwned + Debug,
85        F: Serialize + Sync + ?Sized,
86    {
87        log::info!("POST to {url}...");
88        execute_json(self.post(url).form(form)).await
89    }
90
91    async fn post_json_as_json<T, J>(&self, url: &str, json: &J) -> crate::Result<T>
92    where
93        T: DeserializeOwned + Debug,
94        J: Serialize + Sync + ?Sized,
95    {
96        log::info!("POST to {url}...");
97        execute_json(self.post(url).json(json)).await
98    }
99
100    async fn post_json_as_text<J>(&self, url: &str, json: &J) -> crate::Result<String>
101    where
102        J: Serialize + Sync + ?Sized,
103    {
104        log::info!("POST to {url}...");
105        execute_text(self.post(url).json(json)).await
106    }
107
108    async fn fetch_as_text(&self, url: &str) -> crate::Result<String> {
109        log::info!("GET to {url}...");
110        execute_text(self.get(url)).await
111    }
112
113    async fn fetch_as_json<T: DeserializeOwned + Debug>(&self, url: &str) -> crate::Result<T> {
114        log::info!("GET to {url}...");
115        execute_json(self.get(url)).await
116    }
117
118    async fn patch_json_as_json<T, J>(&self, url: &str, json: &J) -> crate::Result<T>
119    where
120        T: DeserializeOwned + Debug,
121        J: Serialize + Sync + ?Sized,
122    {
123        log::info!("PATCH to {url}...");
124        execute_json(self.patch(url).json(json)).await
125    }
126
127    async fn patch_json_as_text<J>(&self, url: &str, json: &J) -> crate::Result<String>
128    where
129        J: Serialize + Sync + ?Sized,
130    {
131        log::info!("PATCH to {url}...");
132        execute_text(self.patch(url).json(json)).await
133    }
134}
135
136/// Builds a `HeaderMap` with common headers.
137///
138/// # Arguments
139///
140/// * `host` - The value for the `Host` header.
141/// * `with_cookie` - An optional string for the `Cookie` header. If `Some`, the cookie header will be marked as sensitive.
142///
143/// # Errors
144///
145/// Returns an [`Error`] if:
146/// - The `host` string cannot be converted into a valid `HeaderValue`.
147/// - The `with_cookie` string (if present) cannot be converted into a valid `HeaderValue`.
148fn build_headers() -> HeaderMap {
149    let mut headers = HeaderMap::with_capacity(7);
150
151    headers.insert(USER_AGENT, HeaderValue::from_static(random_user_agent()));
152    headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
153    headers.insert(HeaderName::from_static("dnt"), HeaderValue::from_static("1"));
154    headers.insert(HeaderName::from_static("sec-gpc"), HeaderValue::from_static("1"));
155
156    headers.insert(
157        HeaderName::from_static("upgrade-insecure-requests"),
158        HeaderValue::from_static("1"),
159    );
160
161    headers.insert(
162        HeaderName::from_static("sec-fetch-dest"),
163        HeaderValue::from_static("document"),
164    );
165
166    headers.insert(
167        HeaderName::from_static("sec-fetch-mode"),
168        HeaderValue::from_static("navigate"),
169    );
170
171    headers
172}
173
174async fn execute_json<T>(builder: RequestBuilder) -> crate::Result<T>
175where
176    T: DeserializeOwned + Debug,
177{
178    let builder = builder.header(ACCEPT, "application/json");
179    let resp = execute(builder).await?;
180    let data = resp.json::<T>().await?;
181    log::trace!("Response data: {data:#?}");
182    Ok(data)
183}
184
185async fn execute_text(builder: RequestBuilder) -> crate::Result<String> {
186    let resp = execute(builder).await?;
187    let body = resp.text().await?;
188    log::trace!("Response body: {body:#?}");
189    Ok(body)
190}
191
192async fn execute(builder: RequestBuilder) -> crate::Result<Response> {
193    const MAX_ATTEMPTS: u8 = 5;
194
195    let headers = build_headers();
196    let builder = builder.headers(headers);
197
198    log::trace!("builder: {builder:#?}");
199
200    for attempt in 1..=MAX_ATTEMPTS {
201        let Some(builder) = builder.try_clone() else {
202            return Err(crate::Error::NotFound("cannot clone request builder".to_owned()));
203        };
204
205        let resp = builder.send().await?;
206
207        if resp.status() == StatusCode::TOO_MANY_REQUESTS {
208            let wait = Duration::from_secs((2_u64.pow(u32::from(attempt))).min(60));
209
210            log::warn!("429 Too Many Requests. Waiting {wait:?} before retrying...");
211
212            time::sleep(wait).await;
213        } else {
214            return Ok(resp);
215        }
216    }
217
218    let resp = builder.send().await?;
219    Ok(resp)
220}
221
222#[must_use]
223pub fn random_user_agent() -> &'static str {
224    log::debug!("Spoofing user agent...");
225
226    let agents = ua::all_static_agents();
227    let index = fastrand::usize(..agents.len());
228    let ua = agents
229        .get(index)
230        .copied()
231        .unwrap_or_else(|| ua::spoof_random_agent(&mut Rng::new()));
232
233    log::trace!("Spoofed user agent: {ua}");
234
235    ua
236}
237
238#[cfg(test)]
239#[allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
240mod tests {
241    use super::*;
242
243    #[test]
244    fn random_agent_is_not_always_same() {
245        let a1 = random_user_agent();
246        let a2 = random_user_agent();
247        assert_ne!(a1, a2);
248    }
249}
kodik_utils/client_ext.rs

kodik_utils/
client_ext.rs
