scrapr_bindings/
request.rs

1use pyo3::prelude::*;
2use std::collections::HashMap;
3
4#[pyclass]
5#[derive(Clone, Debug, Default)]
6pub struct RequestOptions {
7    #[pyo3(get, set)]
8    pub headers: HashMap<String, String>,
9
10    #[pyo3(get, set)]
11    pub cookies: HashMap<String, String>,
12
13    #[pyo3(get, set)]
14    pub query: HashMap<String, String>,
15}
16
17// impl<'source> FromPyObject<'source> for RequestOptions {
18//     fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult<Self> {
19//         let headers = ob
20//             .getattr("headers")?
21//             .extract::<HashMap<String, String>>()?;
22//         let cookies = ob
23//             .getattr("cookies")?
24//             .extract::<HashMap<String, String>>()?;
25//         let query = ob.getattr("query")?.extract::<HashMap<String, String>>()?;
26//         Ok(RequestOptions {
27//             headers,
28//             cookies,
29//             query,
30//         })
31//     }
32// }
33
34#[pymethods]
35impl RequestOptions {
36    #[new]
37    fn new(
38        headers: Option<HashMap<String, String>>,
39        cookies: Option<HashMap<String, String>>,
40        query: Option<HashMap<String, String>>,
41    ) -> Self {
42        RequestOptions {
43            headers: headers.unwrap_or_default(),
44            cookies: cookies.unwrap_or_default(),
45            query: query.unwrap_or_default(),
46        }
47    }
48}
49
50pub fn build_url(base: &str, path: &str, query: &HashMap<String, String>) -> String {
51    let mut url = format!("{base}{path}");
52    if !query.is_empty() {
53        let query_string = query
54            .iter()
55            .map(|(k, v)| format!("{}={}", k, v))
56            .collect::<Vec<_>>()
57            .join("&");
58        url.push('?');
59        url.push_str(&query_string);
60    }
61    url
62}
63
64pub fn format_headers(host: &str, options: &RequestOptions) -> String {
65    let mut headers = vec![
66        format!("Host: {host}"),
67        "Connection: close".to_string(),
68        "User-Agent: Scraper/0.1".to_string(),
69    ];
70
71    for (k, v) in &options.headers {
72        headers.push(format!("{k}: {v}"));
73    }
74
75    if !options.cookies.is_empty() {
76        let cookie_string = options
77            .cookies
78            .iter()
79            .map(|(k, v)| format!("{k}: {v}"))
80            .collect::<Vec<_>>()
81            .join("; ");
82        headers.push(format!("Cookie: {cookie_string}"));
83    }
84
85    headers.join("\r\n")
86}
scrapr_bindings/request.rs

scrapr_bindings/
request.rs
