scrapr_core/
request.rs

1use std::collections::HashMap;
2
3/// Options for customizing an HTTP request.
4///
5/// Includes optional headers, cookies, and query parameters.
6#[derive(Clone, Debug, Default)]
7pub struct RequestOptions {
8    /// Custom HTTP headers to send with the request.
9    pub headers: HashMap<String, String>,
10
11    /// Key-value pairs representing cookies.
12    pub cookies: HashMap<String, String>,
13
14    /// Query string parameters to include in the URL.
15    pub query: HashMap<String, String>,
16}
17
18
19
20impl RequestOptions {
21    /// Creates a new `RequestOptions` instance from optional headers, cookies, and query params.
22    ///
23    /// # Arguments
24    ///
25    /// * `headers` - Optional map of header names to values.
26    /// * `cookies` - Optional map of cookie names to values.
27    /// * `query` - Optional map of query parameter names to values.
28    pub fn new(
29        headers: Option<HashMap<String, String>>,
30        cookies: Option<HashMap<String, String>>,
31        query: Option<HashMap<String, String>>,
32    ) -> Self {
33        RequestOptions {
34            headers: headers.unwrap_or_default(),
35            cookies: cookies.unwrap_or_default(),
36            query: query.unwrap_or_default(),
37        }
38    }
39}
40
41/// Constructs a URL string with an optional base, path, and query parameters.
42///
43/// # Arguments
44///
45/// * `base` - Optional base URL prefix (can be empty).
46/// * `path` - The resource path (e.g., `/index.html`).
47/// * `query` - A map of query parameters to append to the URL.
48///
49/// # Returns
50///
51/// A full URL with query string if applicable.
52pub fn build_url(base: &str, path: &str, query: &HashMap<String, String>) -> String {
53    let mut url = format!("{base}{path}");
54    if !query.is_empty() {
55        let query_string = query
56            .iter()
57            .map(|(k, v)| format!("{}={}", k, v))
58            .collect::<Vec<_>>()
59            .join("&");
60        url.push('?');
61        url.push_str(&query_string);
62    }
63    url
64}
65
66/// Formats HTTP headers into a request-ready string.
67///
68/// # Arguments
69///
70/// * `host` - The host name to include in the `Host` header.
71/// * `options` - The `RequestOptions` containing custom headers and cookies.
72///
73/// # Returns
74///
75/// A formatted string of HTTP headers.
76pub fn format_headers(host: &str, options: &RequestOptions) -> String {
77    let mut headers = vec![
78        format!("Host: {host}"),
79        "Connection: close".to_string(),
80        "User-Agent: Scraper/0.1".to_string(),
81    ];
82
83    for (k, v) in &options.headers {
84        headers.push(format!("{k}: {v}"));
85    }
86
87    if !options.cookies.is_empty() {
88        // Cookies should be formatted as `key=value` pairs separated by "; "
89        let cookie_string = options
90            .cookies
91            .iter()
92            .map(|(k, v)| format!("{k}={v}"))
93            .collect::<Vec<_>>()
94            .join("; ");
95        headers.push(format!("Cookie: {cookie_string}"));
96    }
97
98    headers.join("\r\n")
99}
scrapr_core/request.rs

scrapr_core/
request.rs
