scraper_trail/
exchange.rs

1use crate::{multi_value::MultiValue, request::Request};
2use bounded_static::{IntoBoundedStatic, ToBoundedStatic};
3use std::borrow::Cow;
4use std::collections::HashMap;
5use std::path::{Path, PathBuf};
6
7#[derive(Debug, thiserror::Error)]
8pub enum Error {
9    #[error("URL parse error")]
10    UrlParse(#[from] url::ParseError),
11    #[error("Invalid request header value")]
12    RequestHeaderValue(#[from] http::header::InvalidHeaderValue),
13    #[error("Invalid response header value")]
14    ResponseHeaderValue(#[from] http::header::ToStrError),
15}
16
17#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
18pub struct Exchange<'a, T> {
19    #[serde(borrow)]
20    pub request: Request<'a>,
21    pub response: Response<'a, T>,
22}
23
24impl<'a, T> Exchange<'a, T> {
25    pub fn map<U, F: FnOnce(T) -> U>(self, f: F) -> Exchange<'a, U> {
26        Exchange {
27            request: self.request,
28            response: self.response.map(f),
29        }
30    }
31}
32
33impl<'a, T: IntoBoundedStatic + 'a> IntoBoundedStatic for Exchange<'a, T> {
34    type Static = Exchange<'static, T::Static>;
35
36    fn into_static(self) -> Self::Static {
37        Self::Static {
38            request: self.request.into_static(),
39            response: self.response.into_static(),
40        }
41    }
42}
43
44impl<T: ToBoundedStatic> ToBoundedStatic for Exchange<'_, T> {
45    type Static = Exchange<'static, T::Static>;
46
47    fn to_static(&self) -> Self::Static {
48        Self::Static {
49            request: self.request.to_static(),
50            response: self.response.to_static(),
51        }
52    }
53}
54
55impl<T: serde::ser::Serialize> Exchange<'_, T> {
56    pub fn save_file<P: AsRef<Path>>(&self, base: P) -> Result<PathBuf, std::io::Error> {
57        std::fs::create_dir_all(&base)?;
58
59        let output_path = base.as_ref().join(format!(
60            "{}.json",
61            self.request.timestamp.timestamp_millis()
62        ));
63
64        // We assume serialization failures are rare and don't need a separate error
65        // representation.
66        let json = serde_json::to_string(self).map_err(std::io::Error::other)?;
67
68        std::fs::write(&output_path, json)?;
69
70        Ok(output_path)
71    }
72}
73
74#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
75pub struct Response<'a, T> {
76    #[serde(borrow)]
77    pub headers: HashMap<Cow<'a, str>, MultiValue<'a>>,
78    pub data: T,
79}
80
81impl<'a, T> Response<'a, T> {
82    pub fn map<U, F: FnOnce(T) -> U>(self, f: F) -> Response<'a, U> {
83        Response {
84            headers: self.headers,
85            data: f(self.data),
86        }
87    }
88
89    pub fn and_then<U, E, F: FnOnce(T) -> Result<U, E>>(self, f: F) -> Result<Response<'a, U>, E> {
90        f(self.data).map(|new_data| Response {
91            headers: self.headers,
92            data: new_data,
93        })
94    }
95}
96
97impl<'a, T: IntoBoundedStatic + 'a> IntoBoundedStatic for Response<'a, T> {
98    type Static = Response<'static, T::Static>;
99
100    fn into_static(self) -> Self::Static {
101        Self::Static {
102            headers: self
103                .headers
104                .into_iter()
105                .map(|(key, values)| (key.into_static(), values.into_static()))
106                .collect(),
107            data: self.data.into_static(),
108        }
109    }
110}
111
112impl<T: ToBoundedStatic> ToBoundedStatic for Response<'_, T> {
113    type Static = Response<'static, T::Static>;
114
115    fn to_static(&self) -> Self::Static {
116        Self::Static {
117            headers: self
118                .headers
119                .iter()
120                .map(|(key, values)| (key.to_static(), values.to_static()))
121                .collect(),
122            data: self.data.to_static(),
123        }
124    }
125}
126
127#[cfg(test)]
128mod tests {
129    use super::Exchange;
130
131    const APPLE_ITUNES_01_EXAMPLE: &str = include_str!("../../examples/apple-itunes-01.json");
132    const GOOGLE_PLAY_01_EXAMPLE: &str = include_str!("../../examples/google-play-01.json");
133
134    #[test]
135    fn deserialize_example_apple_itunes_01() -> Result<(), Box<dyn std::error::Error>> {
136        let example: Exchange<'_, serde_json::Value> =
137            serde_json::from_str(APPLE_ITUNES_01_EXAMPLE)?;
138
139        assert!(
140            example
141                .request
142                .url
143                .as_str()
144                .starts_with("https://itunes.apple.com/lookup")
145        );
146
147        assert_eq!(
148            example.request.timestamp,
149            chrono::DateTime::from_timestamp_millis(1760252742866).unwrap()
150        );
151
152        Ok(())
153    }
154
155    #[test]
156    fn deserialize_example_google_play_01() -> Result<(), Box<dyn std::error::Error>> {
157        let example: Exchange<'_, serde_json::Value> =
158            serde_json::from_str(GOOGLE_PLAY_01_EXAMPLE)?;
159
160        assert!(
161            example
162                .request
163                .url
164                .as_str()
165                .starts_with("https://play.google.com/_/PlayStoreUi/data/")
166        );
167
168        assert_eq!(
169            example.request.timestamp,
170            chrono::DateTime::from_timestamp_millis(1759391955666).unwrap()
171        );
172
173        Ok(())
174    }
175}
scraper_trail/exchange.rs

scraper_trail/
exchange.rs
