jsonld_core/loader/reqwest/
mod.rs

1//! Simple document and context loader based on [`reqwest`](https://crates.io/crates/reqwest)
2use crate::{LoadError, Profile};
3
4use super::{LD_JSON_MEDIA_TYPE, Loader, RemoteDocument};
5use crate::HashSet;
6use iri_rs::{Iri, IriBuf};
7use jstrict::Parse;
8use reqwest::{
9    StatusCode,
10    header::{ACCEPT, CONTENT_TYPE, LINK},
11};
12use reqwest_middleware::ClientWithMiddleware;
13
14mod content_type;
15mod link;
16
17use content_type::ContentType;
18use link::Link;
19
20/// Loader options.
21pub struct Options {
22    /// One or more IRIs to use in the request as a profile parameter.
23    ///
24    /// (See [IANA Considerations](https://www.w3.org/TR/json-ld11/#iana-considerations)).
25    pub request_profile: Vec<Profile>,
26
27    /// Maximum number of allowed `Link` header redirections before the loader
28    /// fails.
29    ///
30    /// Defaults to 8.
31    ///
32    /// Note: this only controls how many times the loader will use a `Link`
33    /// HTTP header to find the target JSON-LD document. The number of allowed
34    /// regular HTTP redirections is controlled by the HTTP
35    /// [`client`](Self::client).
36    pub max_redirections: usize,
37
38    /// Maximum size of a loaded document, in bytes.
39    ///
40    /// Responses larger than this fail with [`Error::TooLarge`] instead of
41    /// buffering unbounded amounts of memory. Defaults to 32 MiB. Set to
42    /// `None` to disable the limit.
43    pub max_document_size: Option<usize>,
44
45    /// Timeout applied to each HTTP request, from connection to the end of
46    /// the response body.
47    ///
48    /// Defaults to 60 seconds, so a hung server cannot stall loading
49    /// indefinitely. Set to `None` to defer to the [`client`](Self::client)
50    /// configuration.
51    pub timeout: Option<std::time::Duration>,
52
53    /// HTTP client.
54    pub client: ClientWithMiddleware,
55}
56
57impl Default for Options {
58    fn default() -> Self {
59        Self {
60            request_profile: Vec::new(),
61            max_redirections: 8,
62            max_document_size: Some(32 * 1024 * 1024),
63            timeout: Some(std::time::Duration::from_mins(1)),
64            client: reqwest_middleware::ClientBuilder::new(reqwest::Client::default()).build(),
65        }
66    }
67}
68
69/// Loading error.
70#[derive(Debug, thiserror::Error)]
71pub enum Error {
72    #[error("HTTP request failed: {0}")]
73    /// The HTTP request could not be completed — a connection, TLS, timeout
74    /// or middleware failure.
75    Reqwest(reqwest_middleware::Error),
76
77    #[error("query failed: status code {0}")]
78    /// The server answered with a non-success status code.
79    QueryFailed(StatusCode),
80
81    #[error("invalid content type")]
82    /// The response `Content-Type` is neither `application/ld+json` nor any
83    /// other JSON media type the loader accepts.
84    InvalidContentType,
85
86    #[error("multiple context link headers")]
87    /// The response carried more than one `Link` header pointing at a
88    /// JSON-LD context, leaving the target document ambiguous.
89    MultipleContextLinkHeaders,
90
91    #[error("too many redirections")]
92    /// The loader followed as many `Link` headers as `Options::max_redirections`
93    /// allows without reaching a JSON-LD document.
94    TooManyRedirections,
95
96    #[error("document exceeds the size limit ({0} bytes)")]
97    /// The response body exceeded the `Options::max_document_size` limit.
98    /// Carries that limit, in bytes.
99    TooLarge(usize),
100
101    #[error("JSON parse error: {0}")]
102    /// The response body is not well-formed JSON.
103    Parse(jstrict::parse::Error<utf8_decode::Utf8Error>),
104}
105
106/// `reqwest`-based loader.
107///
108/// Only works with the [`tokio`](https://tokio.rs/) runtime.
109///
110/// Follows HTTP redirections and JSON-LD `Link` headers, up to
111/// [`Options::max_redirections`].
112///
113/// Loaded documents are not cached: a new network request is made each time a
114/// URL is loaded, even if it has been fetched before.
115pub struct ReqwestLoader {
116    options: Options,
117    accept_header: String,
118}
119
120impl Default for ReqwestLoader {
121    fn default() -> Self {
122        Self::new_using(Options::default())
123    }
124}
125
126impl ReqwestLoader {
127    /// Creates a new loader with the default options.
128    #[must_use]
129    pub fn new() -> Self {
130        Self::default()
131    }
132
133    /// Creates a new loader with the given options.
134    #[must_use]
135    pub fn new_using(options: Options) -> Self {
136        let mut json_ld_params = String::new();
137
138        if !options.request_profile.is_empty() {
139            json_ld_params.push_str("; profile=");
140
141            if options.request_profile.len() > 1 {
142                json_ld_params.push('"');
143            }
144
145            for (i, p) in options.request_profile.iter().enumerate() {
146                if i > 0 {
147                    json_ld_params.push(' ');
148                }
149
150                json_ld_params.push_str(p.iri().as_str());
151            }
152
153            if options.request_profile.len() > 1 {
154                json_ld_params.push('"');
155            }
156        }
157
158        Self {
159            options,
160            accept_header: format!("application/ld+json{json_ld_params}, application/json"),
161        }
162    }
163}
164
165impl Loader for ReqwestLoader {
166    type Error = Error;
167
168    async fn load(&self, url: Iri<&str>) -> Result<RemoteDocument<IriBuf>, LoadError<Self::Error>> {
169        let mut redirection_number = 0;
170        let mut url: IriBuf = url.into();
171        'next_url: loop {
172            if redirection_number > self.options.max_redirections {
173                return Err(LoadError::new(url.clone(), Error::TooManyRedirections));
174            }
175
176            log::debug!("downloading: {url}");
177            let mut request = self.options.client.get(url.as_str()).header(ACCEPT, &self.accept_header);
178
179            if let Some(timeout) = self.options.timeout {
180                request = request.timeout(timeout);
181            }
182
183            let mut response = request.send().await.map_err(|e| LoadError::new(url.clone(), Error::Reqwest(e)))?;
184
185            match response.status() {
186                StatusCode::OK => {
187                    let mut content_types = response.headers().get_all(CONTENT_TYPE).into_iter().filter_map(ContentType::new);
188
189                    if let Some(content_type) = content_types.find(ContentType::is_json_ld) {
190                        let mut context_url = None;
191                        if *content_type.media_type() != LD_JSON_MEDIA_TYPE {
192                            for link in response.headers().get_all(LINK).into_iter().flat_map(Link::parse_header) {
193                                if link.rel() == Some(b"http://www.w3.org/ns/json-ld#context") {
194                                    if context_url.is_some() {
195                                        return Err(LoadError::new(url, Error::MultipleContextLinkHeaders));
196                                    }
197
198                                    if let Ok(resolved) = link.href().resolved(&url)
199                                        && let Ok(iri) = IriBuf::try_from(resolved)
200                                    {
201                                        context_url = Some(iri);
202                                    }
203                                }
204                            }
205                        }
206
207                        let mut profile = HashSet::default();
208                        for p in content_type.profile().into_iter().flat_map(|p| p.split(|b| *b == b' ')) {
209                            if let Ok(p) = std::str::from_utf8(p)
210                                && let Ok(iri) = Iri::parse(p)
211                            {
212                                profile.insert(Profile::new(iri));
213                            }
214                        }
215
216                        if let (Some(limit), Some(len)) = (self.options.max_document_size, response.content_length())
217                            && len > limit as u64
218                        {
219                            return Err(LoadError::new(url, Error::TooLarge(limit)));
220                        }
221
222                        let mut bytes = Vec::new();
223                        while let Some(chunk) = response.chunk().await.map_err(|e| LoadError::new(url.clone(), Error::Reqwest(e.into())))? {
224                            if let Some(limit) = self.options.max_document_size
225                                && bytes.len() + chunk.len() > limit
226                            {
227                                return Err(LoadError::new(url, Error::TooLarge(limit)));
228                            }
229
230                            bytes.extend_from_slice(&chunk);
231                        }
232
233                        let decoder = utf8_decode::Decoder::new(bytes.iter().copied());
234                        let (document, _) = jstrict::Value::parse_utf8(decoder).map_err(|e| LoadError::new(url.clone(), Error::Parse(e)))?;
235
236                        break Ok(RemoteDocument::new_full(
237                            Some(url),
238                            Some(content_type.into_media_type()),
239                            context_url,
240                            profile,
241                            document,
242                        ));
243                    }
244                    log::debug!("no valid media type found");
245                    for link in response.headers().get_all(LINK).into_iter().flat_map(Link::parse_header) {
246                        if link.rel() == Some(b"alternate") && link.type_() == Some(b"application/ld+json") {
247                            log::debug!("link found");
248                            if let Ok(resolved) = link.href().resolved(&url)
249                                && let Ok(next) = IriBuf::try_from(resolved)
250                            {
251                                url = next;
252                                redirection_number += 1;
253                                continue 'next_url;
254                            }
255                        }
256                    }
257
258                    break Err(LoadError::new(url, Error::InvalidContentType));
259                }
260                code => break Err(LoadError::new(url, Error::QueryFailed(code))),
261            }
262        }
263    }
264}
jsonld_core/loader/reqwest/mod.rs

jsonld_core/loader/reqwest/
mod.rs
