spire_core/http/
request_ext.rs

1//! Requests and their crawl metadata.
2
3use bytes::Bytes;
4
5use super::meta::RequestMeta;
6use crate::crawler::{Attempt, Depth, Tag};
7use crate::identity::Identity;
8
9/// An HTTP request, with its body in memory.
10pub type Request<B = Bytes> = ::http::Request<B>;
11
12/// Reads crawl metadata from a [`Request`].
13///
14/// A request gets its [`Tag`], [`Depth`] and [`Attempt`] from the seed it is
15/// made from, in [`Seed::into_request`]. A request built any other way has
16/// no tag, the root depth and the first attempt; to give one metadata, e.g.
17/// to test a handler, build it from a seed:
18///
19/// ```
20/// use spire_core::frontier::seed::Seed;
21/// use spire_core::http::RequestExt;
22/// use spire_core::crawler::Tag;
23///
24/// let request = Seed::get("https://example.com/")?.with_tag("page").into_request();
25/// assert_eq!(request.tag(), Some(&Tag::from("page")));
26/// # Ok::<_, spire_core::Error>(())
27/// ```
28///
29/// [`Seed::into_request`]: crate::frontier::seed::Seed::into_request
30pub trait RequestExt {
31    /// Returns the tag, if the request has one.
32    fn tag(&self) -> Option<&Tag>;
33
34    /// Returns the depth, defaulting to [`Depth::ROOT`].
35    fn depth(&self) -> Depth;
36
37    /// Returns the attempt, defaulting to [`Attempt::FIRST`].
38    fn attempt(&self) -> Attempt;
39
40    /// Returns the identity the request goes out as, if the crawl has
41    /// [identities](crate::identity).
42    fn identity(&self) -> Option<&Identity>;
43}
44
45impl<B> RequestExt for ::http::Request<B> {
46    fn tag(&self) -> Option<&Tag> {
47        self.extensions().get::<RequestMeta>()?.tag.as_ref()
48    }
49
50    fn depth(&self) -> Depth {
51        self.extensions()
52            .get::<RequestMeta>()
53            .map(|meta| meta.depth)
54            .unwrap_or_default()
55    }
56
57    fn attempt(&self) -> Attempt {
58        self.extensions()
59            .get::<RequestMeta>()
60            .map(|meta| meta.attempt)
61            .unwrap_or_default()
62    }
63
64    fn identity(&self) -> Option<&Identity> {
65        self.extensions().get::<RequestMeta>()?.identity.as_ref()
66    }
67}
68
69#[cfg(test)]
70mod tests {
71    use super::*;
72
73    #[test]
74    fn request_metadata_defaults() {
75        let request = Request::new(());
76        assert_eq!(request.tag(), None);
77        assert_eq!(request.depth(), Depth::ROOT);
78        assert_eq!(request.attempt(), Attempt::FIRST);
79    }
80
81    #[test]
82    fn request_metadata_from_extensions() {
83        let mut request = Request::new(());
84        request.extensions_mut().insert(RequestMeta {
85            tag: Some(Tag::from_static("page")),
86            depth: Depth::new(2),
87            attempt: Attempt::FIRST,
88            identity: None,
89        });
90        assert_eq!(request.tag(), Some(&Tag::from("page")));
91        assert_eq!(request.depth().child(), Depth::new(3));
92    }
93}
spire_core/http/request_ext.rs

spire_core/http/
request_ext.rs
