spire_core/backend/
mod.rs

1//! Backends and the sessions they hand out.
2//!
3//! A [`Backend`] is shared by the whole crawl and hands out one [`Session`]
4//! per request: an HTTP client, a pooled browser session, or anything else
5//! that turns a [`Request`] into a [`Response`].
6
7pub mod document;
8mod noop;
9mod status;
10
11use std::future::Future;
12
13pub use noop::Noop;
14pub use status::StatusPolicy;
15
16use crate::Result;
17use crate::http::{Request, Response};
18use crate::identity::Identity;
19
20/// Hands out a [`Session`] for every request of a crawl.
21pub trait Backend: Send + Sync + 'static {
22    /// The session used to fetch a single request.
23    type Session: Session;
24
25    /// Returns a session for the next request, sent directly, from the
26    /// crawler itself, with the backend's own cookies and sessions.
27    ///
28    /// # Errors
29    ///
30    /// Returns an error if no session can be created, e.g. when a pool is
31    /// exhausted or a browser fails to start.
32    fn connect(&self) -> impl Future<Output = Result<Self::Session>> + Send;
33
34    /// Returns a session for the next request, which goes out as `identity`,
35    /// for a crawl with [identities](crate::identity).
36    ///
37    /// The session must send its request through the identity's
38    /// [proxy](Identity::proxy), and keep each identity's cookies and
39    /// session state apart from every other's, so that one identity always
40    /// looks like one visitor. The identity's [headers](Identity::headers)
41    /// are already on the request.
42    ///
43    /// By default, an identity without a proxy connects as
44    /// [`connect`](Self::connect) does, and one with a proxy is refused, so
45    /// a request is never sent from the crawler's own IP by mistake.
46    /// Backends that route through proxies, or keep cookies or sessions,
47    /// override it.
48    ///
49    /// # Errors
50    ///
51    /// Returns an [`Invalid`](crate::ErrorKind::Invalid) error if the
52    /// identity's proxy is not supported, or an error if no session can be
53    /// created.
54    fn connect_as(
55        &self,
56        identity: &Identity,
57    ) -> impl Future<Output = Result<Self::Session>> + Send {
58        let supported = identity.require_no_proxy();
59        async move {
60            supported?;
61            self.connect().await
62        }
63    }
64}
65
66/// Fetches one request using a backend's resources.
67///
68/// A successful fetch returns a complete response with an immutable `Bytes`
69/// body. HTTP sessions return the received bytes; browser sessions return
70/// an HTML snapshot captured after navigation. A live browser document is
71/// a separate optional capability, accessed through [`Self::document`].
72/// [`Context`](crate::Context) caches the first successful response.
73pub trait Session: Send + Sync + 'static {
74    /// Fetches `request`, returning its status, headers, final URI and body.
75    ///
76    /// # Errors
77    ///
78    /// Returns an error if fetching, status checking or reading the body fails.
79    fn fetch(&self, request: &Request) -> impl Future<Output = Result<Response>> + Send;
80
81    /// Returns the live document capability, if this session has a DOM.
82    ///
83    /// This reports support, not navigation state. Operations on the
84    /// document may fail before a page is fetched. Fetch through the
85    /// context before querying it; capture failures must not silently
86    /// fall back to parsing the response snapshot.
87    fn document(&self) -> Option<&dyn document::Document> {
88        None
89    }
90}
spire_core/backend/mod.rs

spire_core/backend/
mod.rs
