spire_core/backend/
mod.rs

1//! Backends and the clients they hand out.
2//!
3//! A [`Backend`] is shared by the whole crawl and hands out one [`Client`]
4//! per request: an HTTP client, a pooled browser session, or anything else
5//! that turns a [`Request`] into a [`Response`].
6
7mod noop;
8mod status;
9
10use std::future::Future;
11
12pub use noop::Noop;
13pub use status::StatusPolicy;
14
15use crate::Result;
16use crate::request::{Request, Response};
17
18/// Hands out a [`Client`] for every request of a crawl.
19pub trait Backend: Send + Sync + 'static {
20    /// The client used to fetch a single request.
21    type Client: Client;
22
23    /// Returns a client for the next request.
24    ///
25    /// # Errors
26    ///
27    /// Returns an error if no client can be created, e.g. when a pool is
28    /// exhausted or a browser fails to start.
29    fn connect(&self) -> impl Future<Output = Result<Self::Client>> + Send;
30}
31
32/// Fetches the [`Response`] for a single [`Request`].
33pub trait Client: Send + 'static {
34    /// Fetches the response for the request, consuming the client.
35    ///
36    /// # Errors
37    ///
38    /// Returns an error if the request cannot be sent or the transfer fails.
39    fn fetch(self, request: Request) -> impl Future<Output = Result<Response>> + Send;
40}
spire_core/backend/mod.rs

spire_core/backend/
mod.rs
