spider_downloader/
traits.rs

1//! Traits for HTTP downloaders in the `spider-lib` framework.
2
3use async_trait::async_trait;
4use bytes::Bytes;
5use http::StatusCode;
6use spider_util::error::SpiderError;
7use spider_util::request::Request;
8use spider_util::response::Response;
9use std::time::Duration;
10
11/// A simple HTTP client trait for fetching web content.
12#[async_trait]
13pub trait SimpleHttpClient: Send + Sync {
14    /// Fetches the content of a URL as text.
15    async fn get_text(
16        &self,
17        url: &str,
18        timeout: Duration,
19    ) -> Result<(StatusCode, Bytes), SpiderError>;
20}
21
22/// A trait for HTTP downloaders that can fetch web pages and apply middleware
23#[async_trait]
24pub trait Downloader: Send + Sync + 'static {
25    type Client: Send + Sync;
26
27    /// Download a web page using the provided request.
28    /// This function focuses solely on executing the HTTP request.
29    async fn download(&self, request: Request) -> Result<Response, SpiderError>;
30
31    /// Returns a reference to the underlying HTTP client.
32    fn client(&self) -> &Self::Client;
33}
spider_downloader/traits.rs

spider_downloader/
traits.rs
