spider_downloader/
traits.rs

1//! Traits for HTTP downloaders in the `spider-lib` framework.
2
3use async_trait::async_trait;
4use bytes::Bytes;
5use http::StatusCode;
6use spider_util::error::SpiderError;
7use spider_util::request::Request;
8use spider_util::response::Response;
9#[cfg(feature = "stream")]
10use spider_util::stream_response::StreamResponse;
11use std::time::Duration;
12
13/// A simple HTTP client trait for fetching web content.
14#[async_trait]
15pub trait SimpleHttpClient: Send + Sync {
16    /// Fetches the content of a URL as text.
17    async fn get_text(
18        &self,
19        url: &str,
20        timeout: Duration,
21    ) -> Result<(StatusCode, Bytes), SpiderError>;
22}
23
24/// A trait for HTTP downloaders that can fetch web pages and apply middleware
25#[async_trait]
26pub trait Downloader: Send + Sync + 'static {
27    type Client: Send + Sync;
28
29    /// Download a web page using the provided request.
30    /// This function focuses solely on executing the HTTP request.
31    async fn download(&self, request: Request) -> Result<Response, SpiderError>;
32
33    /// Returns a reference to the underlying HTTP client.
34    fn client(&self) -> &Self::Client;
35
36    /// Download a web page as a stream response (optional feature).
37    #[cfg(feature = "stream")]
38    async fn download_stream(&self, request: Request) -> Result<StreamResponse, SpiderError> {
39        // Default implementation converts regular response to stream
40        let response = self.download(request).await?;
41        response
42            .to_stream_response()
43            .await
44            .map_err(|e| SpiderError::IoError(e.to_string()))
45    }
46}
47
spider_downloader/traits.rs

spider_downloader/
traits.rs
