spider_lib/
downloader.rs

1use async_trait::async_trait;
2use bytes::Bytes;
3use http::StatusCode;
4use std::time::Duration;
5
6use crate::error::SpiderError;
7use crate::request::Request;
8use crate::response::Response;
9
10/// A simple HTTP client trait for fetching web content.
11#[async_trait]
12pub trait SimpleHttpClient: Send + Sync {
13    /// Fetches the content of a URL as text.
14    async fn get_text(
15        &self,
16        url: &str,
17        timeout: Duration,
18    ) -> Result<(StatusCode, Bytes), SpiderError>;
19}
20
21/// A trait for HTTP downloaders that can fetch web pages and apply middleware
22#[async_trait]
23pub trait Downloader: Send + Sync + 'static {
24    type Client: Send + Sync;
25    /// Download a web page using the provided request.
26    /// This function focuses solely on executing the HTTP request.
27    async fn download(&self, request: Request) -> Result<Response, SpiderError>;
28    /// Returns a reference to the underlying HTTP client.
29    fn client(&self) -> &Self::Client;
30}
31
32
spider_lib/downloader.rs

spider_lib/
downloader.rs
