spider_util/
http_client.rs

1//! Small HTTP client abstraction used by middleware.
2
3use async_trait::async_trait;
4use bytes::Bytes;
5use http::StatusCode;
6use std::time::Duration;
7
8use crate::error::SpiderError;
9
10/// Minimal HTTP client trait for middleware that needs direct fetches.
11#[async_trait]
12pub trait HttpClient: Send + Sync {
13    /// Fetches the content of a URL as text.
14    ///
15    /// # Errors
16    ///
17    /// Returns an error when the request fails, times out, or the response body
18    /// cannot be read.
19    async fn get_text(
20        &self,
21        url: &str,
22        timeout: Duration,
23    ) -> Result<(StatusCode, Bytes), SpiderError>;
24}
25
26#[async_trait]
27impl HttpClient for reqwest::Client {
28    async fn get_text(
29        &self,
30        url: &str,
31        timeout: Duration,
32    ) -> Result<(StatusCode, Bytes), SpiderError> {
33        let resp = self.get(url).timeout(timeout).send().await?;
34        let status = resp.status();
35        let body = resp.bytes().await?;
36        Ok((status, body))
37    }
38}
spider_util/http_client.rs

spider_util/
http_client.rs
