spider_util/
http_client.rs

1//! HTTP Client trait for fetching web content.
2//!
3//! This module provides the `HttpClient` trait, which is a simple abstraction
4//! for HTTP clients used throughout the spider framework.
5
6use async_trait::async_trait;
7use bytes::Bytes;
8use http::StatusCode;
9use std::time::Duration;
10
11use crate::error::SpiderError;
12
13/// A simple HTTP client trait for fetching web content.
14#[async_trait]
15pub trait HttpClient: Send + Sync {
16    /// Fetches the content of a URL as text.
17    async fn get_text(
18        &self,
19        url: &str,
20        timeout: Duration,
21    ) -> Result<(StatusCode, Bytes), SpiderError>;
22}
23
24// Implement the trait for reqwest::Client
25#[async_trait]
26impl HttpClient for reqwest::Client {
27    async fn get_text(
28        &self,
29        url: &str,
30        timeout: Duration,
31    ) -> Result<(StatusCode, Bytes), SpiderError> {
32        let resp = self.get(url).timeout(timeout).send().await?;
33        let status = resp.status();
34        let body = resp.bytes().await?;
35        Ok((status, body))
36    }
37}
spider_util/http_client.rs

spider_util/
http_client.rs
