spider_downloader/lib.rs
1//! # spider-downloader
2//!
3//! Downloader traits and the default reqwest-based implementation used by the
4//! crawler runtime.
5//!
6//! Depend on this crate directly when you want transport-level control without
7//! reworking the request and response types used by the rest of the workspace.
8//!
9//! ## Example
10//!
11//! ```rust,ignore
12//! use spider_downloader::Downloader;
13//! use spider_util::{request::Request, response::Response, error::SpiderError};
14//! use async_trait::async_trait;
15//!
16//! struct MyDownloader {
17//! client: reqwest::Client,
18//! }
19//!
20//! #[async_trait]
21//! impl Downloader for MyDownloader {
22//! type Client = reqwest::Client;
23//! async fn download(&self, _request: Request) -> Result<Response, SpiderError> {
24//! todo!()
25//! }
26//! fn client(&self) -> &Self::Client {
27//! &self.client
28//! }
29//! }
30//! ```
31
32mod client;
33mod traits;
34
35pub use client::ReqwestClientDownloader;
36pub use traits::{Downloader, HttpClient};